source: trunk/email2trac.py.in @ 606

Last change on this file since 606 was 606, checked in by bas, 12 years ago

added strip_signature_regex option, closes #296, #155

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 85.5 KB
Line 
1#!@PYTHON@
2#
3# This file is part of the email2trac utils
4#
5#       Copyright 2002 SARA
6#
7#       Licensed under the Apache License, Version 2.0 (the "License");
8#       you may not use this file except in compliance with the License.
9#       You may obtain a copy of the License at
10#
11#       http://www.apache.org/licenses/LICENSE-2.0
12#
13#       Unless required by applicable law or agreed to in writing, software
14#       distributed under the License is distributed on an "AS IS" BASIS,
15#       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16#       See the License for the specific language governing permissions and
17#       limitations under the License.
18#
19# For vi/emacs or other use tabstop=4 (vi: set ts=4)
20#
21"""
22email2trac.py -- Email -> TRAC tickets
23
24A MTA filter to create Trac tickets from inbound emails.
25
26first proof of concept from:
27 Copyright 2005, Daniel Lundin <daniel@edgewall.com>
28 Copyright 2005, Edgewall Software
29
30Authors:
31  Bas van der Vlies <basv@sara.nl>
32  Walter de Jong <walter@sara.nl>
33
34How to use
35----------
36 * See https://subtrac.sara.nl/oss/email2trac/
37
38 * Commandline opions:
39                -h,--help
40                -d, --debug
41                -f,--file  <configuration file>
42                -n,--dry-run
43                -p, --project <project name>
44                -t, --ticket_prefix <name>
45
46SVN Info:
47        $Id: email2trac.py.in 606 2012-08-30 12:31:39Z bas $
48"""
49import os
50import sys
51import string
52import getopt
53import time
54import email
55import email.Iterators
56import email.Header
57import re
58import urllib
59import unicodedata
60import mimetypes
61import traceback
62import logging
63import logging.handlers
64import UserDict
65import tempfile
66
67from datetime import timedelta, datetime
68from stat import *
69
70try:
71    from trac import __version__ as trac_version
72    from trac import config as trac_config
73
74except ImportError, detail:
75    print "Can not find a a valid trac installation, try setting PYTHONPATH"
76    sys.exit(1)
77
78## Some global variables
79#
80m = None
81
82# This is to for the function AlwaysNotifyReporter
83sender_email = None
84
85class SaraDict(UserDict.UserDict):
86    def __init__(self, dictin = None):
87        UserDict.UserDict.__init__(self)
88        self.name = None
89       
90        if dictin:
91            if dictin.has_key('name'):
92                self.name = dictin['name']
93                del dictin['name']
94            self.data = dictin
95           
96    def get_value(self, name):
97        if self.has_key(name):
98            return self[name]
99        else:
100            return None
101               
102    def __repr__(self):
103        return repr(self.data)
104
105    def __str__(self):
106        return str(self.data)
107           
108    def __getattr__(self, name):
109        """
110        override the class attribute get method. Return the value
111        from the dictionary
112        """
113        if self.data.has_key(name):
114            return self.data[name]
115        else:
116            return None
117           
118    def __setattr__(self, name, value):
119        """
120        override the class attribute set method only when the UserDict
121        has set its class attribute
122        """
123        if self.__dict__.has_key('data'):
124            self.data[name] = value
125        else:
126            self.__dict__[name] = value
127
128    def __iter__(self):
129        return iter(self.data.keys())
130
131class TicketEmailParser(object):
132    env = None
133    comment = '> '
134
135    def __init__(self, env, parameters, logger, version):
136        self.env = env
137
138        # Database connection
139        #
140        self.db = None
141
142        # Save parameters
143        #
144        self.parameters = parameters
145        self.logger = logger
146
147        # Some useful mail constants
148        #
149        self.email_name = None
150        self.email_addr = None
151        self.email_from = None
152        self.author     = None
153        self.id         = None
154
155        ## Will be set to True if the user has ben registered
156        #
157        self.allow_registered_user = False
158       
159        self.STRIP_CONTENT_TYPES = list()
160
161        ## fields properties via body_text
162        #
163        self.properties = dict()
164
165        self.VERSION = version
166
167        self.get_config = self.env.config.get
168
169        ## init function ##
170        #
171        self.setup_parameters()
172
173    def setup_parameters(self):
174        if self.parameters.umask:
175            os.umask(self.parameters.umask)
176
177        if not self.parameters.spam_level:
178            self.parameters.spam_level = 0
179
180        if not self.parameters.spam_header:
181            self.parameters.spam_header = 'X-Spam-Score'
182
183        if not self.parameters.email_quote:
184            self.parameters.email_quote = '> '
185
186        if not self.parameters.ticket_update_by_subject_lookback:
187            self.parameters.ticket_update_by_subject_lookback = 30
188
189        if self.parameters.verbatim_format == None:
190            self.parameters.verbatim_format = 1
191
192        if self.parameters.reflow == None:
193            self.parameters.reflow = 1
194
195        if self.parameters.binhex:
196            self.STRIP_CONTENT_TYPES.append('application/mac-binhex40')
197
198        if self.parameters.applesingle:
199            self.STRIP_CONTENT_TYPES.append('application/applefile')
200
201        if self.parameters.appledouble:
202            self.STRIP_CONTENT_TYPES.append('application/applefile')
203
204        if self.parameters.strip_content_types:
205            items = self.parameters.strip_content_types.split(',')
206            for item in items:
207                self.STRIP_CONTENT_TYPES.append(item.strip())
208
209        if self.parameters.tmpdir:
210            self.parameters.tmpdir = os.path.normcase(str(self.parameters['tmpdir']))
211        else:
212            self.parameters.tmpdir = os.path.normcase('/tmp')
213
214        if self.parameters.email_triggers_workflow == None:
215            self.parameters.email_triggers_workflow = 1
216
217        if not self.parameters.subject_field_separator:
218            self.parameters.subject_field_separator = '&'
219        else:
220            self.parameters.subject_field_separator = self.parameters.subject_field_separator.strip()
221
222        if not self.parameters.strip_signature_regex:
223            self.parameters.strip_signature_regex = '^-----Original Message-----$|^-- $'
224
225
226        self.trac_smtp_from = self.get_config('notification', 'smtp_from')
227        self.smtp_default_domain = self.get_config('notification', 'smtp_default_domain')
228
229
230        self.system = None
231
232########## Email Header Functions ###########################################################
233
234    def spam(self, message):
235        """
236        # X-Spam-Score: *** (3.255) BAYES_50,DNS_FROM_AHBL_RHSBL,HTML_
237        # Note if Spam_level then '*' are included
238        """
239        self.logger.debug('function spam')
240
241        spam = False
242        if message.has_key(self.parameters.spam_header):
243            spam_l = string.split(message[self.parameters.spam_header])
244            #self.logger.info('Spam header: %s' %(message[self.parameters.spam_header]))
245
246            try:
247                number = spam_l[0].count('*')
248            except IndexError, detail:
249                number = 0
250               
251            if number >= self.parameters.spam_level:
252                spam = True
253               
254        ## treat virus mails as spam
255        #
256        elif message.has_key('X-Virus-found'):         
257            spam = True
258
259        ## Astaro firewall spam handling
260        #
261        elif message.get('X-Spam-Flag') == "YES" and message.get('X-Spam-Result') == "Spam":
262            spam = True
263
264        ## How to handle SPAM messages
265        #
266        if self.parameters.drop_spam and spam:
267
268            self.logger.info('Message is a SPAM. Automatic ticket insertion refused (SPAM level > %d)' %self.parameters.spam_level)
269            return 'drop'   
270
271        elif spam:
272
273            return 'Spam'   
274        else:
275
276            return False
277
278    def email_header_acl(self, keyword, message_field, default):
279        """
280        This function wil check if the email address is allowed or denied
281        to send mail to the ticket list
282        """
283        self.logger.debug('function email_header_acl: %s' %keyword)
284
285        try:
286            mail_addresses = self.parameters[keyword]
287
288            # Check if we have an empty string
289            #
290            if not mail_addresses:
291                return default
292
293        except KeyError, detail:
294            self.logger.debug('\t %s not defined, all messages are allowed.' %(keyword))
295
296            return default
297
298        mail_addresses = string.split(mail_addresses, ',')
299        message_addresses = string.split(message_field, ',')
300
301        for entry in mail_addresses:
302            entry = entry.strip()
303            TO_RE = re.compile(entry, re.VERBOSE|re.IGNORECASE)
304
305            for addr in message_addresses:
306                addr = addr.strip()
307
308                if self.parameters.compare_function_list in [ 'matches', 'match']:
309                    s = '\t%s matches %s' %(addr, entry)
310                    result =  TO_RE.match(addr)
311                else:
312                    s = '\t%s contains %s' %(addr, entry)
313                    result =  TO_RE.search(addr)
314
315                if result:
316                    self.logger.debug(s)
317                    return True
318
319        return False
320
321    def email_header_txt(self, m):
322        """
323        Display To and CC addresses in description field
324        """
325        s = ''
326
327        if m['To'] and len(m['To']) > 0:
328            s = "'''To:''' %s\r\n" %(m['To'])
329        if m['Cc'] and len(m['Cc']) > 0:
330            s = "%s'''Cc:''' %s\r\n" % (s, m['Cc'])
331
332        return  self.email_to_unicode(s)
333
334
335    def get_sender_info(self, message):
336        """
337        Get the default author name and email address from the message
338        """
339        self.logger.debug('function get_sender_info')
340
341        to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
342        self.email_to_addrs = list()
343
344        for n,e in to_addrs:
345            self.email_to_addrs.append(e)
346
347        self.email_to_addrs = ','.join(self.email_to_addrs)
348
349
350        self.email_name, self.email_addr  = email.Utils.parseaddr(message['from'])
351
352        ## decode email name can contain charset
353        #
354        self.email_name = self.email_to_unicode(self.email_name)
355
356        dstr = '\t email name: %s, email address: %s' %(self.email_name, self.email_addr) 
357        self.logger.debug(dstr)
358
359        ## Trac can not handle author's name that contains spaces
360        #
361        if self.email_addr.lower() == self.trac_smtp_from.lower():
362            if self.email_name:
363                self.author = self.email_name
364            else:
365                self.author = "email2trac"
366        else:
367            self.author = self.email_addr
368
369        if self.parameters.ignore_trac_user_settings:
370            return
371
372        # Is this a registered user, use email address as search key:
373        # result:
374        #   u : login name
375        #   n : Name that the user has set in the settings tab
376        #   e : email address that the user has set in the settings tab
377        #
378        users = [ (u,n,e) for (u, n, e) in self.env.get_known_users(self.db)
379            if (
380                (e and (e.lower() == self.email_addr.lower())) or
381                (u.lower() + '@' + self.smtp_default_domain.lower() == self.email_addr.lower())
382            )
383            ]
384
385        if len(users) >= 1:
386            self.email_from = users[0][2]
387            self.author = users[0][0]
388
389            self.logger.debug('\tget_sender_info: found registered user: %s' %(self.author) )
390
391            if self.parameters.white_list_registered_users:
392                self.logger.debug('\tget_sender_info: added registered user to white list')
393                self.allow_registered_user = True
394                   
395
396    def set_cc_fields(self, ticket, message, update=False):
397        """
398        Set all the right fields for a new ticket
399        """
400        self.logger.debug('function set_cc_fields')
401
402
403        # Put all CC-addresses in ticket CC field
404        #
405        if self.parameters.reply_all:
406
407            email_cc = ''
408
409            cc_addrs = email.Utils.getaddresses( message.get_all('cc', []) )
410
411            if not cc_addrs:
412                return
413
414            ## Build a list of forbidden CC addresses
415            #
416            #to_addrs = email.Utils.getaddresses( message.get_all('to', []) )
417            #to_list = list()
418            #for n,e in to_addrs:
419            #   to_list.append(e)
420
421            if update:
422                self.logger.debug("\tupdate cc-fields")
423
424            for name,addr in cc_addrs:
425       
426                ## Prevent mail loop
427                #
428                if addr == self.trac_smtp_from:
429                    self.logger.debug("\tSkipping %s email address for CC-field, same as smtp_from address in trac.ini " %(addr))
430                    continue
431
432                ## Alwyas remove reporter email address
433                #
434                elif addr == self.email_addr:
435                    self.logger.debug("\tSkipping reporter email address for CC-field")
436                    continue
437
438                else:
439                    if email_cc:
440                        email_cc = '%s, %s' %(email_cc, addr)
441                    else:
442                        email_cc = addr
443
444            if email_cc:
445                self.logger.debug('\tCC fields: %s' %email_cc)
446                ticket['cc'] = self.email_to_unicode(email_cc)
447
448
449    def acl_list_from_file(self, f, keyword):
450        """
451        Read the email address from a file
452        """
453        self.logger.debug('function acl_list_from_file %s : %s' %(f, keyword))
454       
455        if  not os.path.isfile(f):
456            self.logger.error('%s_file: %s does not exists' %(keyword, f) )
457        else:
458            ## read whole file and replace '\n' with ''
459            #
460            addr_l = open(f, 'r').readlines()
461            s = ','.join(addr_l).replace('\n','')
462
463            try:
464                self.parameters[keyword] = "%s,%s" %(self.parameters[keyword], s)
465            except KeyError, detail:
466                self.parameters[keyword] = s
467
468########## DEBUG functions  ###########################################################
469
470    def debug_body(self, message_body, temp_file):
471        """
472        """
473        self.logger.debug('function debug_body:')
474
475        body_file = "%s.body" %(temp_file)
476        fx = open(body_file, 'wb')
477
478        if self.parameters.dry_run:
479            self.logger.info('DRY-RUN: not saving body to %s' %(body_file))
480            return
481
482        self.logger.debug('writing body to %s' %(body_file))
483        if not message_body:
484                message_body = '(None)'
485
486        message_body = message_body.encode('utf-8')
487
488        fx.write(message_body)
489        fx.close()
490        try:
491            os.chmod(body_file,S_IRWXU|S_IRWXG|S_IRWXO)
492        except OSError:
493            pass
494
495    def debug_attachments(self, message_parts, temp_file):
496        """
497        """
498        self.logger.debug('function debug_attachments')
499       
500        n = 0
501        for item in message_parts:
502
503            ## Skip inline text parts
504            #
505            if not isinstance(item, tuple):
506                continue
507               
508            (original, filename, part) = item
509            self.logger.debug('\t part%d: Content-Type: %s' % (n, part.get_content_type()) )
510
511            dummy_filename, ext = os.path.splitext(filename)
512
513            n = n + 1
514            part_name = 'part%d%s' %(n, ext)
515            part_file = "%s.%s" %(temp_file, part_name)
516
517            s = 'writing %s: filename: %s' %(part_file, filename)
518            self.print_unicode(s)
519
520            ## Forbidden chars, just write part files instead of names
521            #
522            #filename = filename.replace('\\', '_')
523            #filename = filename.replace('/', '_')
524            #filename = filename + '.att_email2trac'
525            # part_file = os.path.join(self.parameters.tmpdir, filename)
526            #part_file = util.text.unicode_quote(part_file)
527            #self.print_unicode(part_file)
528
529            if self.parameters.dry_run:
530                self.logger.info('DRY_RUN: NOT saving attachments')
531                continue
532
533            fx = open(part_file, 'wb')
534            text = part.get_payload(decode=1)
535
536            if not text:
537                text = '(None)'
538
539            fx.write(text)
540            fx.close()
541
542            try:
543                os.chmod(part_file,S_IRWXU|S_IRWXG|S_IRWXO)
544            except OSError:
545                pass
546
547    def save_email_for_debug(self, message, project_name, create_tempfile=False):
548
549        if self.parameters.dry_run:
550            self.logger.debug('DRY_RUN: NOT saving email message')
551            return
552
553        if create_tempfile:
554            (fd, tmp_file) = tempfile.mkstemp('.%s.email2trac' %project_name)
555            fx = os.fdopen(fd, 'wb')
556        else:
557            tmp_file = os.path.join(self.parameters.tmpdir, 'msg.%s.email2trac' %project_name)
558            fx = open(tmp_file, 'wb')
559
560
561        self.logger.debug('saving email to %s' %(tmp_file))
562        fx.write('%s' % message)
563        fx.close()
564       
565        try:
566            os.chmod(tmp_file, S_IRWXU|S_IRWXG|S_IRWXO)
567        except OSError:
568            pass
569
570        message_parts = self.get_message_parts(message)
571        message_parts = self.unique_attachment_names(message_parts)
572        body_text = self.get_body_text(message_parts)
573        self.debug_body(body_text, tmp_file)
574        self.debug_attachments(message_parts, tmp_file)
575
576########## Conversion functions  ###########################################################
577
578    def email_to_unicode(self, message_str):
579        """
580        Email has 7 bit ASCII code, convert it to unicode with the charset
581        that is encoded in 7-bit ASCII code and encode it as utf-8 so Trac
582        understands it.
583        """
584        self.logger.debug("function email_to_unicode")
585
586        results =  email.Header.decode_header(message_str)
587
588        s = None
589        for text,format in results:
590            if format:
591                try:
592                    temp = unicode(text, format)
593                except UnicodeError, detail:
594                    # This always works
595                    #
596                    msg = 'ERROR: Could not find charset: %s, please install' %format
597                    self.logger.error(msg)
598                    temp = unicode(text, 'iso-8859-15')
599                except LookupError, detail:
600                    msg = 'ERROR: Could not find charset: %s, please install' %format
601                    self.logger.error(msg)
602                    #temp = unicode(text, 'iso-8859-15')
603                    temp = message_str
604                       
605            else:
606                temp = string.strip(text)
607                temp = unicode(text, 'iso-8859-15')
608
609            if s:
610                s = '%s %s' %(s, temp)
611            else:
612                s = '%s' %temp
613
614        return s
615
616    def str_to_dict(self, s):
617        """
618        Transfrom a string of the form [<key>=<value>]+ to dict[<key>] = <value>
619        """
620        self.logger.debug("function str_to_dict")
621
622        fields = string.split(s, self.parameters.subject_field_separator)
623
624        result = dict()
625        for field in fields:
626            try:
627                index, value = string.split(field, '=')
628
629                # We can not change the description of a ticket via the subject
630                # line. The description is the body of the email
631                #
632                if index.lower() in ['description']:
633                    continue
634
635                if value:
636                    result[index.lower()] = value
637
638            except ValueError:
639                pass
640        return result
641
642    def print_unicode(self,s):
643        """
644        This function prints unicode strings if possible else it will quote it
645        """
646        try:
647            self.logger.debug(s)
648        except UnicodeEncodeError, detail:
649            self.logger.debug(util.text.unicode_quote(s))
650
651
652    def html_2_txt(self, data):
653        """
654        Various routines to convert html syntax to valid trac wiki syntax
655        """
656        self.logger.debug('function html_2_txt')
657
658        ## This routine make an safe html that can be include
659        #  in trac, but no further text processing can be done
660        #
661#       try:
662#           from lxml.html.clean import Cleaner
663#           tags_rm = list()
664#           tags_rm.append('body')
665#
666#           cleaner = Cleaner(remove_tags=tags_rm )
667#           parsed_data = cleaner.clean_html(data)
668#           parsed_data = '\n{{{\n#!html\n' + parsed_data + '\n}}}\n'
669#
670#           return parsed_data
671#           
672#       except ImportError::
673#           pass
674
675        parsed_data = None
676        if self.parameters.html2text_cmd:
677            (fd, tmp_file) = tempfile.mkstemp('email2trac.html')
678            f = os.fdopen(fd, 'w')
679
680            cmd = '%s %s' %(self.parameters.html2text_cmd, tmp_file)
681            self.logger.debug('\t html2text conversion %s'%(cmd))
682   
683            if self.parameters.dry_run:
684                self.logger.info('DRY_RUN: html2text conversion command: %s\n' %(cmd))
685
686            else:
687                f.write(data)
688                f.close()
689
690                lines = os.popen(cmd).readlines()
691                parsed_data =  ''.join(lines)
692
693                os.unlink(tmp_file)
694
695        else:
696            self.logger.debug('\t No html2text conversion tool specified in email2trac.conf')
697
698        return parsed_data
699
700    def check_filename_length(self, name):
701        """
702        To bypass a bug in Trac
703        check if the filename lenght is not larger then OS limit.
704          yes : return truncated filename
705          no  : return unmodified filename
706        """
707        self.logger.debug('function check_filename_length: ')
708
709        if not name:
710            return 'None'
711       
712        dummy_filename, ext = os.path.splitext(name)
713
714        ## Trac uses this format
715        #
716        quote_format = util.text.unicode_quote(dummy_filename)
717
718        ## Determine max filename lenght
719        #
720        try:
721            filemax_length = os.pathconf('/', 'PC_NAME_MAX')
722        except AttributeError, detail:
723            filemax_lenght = 240
724
725        if len(quote_format) <= filemax_length:
726            return name
727
728        else:
729            ## Truncate file to filemax_length and reserve room for extension
730            #  We must break on a boundry
731            #
732            length = filemax_length - 6
733
734            for i in range(0,10):
735
736                truncated = quote_format[ : (length - i)]
737
738                try:
739                    unqoute_truncated = util.text.unicode_unquote(truncated)
740                    unqoute_truncated = unqoute_truncated + ext
741
742                    self.print_unicode('\t ' + unqoute_truncated)
743
744                    break
745
746                except UnicodeDecodeError, detail:
747                    continue
748
749
750            return unqoute_truncated
751
752########## TRAC ticket functions  ###########################################################
753
754    def mail_workflow(self, tkt):
755        """
756        """
757        self.logger.debug('function mail_workflow: ')
758       
759        req = Mock(authname=self.author, perm=MockPerm(), args={})
760        ticket_system = TicketSystem(self.env)
761       
762        try:
763            workflow = self.parameters['workflow_%s' %tkt['status'].lower()]
764
765        except KeyError:
766            ## fallback for compability (Will be deprecated)
767            #  workflow can be none.
768            #
769            workflow = None
770            if tkt['status'] in ['closed']:
771                workflow = self.parameters.workflow
772
773        if workflow:
774
775            ## process all workflow implementations
776            #
777            tkt_module = TicketModule(self.env)
778            field_changes, problems = tkt_module.get_ticket_changes(req, tkt, workflow)
779
780            for field in field_changes.keys():
781
782                ## We have already processed these fields
783                #
784                if not field in ['summary', 'description']:
785                    s = 'workflow : %s, field %s : %s, by %s' \
786                       %(workflow, field, field_changes[field]['new'],field_changes[field]['by'] )
787                    self.logger.debug(s)
788
789                    tkt[field] = field_changes[field]['new']
790
791            return True
792
793        else:
794            return False
795
796    def check_permission_participants(self, tkt, action):
797        """
798        Check if the mailer is allowed to update the ticket
799        """
800        self.logger.debug('function check_permission_participants %s')
801
802        if tkt['reporter'].lower() in [self.author.lower(), self.email_addr.lower()]:
803            self.logger.debug('ALLOW, %s is the ticket reporter' %(self.email_addr))
804
805            return True
806
807        perm = PermissionSystem(self.env)
808        if perm.check_permission(action, self.author):
809            self.logger.debug('ALLOW, %s has trac permission to update the ticket' %(self.author))
810
811            return True
812       
813        # Is the updater in the CC?
814        try:
815            cc_list = tkt['cc'].split(',')
816            for cc in cc_list:
817                if self.email_addr.lower() in cc.lower.strip():
818                    self.logger.debug('ALLOW, %s is in the CC' %(self.email_addr))
819
820                    return True
821
822        except KeyError:
823            pass
824
825        return False
826
827    def check_permission(self, tkt, action):
828        """
829        check if the reporter has the right permission for the action:
830          - TICKET_CREATE
831          - TICKET_MODIFY
832          - TICKET_APPEND
833          - TICKET_CHGPROP
834
835        There are three models:
836            - None      : no checking at all
837            - trac      : check the permission via trac permission model
838            - email2trac: ....
839        """
840        self.logger.debug("function check_permission: %s" %(action))
841
842        if self.parameters.ticket_permission_system in ['trac']:
843
844            perm = PermissionCache(self.env, self.author)
845            if perm.has_permission(action):
846                return True
847            else:
848                return False
849
850        elif self.parameters.ticket_permission_system in ['update_restricted_to_participants']:
851            return (self.check_permission_participants(tkt, action))   
852
853        ## Default is to allow everybody ticket updates and ticket creation
854        #
855        else:
856                return True
857
858
859    def update_ticket_fields(self, ticket, user_dict, new=None):
860        """
861        This will update the ticket fields. It will check if the
862        given fields are known and if the right values are specified
863        It will only update the ticket field value:
864            - If the field is known
865            - If the value supplied is valid for the ticket field.
866              If not then there are two options:
867               1) Skip the value (new=None)
868               2) Set default value for field (new=1)
869        """
870        self.logger.debug("function update_ticket_fields")
871
872        ## Check only permission model on ticket updates
873        #
874        if not new:
875            if self.parameters.ticket_permission_system:
876                if not self.check_permission(ticket, 'TICKET_CHGPROP'):
877                    self.logger.info('Reporter: %s has no permission to change ticket properties' %self.author)
878                    return False
879
880        ## Build a system dictionary from the ticket fields
881        #  with field as index and option as value
882        #
883        sys_dict = dict()
884
885        for field in ticket.fields:
886
887            try:
888                sys_dict[field['name']] = field['options']
889
890            except KeyError:
891                sys_dict[field['name']] = None
892                pass
893
894        ## Check user supplied fields an compare them with the
895        #  system one's
896        #
897        for field,value in user_dict.items():
898       
899            s = 'user_field\t %s = %s' %(field,value)
900            self.print_unicode(s)
901
902            if not field in sys_dict.keys(): 
903                self.logger.debug('%s is not a valid field for tickets' %(field))
904                continue
905
906            ## To prevent mail loop
907            #
908            if field == 'cc':
909
910                cc_list = user_dict['cc'].split(',')
911
912                if self.trac_smtp_from in cc_list:
913                    self.logger.debug('MAIL LOOP: %s is not allowed as CC address' %(self.trac_smtp_from))
914
915                    cc_list.remove(self.trac_smtp_from)
916
917                value = ','.join(cc_list)
918               
919
920            ## Check if every value is allowed for this field
921            #
922            if sys_dict[field]:
923
924                if value in sys_dict[field]:
925                    ticket[field] = value
926                else:
927                    ## Must we set a default if value is not allowed
928                    #
929                    if new:
930                        value = self.get_config('ticket', 'default_%s' %(field) )
931
932            else:
933
934                ticket[field] = value
935
936            s = 'ticket_field\t %s = %s' %(field,  ticket[field])
937            self.print_unicode(s)
938
939    def ticket_update(self, m, id, spam):
940        """
941        If the current email is a reply to an existing ticket, this function
942        will append the contents of this email to that ticket, instead of
943        creating a new one.
944        """
945        self.logger.debug("function ticket_update")
946
947        if not self.parameters.ticket_update:
948            self.logger.debug("ticket_update disabled")
949            return False
950
951        ## Must we update ticket fields
952        #
953        update_fields = dict()
954        try:
955            id, keywords = string.split(id, '?')
956
957            update_fields = self.str_to_dict(keywords)
958
959            ## Strip '#'
960            #
961            self.id = int(id[1:])
962
963        except ValueError:
964
965            ## Strip '#'
966            #
967            self.id = int(id[1:])
968
969        self.logger.debug("\tticket id: %s" %id)
970
971        ## When is the change committed
972        #
973        if self.VERSION < 0.11:
974            when = int(time.time())
975        else:
976            when = datetime.now(util.datefmt.utc)
977
978        try:
979            tkt = Ticket(self.env, self.id, self.db)
980
981        except util.TracError, detail:
982
983            ## Not a valid ticket
984            #
985            self.logger.info("\tCreating a new ticket, ticket id: %s does not exists" %id)
986            self.id = None
987            return False
988
989        ## Check the permission of the reporter
990        #
991        if self.parameters.ticket_permission_system:
992            if not self.check_permission(tkt, 'TICKET_APPEND'):
993                self.logger.info('Reporter: %s has no permission to add comments or attachments to tickets' %self.author)
994                return False
995
996        ## How many changes has this ticket
997        #
998        grouped = TicketModule(self.env).grouped_changelog_entries(tkt, self.db)
999        cnum = sum(1 for e in grouped) + 1
1000
1001
1002        ## reopen the ticket if it is was closed
1003        #  We must use the ticket workflow framework
1004        #
1005        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1006
1007            if not self.mail_workflow(tkt):
1008
1009                if tkt['status'] in ['closed']:
1010                    tkt['status'] = 'reopened'
1011                    tkt['resolution'] = ''
1012
1013        ## Old pre 0.11 situation
1014        #
1015        elif self.parameters.email_triggers_workflow:
1016
1017            self.logger.debug('\temail triggers workflow pre trac 0.11')
1018
1019            if tkt['status'] in ['closed']:
1020                tkt['status'] = 'reopened'
1021                tkt['resolution'] = ''
1022
1023        else:
1024            self.logger.debug('\temail triggers workflow disabled')
1025
1026        ## Must we update some ticket fields properties via subject line
1027        #
1028        if update_fields:
1029            self.update_ticket_fields(tkt, update_fields)
1030
1031
1032        message_parts = self.get_message_parts(m)
1033        message_parts = self.unique_attachment_names(message_parts)
1034
1035        ## Must we update some ticket fields properties via inline comments
1036        # in body_text
1037        #
1038        if self.properties:
1039                self.update_ticket_fields(tkt, self.properties)
1040
1041        ## Must we update the CC ticket field
1042        #
1043        self.set_cc_fields(tkt, m, update=True)
1044
1045        if self.parameters.email_header:
1046            message_parts.insert(0, self.email_header_txt(m))
1047
1048        body_text = self.get_body_text(message_parts)
1049
1050        error_with_attachments = self.attach_attachments(message_parts)
1051
1052        if body_text.strip() or update_fields or self.properties:
1053
1054            if self.parameters.dry_run:
1055                s = 'DRY_RUN: tkt.save_changes(self.author, body_text, ticket_change_number) %s %s' %(self.author, cnum)
1056                self.logger.info(s)
1057
1058            else:
1059                if error_with_attachments:
1060                    body_text = '%s\\%s' %(error_with_attachments, body_text)
1061
1062                self.logger.debug('\ttkt.save_changes(%s, %d)' %(self.author, cnum))
1063                tkt.save_changes(self.author, body_text, when, None, str(cnum))
1064           
1065        if not spam:
1066            self.notify(tkt, False, when)
1067
1068        return True
1069
1070    def set_ticket_fields(self, ticket):
1071        """
1072        set the ticket fields to value specified
1073            - /etc/email2trac.conf with <prefix>_<field>
1074            - trac default values, trac.ini
1075        """
1076        self.logger.debug('function set_ticket_fields')
1077
1078        user_dict = dict()
1079
1080        for field in ticket.fields:
1081
1082            name = field['name']
1083
1084            ## default trac value
1085            #
1086            if not field.get('custom'):
1087                value = self.get_config('ticket', 'default_%s' %(name) )
1088
1089                ## skip this field can only be set by email2trac.conf
1090                #
1091                if name in ['resolution']:
1092                    value = None
1093
1094            else:
1095                ##  Else get the default value for reporter
1096                #
1097                value = field.get('value')
1098                options = field.get('options')
1099
1100                if value and options and (value not in options):
1101                     value = options[int(value)]
1102   
1103            s = 'trac[%s] = %s' %(name, value)
1104            self.print_unicode(s)
1105
1106            ## email2trac.conf settings
1107            #
1108            prefix = self.parameters.ticket_prefix
1109            try:
1110                value = self.parameters['%s_%s' %(prefix, name)]
1111
1112                s = 'email2trac[%s] = %s ' %(name, value)
1113                self.print_unicode(s)
1114
1115            except KeyError, detail:
1116                pass
1117       
1118            if value:
1119                user_dict[name] = value
1120
1121                s = 'used %s = %s' %(name, value)
1122                self.print_unicode(s)
1123
1124        self.update_ticket_fields(ticket, user_dict, new=1)
1125
1126        if 'status' not in user_dict.keys():
1127            ticket['status'] = 'new'
1128
1129
1130    def ticket_update_by_subject(self, subject):
1131        """
1132        This list of Re: prefixes is probably incomplete. Taken from
1133        wikipedia. Here is how the subject is matched
1134          - Re: <subject>
1135          - Re: (<Mail list label>:)+ <subject>
1136
1137        So we must have the last column
1138        """
1139        self.logger.debug('function ticket_update_by_subject')
1140
1141        found_id = None
1142        if self.parameters.ticket_update and self.parameters.ticket_update_by_subject:
1143               
1144            SUBJECT_RE = re.compile(r'^(?:(?:RE|AW|VS|SV|FW|FWD):\s*)+(.*)', re.IGNORECASE)
1145            result = SUBJECT_RE.search(subject)
1146
1147            if result:
1148                ## This is a reply
1149                #
1150                orig_subject = result.group(1)
1151
1152                self.logger.debug('subject search string: %s' %(orig_subject))
1153
1154                cursor = self.db.cursor()
1155                summaries = [orig_subject, '%%: %s' % orig_subject]
1156
1157                ## Time resolution is in micoseconds
1158                #
1159                search_date = datetime.now(util.datefmt.utc) - timedelta(days=self.parameters.ticket_update_by_subject_lookback)
1160
1161                if self.VERSION < 0.12:
1162                    lookback = util.datefmt.to_timestamp(search_date)
1163                else:
1164                    lookback = util.datefmt.to_utimestamp(search_date)
1165
1166                for summary in summaries:
1167                    self.logger.debug('Looking for summary matching: "%s"' % summary)
1168
1169                    sql = """SELECT id, reporter FROM ticket
1170                            WHERE changetime >= %s AND summary LIKE %s
1171                            ORDER BY changetime DESC"""
1172
1173                    cursor.execute(sql, [lookback, summary.strip()])
1174
1175                    for row in cursor:
1176
1177                        (matched_id, sql_reporter) = row
1178
1179                        ## Save first entry.
1180                        #
1181                        if not found_id:
1182                            found_id = matched_id
1183                           
1184                        ## If subject and reporter are the same. The we certainly have found the right ticket
1185                        #
1186                        if sql_reporter == self.author:
1187                            self.logger.debug('Found matching reporter: %s with ticket id: %d' %(sql_reporter, matched_id))
1188                            found_id = matched_id
1189                            break
1190
1191                    if found_id:
1192                        self.logger.debug('Found matching ticket id: %d' % found_id)
1193                        found_id = '#%d' % found_id
1194                        return (found_id, orig_subject)
1195                   
1196                    # obsolete !?? 12 Aug 2011
1197                    #subject = orig_subject
1198
1199        return (found_id, subject)
1200
1201
1202    def new_ticket(self, msg, subject, spam, set_fields = None):
1203        """
1204        Create a new ticket
1205        """
1206        self.logger.debug('function new_ticket')
1207
1208        tkt = Ticket(self.env)
1209
1210        ## self.author can be email address of an username
1211        #
1212        tkt['reporter'] = self.author
1213
1214        self.set_cc_fields(tkt, msg)
1215
1216        self.set_ticket_fields(tkt)
1217
1218        ## Check the permission of the reporter
1219        #
1220        if self.parameters.ticket_permission_system:
1221            if not self.check_permission(tkt, 'TICKET_CREATE'):
1222                self.logger.info('Reporter: %s has no permission to create tickets' %self.author)
1223                return False
1224
1225        ## Old style setting for component, will be removed
1226        #
1227        if spam:
1228            tkt['component'] = 'Spam'
1229
1230        elif self.parameters.has_key('component'):
1231            tkt['component'] = self.parameters['component']
1232
1233        if not msg['Subject']:
1234            tkt['summary'] = u'(No subject)'
1235        else:
1236            tkt['summary'] = subject
1237
1238
1239        if set_fields:
1240            rest, keywords = string.split(set_fields, '?')
1241
1242            if keywords:
1243                update_fields = self.str_to_dict(keywords)
1244                self.update_ticket_fields(tkt, update_fields)
1245
1246
1247        message_parts = self.get_message_parts(msg, True)
1248
1249        ## Must we update some ticket fields properties via body_text
1250        #
1251        if self.properties:
1252                self.update_ticket_fields(tkt, self.properties)
1253
1254        message_parts = self.unique_attachment_names(message_parts)
1255       
1256        ## produce e-mail like header
1257        #
1258        head = ''
1259        if self.parameters.email_header:
1260            head = self.email_header_txt(msg)
1261            message_parts.insert(0, head)
1262           
1263        body_text = self.get_body_text(message_parts)
1264
1265        tkt['description'] = body_text
1266
1267        ## When is the change committed
1268        #
1269        if self.VERSION < 0.11:
1270            when = int(time.time())
1271        else:
1272            when = datetime.now(util.datefmt.utc)
1273
1274        if self.parameters.dry_run:
1275            self.logger.info('DRY_RUN: tkt.insert()')
1276        else:
1277            self.id = tkt.insert()
1278   
1279        changed = False
1280        comment = ''
1281
1282        ## some routines in trac are dependend on ticket id
1283        #  like alternate notify template
1284        #
1285        if self.parameters.alternate_notify_template:
1286            tkt['id'] = self.id
1287            changed = True
1288
1289        ## Rewrite the description if we have mailto enabled
1290        #
1291        if self.parameters.mailto_link:
1292            changed = True
1293            comment = u'\nadded mailto line\n'
1294
1295            #mailto = self.html_mailto_link( m['Subject'])
1296            mailto = self.html_mailto_link(subject)
1297
1298            tkt['description'] = u'%s\r\n%s%s\r\n' \
1299                %(head, mailto, body_text)
1300   
1301        ## Save the attachments to the ticket   
1302        #
1303        error_with_attachments =  self.attach_attachments(message_parts)
1304
1305        if error_with_attachments:
1306            changed = True
1307            comment = '%s\n%s\n' %(comment, error_with_attachments)
1308
1309        if self.parameters.email_triggers_workflow and (self.VERSION >= 0.11):
1310            if self.mail_workflow(tkt):
1311                changed = True
1312
1313        if changed:
1314            if self.parameters.dry_run:
1315                s = 'DRY_RUN: tkt.save_changes(%s, comment) real reporter = %s' %( tkt['reporter'], self.author)
1316                self.logger.info(s)
1317
1318            else:
1319                tkt.save_changes(tkt['reporter'], comment)
1320
1321        if not spam:
1322            self.notify(tkt, True)
1323
1324
1325    def attach_attachments(self, message_parts, update=False):
1326        '''
1327        save any attachments as files in the ticket's directory
1328        '''
1329        self.logger.debug('function attach_attachments()')
1330
1331        if self.parameters.dry_run:
1332            self.logger.debug("DRY_RUN: no attachments attached to tickets")
1333            return ''
1334
1335        count = 0
1336
1337        ## Get Maxium attachment size
1338        #
1339        max_size = int(self.get_config('attachment', 'max_size'))
1340        status   = None
1341       
1342        for item in message_parts:
1343            ## Skip body parts
1344            #
1345            if not isinstance(item, tuple):
1346                continue
1347               
1348            (original, filename, part) = item
1349
1350            ## We have to determine the size so we use this temporary solution.
1351            #
1352            path, fd =  util.create_unique_file(os.path.join(self.parameters.tmpdir, 'email2trac_tmp.att'))
1353            text = part.get_payload(decode=1)
1354            if not text:
1355                text = '(None)'
1356            fd.write(text)
1357            fd.close()
1358
1359            ## get the file_size
1360            #
1361            stats = os.lstat(path)
1362            file_size = stats[ST_SIZE]
1363
1364            ## Check if the attachment size is allowed
1365            #
1366            if (max_size != -1) and (file_size > max_size):
1367                status = '%s\nFile %s is larger then allowed attachment size (%d > %d)\n\n' \
1368                    %(status, original, file_size, max_size)
1369
1370                os.unlink(path)
1371                continue
1372            else:
1373                count = count + 1
1374                   
1375            ## Insert the attachment
1376            #
1377            fd = open(path, 'rb')
1378            if self.system == 'discussion':
1379                att = attachment.Attachment(self.env, 'discussion', 'topic/%s' % (self.id,))
1380            elif self.system == 'blog':
1381                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,))
1382            else:
1383                s = 'Attach %s to ticket %d' %(filename, self.id)
1384                self.print_unicode(s)
1385                att = attachment.Attachment(self.env, 'ticket', self.id)
1386 
1387            ## This will break the ticket_update system, the body_text is vaporized
1388            #  ;-(
1389            #
1390            if not update:
1391                att.author = self.author
1392                att.description = self.email_to_unicode('Added by email2trac')
1393
1394            try:
1395
1396                self.logger.debug('Insert atachment')
1397                att.insert(filename, fd, file_size)
1398
1399            except OSError, detail:
1400
1401                self.logger.info('%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail))
1402                status = '%s\nFilename %s could not be saved, problem: %s' %(status, filename, detail)
1403
1404            ## Remove the created temporary filename
1405            #
1406            fd.close()
1407            os.unlink(path)
1408
1409        ## return error
1410        #
1411        return status
1412
1413########## Fullblog functions  #################################################
1414
1415    def blog(self, msg, subject, id, params):
1416        """
1417        The blog create/update function
1418        """
1419        ## import the modules
1420        #
1421        from tracfullblog.core import FullBlogCore
1422        from tracfullblog.model import BlogPost, BlogComment
1423
1424        ## instantiate blog core
1425        #
1426        blog = FullBlogCore(self.env)
1427        req = Mock(authname='anonymous', perm=MockPerm(), args={})
1428
1429        ## parameters from the subject
1430        #
1431        params = self.str_to_dict((params or '').lstrip('?'))
1432
1433        ## preferably get the time from the email date header, or else
1434        #  use current date and time
1435        date = email.Utils.parsedate_tz(msg.get('date'))
1436        if date:
1437            dt = util.datefmt.to_datetime(email.Utils.mktime_tz(date), util.datefmt.utc)
1438        else:
1439            self.logger.warn("No valid date header found")
1440            dt = util.datefmt.to_datetime(None, util.datefmt.utc)
1441
1442        ## blog entry affected
1443        #
1444        self.id = id or util.datefmt.format_datetime(dt, "%Y%m%d%H%M%S", util.datefmt.utc)
1445
1446        ## check wether a blog post exists
1447        #
1448        post = BlogPost(self.env, self.id)
1449        force_update = self.properties.get('update', params.get('update'))
1450
1451        ## message parts
1452        #
1453        message_parts = self.get_message_parts(msg)
1454        message_parts = self.unique_attachment_names(message_parts)
1455
1456        if post.get_versions() and not force_update:
1457
1458            ## add comment to blog entry
1459            #
1460            comment = BlogComment(self.env, self.id)
1461            comment.author = self.properties.get('author', params.get('author', self.author))
1462            comment.comment = self.get_body_text(message_parts)
1463            comment.time = dt
1464
1465            if self.parameters.dry_run:
1466                self.logger.info('DRY-RUN: not adding comment for blog entry "%s"' % id)
1467                return
1468            warnings = blog.create_comment(req, comment)
1469
1470        else:
1471            ## create or update blog entry
1472            #
1473            post.author = self.properties.get('author', params.get('author', self.author))
1474            post.categories = self.properties.get('categories', params.get('categories', ''))
1475            post.title = subject.strip()
1476            post.publish_time = dt
1477            post.body = self.get_body_text(message_parts)
1478           
1479            if self.parameters.dry_run:
1480                self.logger.info('DRY-RUN: not creating blog entry "%s"' % post.title)
1481                return
1482            warnings = blog.create_post(req, post, self.author, u'Created by email2trac', False)
1483
1484        ## check for problems
1485        #
1486        if warnings:
1487            raise TracError(', '.join('blog:%s:%s' % (w[0], w[1]) for w in warnings))
1488       
1489        ## all seems well, attach attachments
1490        #
1491        self.attach_attachments(message_parts)
1492
1493
1494########## Discussion functions  ##############################################
1495
1496    def discussion_topic(self, content, subject):
1497
1498        ## Import modules.
1499        #
1500        from tracdiscussion.api import DiscussionApi
1501        from trac.util.datefmt import to_timestamp, utc
1502
1503        self.logger.debug('Creating a new topic in forum:', self.id)
1504
1505        ## Get dissussion API component.
1506        #
1507        api = self.env[DiscussionApi]
1508        context = self._create_context(content, subject)
1509
1510        ## Get forum for new topic.
1511        #
1512        forum = api.get_forum(context, self.id)
1513
1514        if not forum:
1515            self.logger.error("ERROR: Replied forum doesn't exist")
1516
1517        ## Prepare topic.
1518        #
1519        topic = {'forum' : forum['id'],
1520                 'subject' : context.subject,
1521                 'time': to_timestamp(datetime.now(utc)),
1522                 'author' : self.author,
1523                 'subscribers' : [self.email_addr],
1524                 'body' : self.get_body_text(context.content_parts)}
1525
1526        ## Add topic to DB and commit it.
1527        #
1528        self._add_topic(api, context, topic)
1529        self.db.commit()
1530
1531    def discussion_topic_reply(self, content, subject):
1532
1533        ## Import modules.
1534        #
1535        from tracdiscussion.api import DiscussionApi
1536        from trac.util.datefmt import to_timestamp, utc
1537
1538        self.logger.debug('Replying to discussion topic', self.id)
1539
1540        ## Get dissussion API component.
1541        #
1542        api = self.env[DiscussionApi]
1543        context = self._create_context(content, subject)
1544
1545        ## Get replied topic.
1546        #
1547        topic = api.get_topic(context, self.id)
1548
1549        if not topic:
1550            self.logger.error("ERROR: Replied topic doesn't exist")
1551
1552        ## Prepare message.
1553        #
1554        message = {'forum' : topic['forum'],
1555                   'topic' : topic['id'],
1556                   'replyto' : -1,
1557                   'time' : to_timestamp(datetime.now(utc)),
1558                   'author' : self.author,
1559                   'body' : self.get_body_text(context.content_parts)}
1560
1561        ## Add message to DB and commit it.
1562        #
1563        self._add_message(api, context, message)
1564        self.db.commit()
1565
1566    def discussion_message_reply(self, content, subject):
1567
1568        ## Import modules.
1569        #
1570        from tracdiscussion.api import DiscussionApi
1571        from trac.util.datefmt import to_timestamp, utc
1572
1573        self.logger.debug('Replying to discussion message', self.id)
1574
1575        ## Get dissussion API component.
1576        #
1577        api = self.env[DiscussionApi]
1578        context = self._create_context(content, subject)
1579
1580        ## Get replied message.
1581        #
1582        message = api.get_message(context, self.id)
1583
1584        if not message:
1585            self.logger.error("ERROR: Replied message doesn't exist")
1586
1587        ## Prepare message.
1588        #
1589        message = {'forum' : message['forum'],
1590                   'topic' : message['topic'],
1591                   'replyto' : message['id'],
1592                   'time' : to_timestamp(datetime.now(utc)),
1593                   'author' : self.author,
1594                   'body' : self.get_body_text(context.content_parts)}
1595
1596        ## Add message to DB and commit it.
1597        #
1598        self._add_message(api, context, message)
1599        self.db.commit()
1600
1601    def _create_context(self, content, subject):
1602
1603        ## Import modules.
1604        #
1605        from trac.mimeview import Context
1606        from trac.web.api import Request
1607        from trac.perm import PermissionCache
1608
1609        ## TODO: Read server base URL from config.
1610        #  Create request object to mockup context creation.
1611        #
1612        environ = {'SERVER_PORT' : 80,
1613                   'SERVER_NAME' : 'test',
1614                   'REQUEST_METHOD' : 'POST',
1615                   'wsgi.url_scheme' : 'http',
1616                   'wsgi.input' : sys.stdin}
1617        chrome =  {'links': {},
1618                   'scripts': [],
1619                   'ctxtnav': [],
1620                   'warnings': [],
1621                   'notices': []}
1622
1623        if self.env.base_url_for_redirect:
1624            environ['trac.base_url'] = self.env.base_url
1625
1626        req = Request(environ, None)
1627        req.chrome = chrome
1628        req.tz = 'missing'
1629        req.authname = self.author
1630        req.perm = PermissionCache(self.env, self.author)
1631        req.locale = None
1632
1633        ## Create and return context.
1634        #
1635        context = Context.from_request(req)
1636        context.realm = 'discussion-email2trac'
1637        context.cursor = self.db.cursor()
1638        context.content = content
1639        context.subject = subject
1640
1641        ## Read content parts from content.
1642        #
1643        context.content_parts = self.get_message_parts(content)
1644        context.content_parts = self.unique_attachment_names(
1645          context.content_parts)
1646
1647        return context
1648
1649    def _add_topic(self, api, context, topic):
1650        context.req.perm.assert_permission('DISCUSSION_APPEND')
1651
1652        ## Filter topic.
1653        #
1654        for discussion_filter in api.discussion_filters:
1655            accept, topic_or_error = discussion_filter.filter_topic(
1656              context, topic)
1657            if accept:
1658                topic = topic_or_error
1659            else:
1660                raise TracError(topic_or_error)
1661
1662        ## Add a new topic.
1663        #
1664        api.add_topic(context, topic)
1665
1666        ## Get inserted topic with new ID.
1667        #
1668        topic = api.get_topic_by_time(context, topic['time'])
1669
1670        ## Attach attachments.
1671        #
1672        self.id = topic['id']
1673        self.attach_attachments(context.content_parts, True)
1674
1675        ## Notify change listeners.
1676        #
1677        for listener in api.topic_change_listeners:
1678            listener.topic_created(context, topic)
1679
1680    def _add_message(self, api, context, message):
1681        context.req.perm.assert_permission('DISCUSSION_APPEND')
1682
1683        ## Filter message.
1684        #
1685        for discussion_filter in api.discussion_filters:
1686            accept, message_or_error = discussion_filter.filter_message(
1687              context, message)
1688            if accept:
1689                message = message_or_error
1690            else:
1691                raise TracError(message_or_error)
1692
1693        ## Add message.
1694        #
1695        api.add_message(context, message)
1696
1697        ## Get inserted message with new ID.
1698        #
1699        message = api.get_message_by_time(context, message['time'])
1700
1701        ## Attach attachments.
1702        #
1703
1704        self.attach_attachments(context.content_parts, True)
1705
1706        ## Notify change listeners.
1707        #
1708        for listener in api.message_change_listeners:
1709            listener.message_created(context, message)
1710
1711########## MAIN function  ######################################################
1712
1713    def parse(self, fp):
1714        """
1715        """
1716        self.logger.debug('Main function parse')
1717        global m
1718
1719        m = email.message_from_file(fp)
1720       
1721        if not m:
1722            self.logger.debug('This is not a valid email message format')
1723            return
1724           
1725        ## Work around lack of header folding in Python; see http://bugs.python.org/issue4696
1726        #
1727        try:
1728            m.replace_header('Subject', m['Subject'].replace('\r', '').replace('\n', ''))
1729        except AttributeError, detail:
1730            pass
1731
1732        if self.parameters.debug:     # save the entire e-mail message text
1733            self.save_email_for_debug(m, self.parameters.project_name, True)
1734
1735        self.db = self.env.get_db_cnx()
1736        self.get_sender_info(m)
1737
1738
1739        if self.parameters.white_list_file:
1740            self.acl_list_from_file(self.parameters.white_list_file, 'white_list')
1741
1742        if not ( self.email_header_acl('white_list', self.email_addr, True) or self.allow_registered_user ) :
1743
1744            self.logger.info('Message rejected : %s not in white list' %(self.email_addr))
1745            return False
1746
1747        if self.email_header_acl('black_list', self.email_addr, False):
1748            self.logger.info('Message rejected : %s in black list' %(self.email_addr))
1749            return False
1750
1751        if not self.email_header_acl('recipient_list', self.email_to_addrs, True):
1752            self.logger.info('Message rejected : %s not in recipient list' %(self.email_to_addrs))
1753            return False
1754
1755        ## If spam drop the message
1756        #
1757        if self.spam(m) == 'drop':
1758            return False
1759
1760        elif self.spam(m) == 'spam':
1761            spam_msg = True
1762        else:
1763            spam_msg = False
1764
1765        if not m['Subject']:
1766            subject  = 'No Subject'
1767        else:
1768            subject  = self.email_to_unicode(m['Subject'])
1769
1770        ## Bug in python logging module <2.6
1771        #
1772        self.logger.info('subject: %s' %repr(subject))
1773
1774        ## [hic] #1529: Re: LRZ
1775        #  [hic] #1529?owner=bas,priority=medium: Re: LRZ
1776        #
1777        ticket_regex = r'''
1778            (?P<new_fields>[#][?].*)
1779            |(?P<reply>(?P<id>[#][\d]+)(?P<fields>\?.*)?:)
1780            '''
1781        ## Check if  FullBlogPlugin is installed
1782        #
1783        blog_enabled = None
1784        blog_regex = ''
1785        if self.get_config('components', 'tracfullblog.*') in ['enabled']:
1786            self.logger.debug('Trac BLOG support enabled')
1787            blog_enabled = True
1788            blog_regex = '''|(?P<blog>blog(?P<blog_params>[?][^:]*)?:(?P<blog_id>\S*))'''
1789
1790
1791        ## Check if DiscussionPlugin is installed
1792        #
1793        discussion_enabled = None
1794        discussion_regex = ''
1795        if self.get_config('components', 'tracdiscussion.api.discussionapi') in ['enabled']:
1796            self.logger.debug('Trac Discussion support enabled')
1797            discussion_enabled = True
1798            discussion_regex = r'''
1799            |(?P<forum>Forum[ ][#](?P<forum_id>\d+)[ ]-[ ]?)
1800            |(?P<topic>Topic[ ][#](?P<topic_id>\d+)[ ]-[ ]?)
1801            |(?P<message>Message[ ][#](?P<message_id>\d+)[ ]-[ ]?)
1802            '''
1803
1804
1805        regex_str = ticket_regex + blog_regex + discussion_regex
1806        SYSTEM_RE = re.compile(regex_str, re.VERBOSE)
1807
1808        ## Find out if this is a ticket, a blog or a discussion
1809        #
1810        result =  SYSTEM_RE.search(subject)
1811
1812        if result:
1813            ## update ticket + fields
1814            #
1815            if result.group('reply'):
1816                self.system = 'ticket'
1817
1818                ## Skip the last ':' character
1819                #
1820                if not self.ticket_update(m, result.group('reply')[:-1], spam_msg):
1821                    self.new_ticket(m, subject, spam_msg)
1822
1823            ## New ticket + fields
1824            #
1825            elif result.group('new_fields'):
1826                self.system = 'ticket'
1827                self.new_ticket(m, subject[:result.start('new_fields')], spam_msg, result.group('new_fields'))
1828
1829            if blog_enabled:
1830                if result.group('blog'):
1831                    self.system = 'blog'
1832                    self.blog(m, subject[result.end('blog'):], result.group('blog_id'), result.group('blog_params'))
1833
1834            if discussion_enabled:
1835                ## New topic.
1836                #
1837                if result.group('forum'):
1838                    self.system = 'discussion'
1839                    self.id = int(result.group('forum_id'))
1840                    self.discussion_topic(m, subject[result.end('forum'):])
1841
1842                ## Reply to topic.
1843                #
1844                elif result.group('topic'):
1845                    self.system = 'discussion'
1846                    self.id = int(result.group('topic_id'))
1847                    self.discussion_topic_reply(m, subject[result.end('topic'):])
1848
1849                ## Reply to topic message.
1850                #
1851                elif result.group('message'):
1852                    self.system = 'discussion'
1853                    self.id = int(result.group('message_id'))
1854                    self.discussion_message_reply(m, subject[result.end('message'):])
1855
1856        else:
1857
1858            self.system = 'ticket'
1859            (matched_id, subject) = self.ticket_update_by_subject(subject)
1860
1861            if matched_id:
1862
1863                if not self.ticket_update(m, matched_id, spam_msg):
1864                    self.new_ticket(m, subject, spam_msg)
1865
1866            else:
1867                ## No update by subject, so just create a new ticket
1868                #
1869                self.new_ticket(m, subject, spam_msg)
1870
1871
1872########## BODY TEXT functions  ###########################################################
1873
1874    def strip_signature(self, text):
1875        """
1876        Strip signature from message, inspired by Mailman software
1877        """
1878        self.logger.debug('function strip_signature')
1879
1880        body = []
1881
1882        STRIP_RE = re.compile( self.parameters.strip_signature_regex )
1883        for line in text.splitlines():
1884
1885            match = STRIP_RE.match(line)
1886            if match:
1887                self.logger.debug('\t"%s "  matched, skiping rest of message' %line)
1888                break
1889
1890            body.append(line)
1891
1892        return ('\n'.join(body))
1893
1894    def reflow(self, text, delsp = 0):
1895        """
1896        Reflow the message based on the format="flowed" specification (RFC 3676)
1897        """
1898        flowedlines = []
1899        quotelevel = 0
1900        prevflowed = 0
1901
1902        for line in text.splitlines():
1903            from re import match
1904           
1905            ## Figure out the quote level and the content of the current line
1906            #
1907            m = match('(>*)( ?)(.*)', line)
1908            linequotelevel = len(m.group(1))
1909            line = m.group(3)
1910
1911            ## Determine whether this line is flowed
1912            #
1913            if line and line != '-- ' and line[-1] == ' ':
1914                flowed = 1
1915            else:
1916                flowed = 0
1917
1918            if flowed and delsp and line and line[-1] == ' ':
1919                line = line[:-1]
1920
1921            ## If the previous line is flowed, append this line to it
1922            #
1923            if prevflowed and line != '-- ' and linequotelevel == quotelevel:
1924                flowedlines[-1] += line
1925
1926            ## Otherwise, start a new line
1927            #
1928            else:
1929                flowedlines.append('>' * linequotelevel + line)
1930
1931            prevflowed = flowed
1932           
1933
1934        return '\n'.join(flowedlines)
1935
1936    def strip_quotes(self, text):
1937        """
1938        Strip quotes from message by Nicolas Mendoza
1939        """
1940        self.logger.debug('function strip_quotes')
1941
1942        body = []
1943        for line in text.splitlines():
1944            try:
1945
1946                if line.startswith(self.parameters.email_quote):
1947                    continue
1948
1949            except UnicodeDecodeError:
1950
1951                tmp_line = self.email_to_unicode(line)
1952                if tmp_line.startswith(self.parameters.email_quote):
1953                    continue
1954               
1955            body.append(line)
1956
1957        return ('\n'.join(body))
1958
1959    def inline_properties(self, text):
1960        """
1961        Parse text if we use inline keywords to set ticket fields
1962        """
1963        self.logger.debug('function inline_properties')
1964
1965        properties = dict()
1966        body = list()
1967
1968        INLINE_EXP = re.compile('\s*[@]\s*(\w+)\s*:(.*)$')
1969
1970        for line in text.splitlines():
1971            match = INLINE_EXP.match(line)
1972            if match:
1973                keyword, value = match.groups()
1974
1975                if self.parameters.inline_properties_first_wins:
1976                    if keyword in self.properties.keys():
1977                        continue
1978
1979                self.properties[keyword] = value.strip()
1980                self.logger.debug('\tinline properties: %s : %s' %(keyword,value))
1981
1982            else:
1983                body.append(line)
1984               
1985        return '\n'.join(body)
1986
1987
1988    def wrap_text(self, text, replace_whitespace = False):
1989        """
1990        Will break a lines longer then given length into several small
1991        lines of size given length
1992        """
1993        import textwrap
1994
1995        LINESEPARATOR = '\n'
1996        reformat = ''
1997
1998        for s in text.split(LINESEPARATOR):
1999            tmp = textwrap.fill(s, self.parameters.use_textwrap)
2000            if tmp:
2001                reformat = '%s\n%s' %(reformat,tmp)
2002            else:
2003                reformat = '%s\n' %reformat
2004
2005        return reformat
2006
2007        # Python2.4 and higher
2008        #
2009        #return LINESEPARATOR.join(textwrap.fill(s,width) for s in str.split(LINESEPARATOR))
2010        #
2011
2012########## EMAIL attachements functions ###########################################################
2013
2014    def inline_part(self, part):
2015        """
2016        """
2017        self.logger.debug('function inline_part()')
2018
2019        return part.get_param('inline', None, 'Content-Disposition') == '' or not part.has_key('Content-Disposition')
2020
2021    def get_message_parts(self, msg, new_email=False):
2022        """
2023        parses the email message and returns a list of body parts and attachments
2024        body parts are returned as strings, attachments are returned as tuples of (filename, Message object)
2025        """
2026        self.logger.debug('function get_message_parts()')
2027
2028        message_parts = list()
2029   
2030        ALTERNATIVE_MULTIPART = False
2031
2032        for part in msg.walk():
2033            content_maintype = part.get_content_maintype()
2034            content_type =  part.get_content_type()
2035
2036            self.logger.debug('\t Message part: Main-Type: %s' % content_maintype)
2037            self.logger.debug('\t Message part: Content-Type: %s' % content_type)
2038
2039            ## Check content type
2040            #
2041            if content_type in self.STRIP_CONTENT_TYPES:
2042                self.logger.debug("\t A %s attachment named '%s' was skipped" %(content_type, part.get_filename()))
2043                continue
2044
2045            ## Catch some mulitpart execptions
2046            #
2047            if content_type == 'multipart/alternative':
2048                ALTERNATIVE_MULTIPART = True
2049                continue
2050
2051            ## Skip multipart containers
2052            #
2053            if content_maintype == 'multipart':
2054                self.logger.debug("\t Skipping multipart container")
2055                continue
2056           
2057            ## Check if this is an inline part. It's inline if there is co Cont-Disp header,
2058            #  or if there is one and it says "inline"
2059            #
2060            inline = self.inline_part(part)
2061
2062            ## Drop HTML message
2063            #
2064            if ALTERNATIVE_MULTIPART and self.parameters.drop_alternative_html_version:
2065
2066                if content_type == 'text/html':
2067                    self.logger.debug('\t Skipping alternative HTML message')
2068                    ALTERNATIVE_MULTIPART = False
2069                    continue
2070
2071            filename = part.get_filename()
2072            s = '\t unicode filename: %s' %(filename)
2073            self.print_unicode(s)
2074            self.logger.debug('\t raw filename: %s' %repr(filename))
2075
2076            filename = self.check_filename_length(filename)
2077
2078            ## Save all non plain text message as attachment
2079            #
2080            if not content_type in ['text/plain']:
2081
2082                message_parts.append( (filename, part) )
2083
2084                ## We only convert html messages
2085                #
2086                if not content_type == 'text/html':
2087                    self.logger.debug('\t Appending %s (%s)' %(repr(filename), content_type))
2088                    continue
2089
2090
2091            ## We have an text or html message
2092            #
2093            if not inline:
2094                    self.logger.debug('\t Appending %s (%s), not an inline messsage part' %(repr(filename), content_type))
2095                    message_parts.append( (filename, part) )
2096                    continue
2097               
2098            ## Try to decode message part. We have a html or plain text messafe
2099            #
2100            body_text = part.get_payload(decode=1)
2101            if not body_text:           
2102                body_text = part.get_payload(decode=0)
2103
2104            ## Try to convert html message
2105            #
2106            if content_type == 'text/html':
2107
2108                body_text = self.html_2_txt(body_text)
2109                if not body_text:
2110                    continue
2111
2112            format = email.Utils.collapse_rfc2231_value(part.get_param('Format', 'fixed')).lower()
2113            delsp = email.Utils.collapse_rfc2231_value(part.get_param('DelSp', 'no')).lower()
2114
2115            if self.parameters.reflow and not self.parameters.verbatim_format and format == 'flowed':
2116                body_text = self.reflow(body_text, delsp == 'yes')
2117
2118            if new_email and self.parameters.only_strip_on_update:
2119                self.logger.debug('Skip signature/quote stripping for new messages')
2120            else:
2121                if self.parameters.strip_signature:
2122                    body_text = self.strip_signature(body_text)
2123
2124                if self.parameters.strip_quotes:
2125                    body_text = self.strip_quotes(body_text)
2126
2127            if self.parameters.inline_properties:
2128                body_text = self.inline_properties(body_text)
2129
2130            if self.parameters.use_textwrap:
2131                body_text = self.wrap_text(body_text)
2132
2133            ## Get contents charset (iso-8859-15 if not defined in mail headers)
2134            #
2135            charset = part.get_content_charset()
2136            if not charset:
2137                charset = 'iso-8859-15'
2138
2139            try:
2140                ubody_text = unicode(body_text, charset)
2141
2142            except UnicodeError, detail:
2143                ubody_text = unicode(body_text, 'iso-8859-15')
2144
2145            except LookupError, detail:
2146                ubody_text = 'ERROR: Could not find charset: %s, please install' %(charset)
2147
2148            if self.parameters.verbatim_format:
2149                message_parts.append('{{{\r\n%s\r\n}}}' %ubody_text)
2150            else:
2151                message_parts.append('%s' %ubody_text)
2152
2153        return message_parts
2154       
2155    def unique_attachment_names(self, message_parts):
2156        """
2157        Make sure we have unique names attachments:
2158          - check if it contains illegal characters
2159          - Rename "None" filenames to "untitled-part"
2160        """
2161        self.logger.debug('function unique_attachment_names()')
2162        renamed_parts = []
2163        attachment_names = set()
2164
2165        for item in message_parts:
2166           
2167            ## If not an attachment, leave it alone
2168            #
2169            if not isinstance(item, tuple):
2170                renamed_parts.append(item)
2171                continue
2172               
2173            (filename, part) = item
2174
2175            ## If filename = None, use a default one
2176            #
2177            if filename in [ 'None']:
2178                filename = 'untitled-part'
2179                self.logger.info('\t Rename filename "None" to: %s' %filename)
2180
2181                ## Guess the extension from the content type, use non strict mode
2182                #  some additional non-standard but commonly used MIME types
2183                #  are also recognized
2184                #
2185                ext = mimetypes.guess_extension(part.get_content_type(), False)
2186                if not ext:
2187                    ext = '.bin'
2188
2189                filename = '%s%s' % (filename, ext)
2190
2191            ## Discard relative paths for windows/unix in attachment names
2192            #
2193            filename = filename.replace('\\', '_')
2194            filename = filename.replace('/', '_')
2195
2196            ## remove linefeed char
2197            #
2198            for forbidden_char in ['\r', '\n']:
2199                filename = filename.replace(forbidden_char,'')
2200
2201            ## We try to normalize the filename to utf-8 NFC if we can.
2202            #  Files uploaded from OS X might be in NFD.
2203            #  Check python version and then try it
2204            #
2205            #if sys.version_info[0] > 2 or (sys.version_info[0] == 2 and sys.version_info[1] >= 3):
2206            #   try:
2207            #       filename = unicodedata.normalize('NFC', unicode(filename, 'utf-8')).encode('utf-8') 
2208            #   except TypeError:
2209            #       pass
2210
2211            ## Make the filename unique for this ticket
2212            #
2213            num = 0
2214            unique_filename = filename
2215            dummy_filename, ext = os.path.splitext(filename)
2216
2217            while (unique_filename in attachment_names) or self.attachment_exists(unique_filename):
2218                num += 1
2219                unique_filename = "%s-%s%s" % (dummy_filename, num, ext)
2220               
2221            s = '\t Attachment with filename %s will be saved as %s' % (filename, unique_filename)
2222            self.print_unicode(s)
2223
2224            attachment_names.add(unique_filename)
2225
2226            renamed_parts.append((filename, unique_filename, part))
2227   
2228        return renamed_parts
2229           
2230           
2231    def attachment_exists(self, filename):
2232
2233        self.logger.debug("function attachment_exists")
2234
2235        s = '\t check if attachment already exists: Id : %s, Filename : %s' %(self.id, filename)
2236        self.print_unicode(s)
2237
2238        ## Do we have a valid ticket id
2239        #
2240        if not self.id:
2241            return False
2242
2243        try:
2244            if self.system == 'discussion':
2245
2246                att = attachment.Attachment(self.env, 'discussion', 'ticket/%s' % (self.id,), filename)
2247
2248            elif self.system == 'blog':
2249
2250                att = attachment.Attachment(self.env, 'blog', '%s' % (self.id,), filename)
2251
2252            else:
2253
2254                att = attachment.Attachment(self.env, 'ticket', self.id, filename)
2255
2256            return True
2257
2258        except attachment.ResourceNotFound:
2259
2260            return False
2261
2262########## TRAC Ticket Text ###########################################################
2263           
2264    def get_body_text(self, message_parts):
2265        """
2266        """
2267        self.logger.debug('function get_body_text()')
2268
2269        body_text = []
2270       
2271        for part in message_parts:
2272       
2273            ## Plain text part, append it
2274            #
2275            if not isinstance(part, tuple):
2276                body_text.extend(part.strip().splitlines())
2277                body_text.append("")
2278                continue
2279
2280            (original, filename, part) = part
2281            inline = self.inline_part(part)
2282
2283            ## Skip generation of attachment link if html is converted to text
2284            #
2285            if part.get_content_type() == 'text/html' and self.parameters.html2text_cmd and inline:
2286                s = 'Skipping attachment link for html part: %s' %(filename)
2287                self.print_unicode(s)
2288                continue
2289           
2290            if part.get_content_maintype() == 'image' and inline:
2291
2292                if self.system != 'discussion':
2293                    s = 'wiki image link for: %s' %(filename)
2294                    self.print_unicode(s)
2295                    body_text.append('[[Image(%s)]]' % filename)
2296
2297                body_text.append("")
2298
2299            else:
2300
2301                if self.system != 'discussion':
2302
2303                    s = 'wiki attachment link for: %s' %(filename)
2304                    self.print_unicode(s)
2305                    body_text.append('[attachment:"%s"]' % filename)
2306
2307                body_text.append("")
2308
2309        ## Convert list body_texts to string
2310        #
2311        body_text = '\r\n'.join(body_text)
2312        return body_text
2313
2314    def html_mailto_link(self, subject):
2315        """
2316        This function returns a HTML mailto tag with the ticket id and author email address
2317        """
2318        self.logger.debug("function html_mailto_link")
2319
2320        if not self.author:
2321            author = self.email_addr
2322        else:   
2323            author = self.author
2324
2325        if not self.parameters.mailto_cc:
2326            self.parameters.mailto_cc = ''
2327           
2328        ## Bug in urllib.quote function
2329        #
2330        if isinstance(subject, unicode):
2331            subject = subject.encode('utf-8')
2332
2333        ## use urllib to escape the chars
2334        #
2335        s = '%s?Subject=%s&cc=%s' %(
2336               urllib.quote(self.email_addr),
2337               urllib.quote('Re: #%s: %s' %(self.id, subject)),
2338               urllib.quote(self.parameters.mailto_cc)
2339               )
2340
2341        if self.VERSION in [ 0.10 ]:
2342            s = '\r\n{{{\r\n#!html\r\n<a\r\n href="mailto:%s">Reply to: %s\r\n</a>\r\n}}}\r\n' %(s, author)
2343        else:
2344            s = '[mailto:"%s" Reply to: %s]' %(s, author)
2345
2346        self.logger.debug("\tmailto link %s" %s)
2347        return s
2348
2349########## TRAC notify section ###########################################################
2350
2351    def notify(self, tkt, new=True, modtime=0):
2352        """
2353        A wrapper for the TRAC notify function. So we can use templates
2354        """
2355        self.logger.debug('function notify()')
2356
2357        if self.parameters.notify_reporter:
2358            self.logger.debug('\t Notify reporter set')
2359            global sender_email
2360            sender_email = self.email_addr
2361 
2362            self.logger.debug('\t Using Email2TracNotification function AlwaysNotifyReporter')
2363            import trac.notification as Email2TracNotification
2364            Email2TracNotification.Notify.notify = AlwaysNotifyReporter
2365
2366        if self.parameters.dry_run  :
2367                self.logger.info('DRY_RUN: self.notify(tkt, True) reporter = %s' %tkt['reporter'])
2368                return
2369        try:
2370
2371            #from trac.ticket.notification import TicketNotificationSystem
2372            #tn_sys = TicketNotificationSystem(self.env)
2373            #print tn_sys
2374            #print tn_sys.__dict__
2375            #sys.exit(0)
2376
2377            ## create false {abs_}href properties, to trick Notify()
2378            #
2379            if self.VERSION in [0.10]:
2380                self.env.abs_href = Href(self.get_config('project', 'url'))
2381                self.env.href = Href(self.get_config('project', 'url'))
2382
2383            tn = TicketNotifyEmail(self.env)
2384
2385            if self.parameters.alternate_notify_template:
2386
2387                if self.VERSION >= 0.11:
2388
2389                    from trac.web.chrome import Chrome
2390
2391                    if  self.parameters.alternate_notify_template_update and not new:
2392                        tn.template_name = self.parameters.alternate_notify_template_update
2393                    else:
2394                        tn.template_name = self.parameters.alternate_notify_template
2395
2396                    tn.template = Chrome(tn.env).load_template(tn.template_name, method='text')
2397                       
2398                else:
2399
2400                    tn.template_name = self.parameters.alternate_notify_template
2401
2402            tn.notify(tkt, new, modtime)
2403
2404        except Exception, e:
2405            self.logger.error('Failure sending notification on creation of ticket #%s: %s' %(self.id, e))
2406
2407########## END Class Definition  ########################################################
2408
2409########## Global Notificaition Function ################################################
2410def AlwaysNotifyReporter(self, resid):
2411    """
2412    Copy of def notify() to manipulate recipents to always include reporter for the
2413    notification.
2414    """
2415    #print sender_email, resid
2416    (torcpts, ccrcpts) = self.get_recipients(resid)
2417    #print "get_recipients finished"
2418
2419    if not tktparser.email_header_acl('notify_reporter_black_list', sender_email, False):
2420        ## additionally append sender (regardeless of settings in trac.ini)
2421        #
2422        if not sender_email in torcpts:
2423            torcpts.append(sender_email)
2424
2425    self.begin_send()
2426    self.send(torcpts, ccrcpts)
2427    self.finish_send()
2428
2429########## Parse Config File  ###########################################################
2430
2431def ReadConfig(file, name):
2432    """
2433    Parse the config file
2434    """
2435    if not os.path.isfile(file):
2436        print 'File %s does not exist' %file
2437        sys.exit(1)
2438
2439    config = trac_config.Configuration(file)
2440   
2441    parentdir = config.get('DEFAULT', 'parentdir')
2442    sections = config.sections()
2443
2444    ## use some trac internals to get the defaults
2445    #
2446    tmp = config.parser.defaults()
2447    project =  SaraDict()
2448
2449    for option, value in tmp.items():
2450        try:
2451            project[option] = int(value)
2452        except ValueError:
2453            project[option] = value
2454
2455    if name:
2456        if name in sections:
2457            project =  SaraDict()
2458            for option, value in  config.options(name):
2459                try:
2460                    project[option] = int(value)
2461                except ValueError:
2462                    project[option] = value
2463
2464        elif not parentdir:
2465            print "Not a valid project name: %s, valid names are: %s" %(name, sections)
2466            print "or set parentdir in the [DEFAULT] section"
2467            sys.exit(1)
2468
2469    ## If parentdir then set project dir to parentdir + name
2470    #
2471    if not project.has_key('project'):
2472        if not parentdir:
2473            print "You must set project or parentdir in your configuration file"
2474            sys.exit(1)
2475        elif not name:
2476            print "You must configure a  project section in your configuration file"
2477        else:
2478            project['project'] = os.path.join(parentdir, name)
2479
2480    ##
2481    # Save the project name
2482    #
2483    project['project_name'] = os.path.basename(project['project'])
2484
2485    return project
2486
2487########## Setup Logging ###############################################################
2488
2489def setup_log(parameters, project_name, interactive=None):
2490    """
2491    Setup logging
2492
2493    Note for log format the usage of `$(...)s` instead of `%(...)s` as the latter form
2494    would be interpreted by the ConfigParser itself.
2495    """
2496    logger = logging.getLogger('email2trac %s' %project_name)
2497
2498    if interactive:
2499        parameters.log_type = 'stderr'
2500
2501    if not parameters.log_type:
2502        if sys.platform in ['win32', 'cygwin']:
2503            parameters.log_type = 'eventlog'
2504        else:
2505            parameters.log_type = 'syslog'
2506
2507    if parameters.log_type == 'file':
2508
2509        if not parameters.log_file:
2510            parameters.log_file = 'email2trac.log'
2511
2512        if not os.path.isabs(parameters.log_file):
2513            parameters.log_file = os.path.join(tempfile.gettempdir(), parameters.log_file)
2514
2515        log_handler = logging.FileHandler(parameters.log_file)
2516
2517    elif parameters.log_type in ('winlog', 'eventlog', 'nteventlog'):
2518        ## Requires win32 extensions
2519        #
2520        logid = "email2trac"
2521        log_handler = logging.handlers.NTEventLogHandler(logid, logtype='Application')
2522
2523    elif parameters.log_type in ('syslog', 'unix'):
2524        log_handler = logging.handlers.SysLogHandler('/dev/log')
2525
2526    elif parameters.log_type in ('stderr'):
2527        log_handler = logging.StreamHandler(sys.stderr)
2528
2529    else:
2530        log_handler = logging.handlers.BufferingHandler(0)
2531
2532    if parameters.log_format:
2533        parameters.log_format = parameters.log_format.replace('$(', '%(')
2534    else:
2535        parameters.log_format = '%(name)s: %(message)s'
2536        if parameters.log_type in ('file', 'stderr'):
2537            parameters.log_format = '%(asctime)s ' + parameters.log_format
2538
2539    log_formatter = logging.Formatter(parameters.log_format)
2540    log_handler.setFormatter(log_formatter)
2541    logger.addHandler(log_handler)
2542
2543    if (parameters.log_level in ['DEBUG', 'ALL']) or (parameters.debug > 0):
2544        logger.setLevel(logging.DEBUG)
2545        parameters.debug = 1
2546
2547    elif parameters.log_level in ['INFO'] or parameters.verbose:
2548        logger.setLevel(logging.INFO)
2549
2550    elif parameters.log_level in ['WARNING']:
2551        logger.setLevel(logging.WARNING)
2552
2553    elif parameters.log_level in ['ERROR']:
2554        logger.setLevel(logging.ERROR)
2555
2556    elif parameters.log_level in ['CRITICAL']:
2557        logger.setLevel(logging.CRITICAL)
2558
2559    else:
2560        logger.setLevel(logging.INFO)
2561
2562    return logger
2563
2564
2565if __name__ == '__main__':
2566    ## Default config file
2567    #
2568    configfile = '@email2trac_conf@'
2569    project = ''
2570    component = ''
2571    ticket_prefix = 'default'
2572    dry_run = None
2573    verbose = None
2574    debug_interactive = None
2575
2576    SHORT_OPT = 'cdhf:np:t:v'
2577    LONG_OPT  =  ['component=', 'debug', 'dry-run', 'help', 'file=', 'project=', 'ticket_prefix=', 'verbose']
2578
2579    try:
2580        opts, args = getopt.getopt(sys.argv[1:], SHORT_OPT, LONG_OPT)
2581    except getopt.error,detail:
2582        print __doc__
2583        print detail
2584        sys.exit(1)
2585   
2586    project_name = None
2587    for opt,value in opts:
2588        if opt in [ '-h', '--help']:
2589            print __doc__
2590            sys.exit(0)
2591        elif opt in ['-c', '--component']:
2592            component = value
2593        elif opt in ['-d', '--debug']:
2594            debug_interactive = 1
2595        elif opt in ['-f', '--file']:
2596            configfile = value
2597        elif opt in ['-n', '--dry-run']:
2598            dry_run = True
2599        elif opt in ['-p', '--project']:
2600            project_name = value
2601        elif opt in ['-t', '--ticket_prefix']:
2602            ticket_prefix = value
2603        elif opt in ['-v', '--verbose']:
2604            verbose = True
2605   
2606    settings = ReadConfig(configfile, project_name)
2607
2608    ## The default prefix for ticket values in email2trac.conf
2609    #
2610    settings.ticket_prefix = ticket_prefix
2611    settings.dry_run = dry_run
2612    settings.verbose = verbose
2613
2614    if not settings.debug and debug_interactive:
2615        settings.debug = debug_interactive
2616
2617    if not settings.project:
2618        print __doc__
2619        print 'No Trac project is defined in the email2trac config file.'
2620        sys.exit(1)
2621
2622    logger = setup_log(settings, os.path.basename(settings.project), debug_interactive)
2623   
2624    if component:
2625        settings['component'] = component
2626
2627    ## Determine major trac version used to be in email2trac.conf
2628    # Quick hack for 0.12
2629    #
2630    version = '0.%s' %(trac_version.split('.')[1])
2631    if version.startswith('0.12'):
2632        version = '0.12'
2633    elif version.startswith('0.13'):
2634        version = '0.13'
2635
2636    logger.debug("Found trac version: %s" %(version))
2637   
2638    try:
2639        if version == '0.10':
2640            from trac import attachment
2641            from trac.env import Environment
2642            from trac.ticket import Ticket
2643            from trac.web.href import Href
2644            from trac import util
2645            from trac.ticket.web_ui import TicketModule
2646
2647            #
2648            # return  util.text.to_unicode(str)
2649            #
2650            # see http://projects.edgewall.com/trac/changeset/2799
2651            from trac.ticket.notification import TicketNotifyEmail
2652            from trac import config as trac_config
2653            from trac.core import TracError
2654
2655        elif version in ['0.11', '0.12', '0.13']:
2656            from trac import attachment
2657            from trac import config as trac_config
2658            from trac import util
2659            from trac.core import TracError
2660            from trac.env import Environment
2661            from trac.perm import PermissionSystem
2662            from trac.perm import PermissionCache
2663            from trac.test import Mock, MockPerm
2664            from trac.ticket import Ticket
2665            from trac.ticket.api import TicketSystem
2666            from trac.ticket.web_ui import TicketModule
2667            from trac.web.href import Href
2668
2669            #
2670            # return  util.text.to_unicode(str)
2671            #
2672            # see http://projects.edgewall.com/trac/changeset/2799
2673            from trac.ticket.notification import TicketNotifyEmail
2674
2675        else:
2676            logger.error('TRAC version %s is not supported' %version)
2677            sys.exit(0)
2678
2679        ## Must be set before environment is created
2680        #
2681        if settings.has_key('python_egg_cache'):
2682            python_egg_cache = str(settings['python_egg_cache'])
2683            os.environ['PYTHON_EGG_CACHE'] = python_egg_cache
2684
2685        if settings.debug > 0:
2686            logger.debug('Loading environment %s', settings.project)
2687
2688        try:
2689            env = Environment(settings['project'], create=0)
2690        except IOError, detail:
2691            logger.error("trac error: %s" %detail)
2692            sys.exit(0)
2693
2694        tktparser = TicketEmailParser(env, settings, logger, float(version))
2695        tktparser.parse(sys.stdin)
2696
2697    ## Catch all errors and use the logging module
2698    #
2699    except Exception, error:
2700
2701        etype, evalue, etb = sys.exc_info()
2702        for e in traceback.format_exception(etype, evalue, etb):
2703            logger.critical(e)
2704
2705        if m:
2706            tktparser.save_email_for_debug(m, settings.project_name, True)
2707
2708        sys.exit(1)
2709# EOB
Note: See TracBrowser for help on using the repository browser.