Email deliverability improvement (#660)

apprise/conversion.py

@@ -131,8 +131,14 @@ class HTMLConverter(HTMLParser, object):
             #
             # This is required since the unescape() nbsp; with \xa0 when
             # using Python 2.7
+            try:
                 self.converted = self.converted.replace(u'\xa0', u' ')
 
+            except UnicodeDecodeError:
+                # Python v2.7 isn't the greatest for handling unicode
+                self.converted = \
+                    self.converted.decode('utf-8').replace(u'\xa0', u' ')
+
     def _finalize(self, result):
         """
         Combines and strips consecutive strings, then converts consecutive

apprise/plugins/NotifyEmail.py

@@ -29,7 +29,7 @@ import smtplib
 from email.mime.text import MIMEText
 from email.mime.application import MIMEApplication
 from email.mime.multipart import MIMEMultipart
-from email.utils import formataddr
+from email.utils import formataddr, make_msgid
 from email.header import Header
 from email import charset
 
@@ -38,10 +38,9 @@ from datetime import datetime
 
 from .NotifyBase import NotifyBase
 from ..URLBase import PrivacyMode
-from ..common import NotifyFormat
+from ..common import NotifyFormat, NotifyType
-from ..common import NotifyType
+from ..conversion import convert_between
-from ..utils import is_email
+from ..utils import is_email, parse_emails
-from ..utils import parse_emails
 from ..AppriseLocale import gettext_lazy as _
 
 # Globally Default encoding mode set to Quoted Printable.
@@ -397,6 +396,11 @@ class NotifyEmail(NotifyBase):
             'default': SecureMailMode.STARTTLS,
             'map_to': 'secure_mode',
         },
+        'reply': {
+            'name': _('Reply To'),
+            'type': 'list:string',
+            'map_to': 'reply_to',
+        },
     })
 
     # Define any kwargs we're using
@@ -409,7 +413,7 @@ class NotifyEmail(NotifyBase):
 
     def __init__(self, smtp_host=None, from_name=None,
                  from_addr=None, secure_mode=None, targets=None, cc=None,
-                 bcc=None, headers=None, **kwargs):
+                 bcc=None, reply_to=None, headers=None, **kwargs):
         """
         Initialize Email Object
 
@@ -435,6 +439,9 @@ class NotifyEmail(NotifyBase):
         # Acquire Blind Carbon Copies
         self.bcc = set()
 
+        # Acquire Reply To
+        self.reply_to = set()
+
         # For tracking our email -> name lookups
         self.names = {}
 
@@ -467,6 +474,10 @@ class NotifyEmail(NotifyBase):
         # Set our from name
         self.from_name = from_name if from_name else result['name']
 
+        # Store our lookup
+        self.names[self.from_addr] = \
+            self.from_name if self.from_name else False
+
         # Now detect the SMTP Server
         self.smtp_host = \
             smtp_host if isinstance(smtp_host, six.string_types) else ''
@@ -533,6 +544,26 @@ class NotifyEmail(NotifyBase):
                 '({}) specified.'.format(recipient),
             )
 
+        if not reply_to:
+            # Add ourselves to the Reply-To directive
+            self.reply_to.add(self.from_addr)
+
+        # Validate recipients (reply-to:) and drop bad ones:
+        for recipient in parse_emails(reply_to):
+            email = is_email(recipient)
+            if email:
+                self.reply_to.add(email['full_email'])
+
+                # Index our name (if one exists)
+                self.names[email['full_email']] = \
+                    email['name'] if email['name'] else False
+                continue
+
+            self.logger.warning(
+                'Dropped invalid Reply To email '
+                '({}) specified.'.format(recipient),
+            )
+
         # Apply any defaults based on certain known configurations
         self.NotifyEmailDefaults(secure_mode=secure_mode, **kwargs)
 
@@ -612,6 +643,18 @@ class NotifyEmail(NotifyBase):
 
                 break
 
+    def _get_charset(self, input_string):
+        """
+        Get utf-8 charset if non ascii string only
+
+        Encode an ascii string to utf-8 is bad for email deliverability
+        because some anti-spam gives a bad score for that
+        like SUBJ_EXCESS_QP flag on Rspamd
+        """
+        if not input_string:
+            return None
+        return 'utf-8' if not all(ord(c) < 128 for c in input_string) else None
+
     def send(self, body, title='', notify_type=NotifyType.INFO, attach=None,
              **kwargs):
         """
@@ -642,6 +685,9 @@ class NotifyEmail(NotifyBase):
             # Strip target out of bcc list if in To
             bcc = (self.bcc - set([to_addr]))
 
+            # Strip target out of reply_to list if in To
+            reply_to = (self.reply_to - set([to_addr]))
+
             try:
                 # Format our cc addresses to support the Name field
                 cc = [formataddr(
@@ -653,6 +699,11 @@ class NotifyEmail(NotifyBase):
                     (self.names.get(addr, False), addr), charset='utf-8')
                     for addr in bcc]
 
+                # Format our reply-to addresses to support the Name field
+                reply_to = [formataddr(
+                    (self.names.get(addr, False), addr), charset='utf-8')
+                    for addr in reply_to]
+
             except TypeError:
                 # Python v2.x Support (no charset keyword)
                 # Format our cc addresses to support the Name field
@@ -663,6 +714,10 @@ class NotifyEmail(NotifyBase):
                 bcc = [formataddr(  # pragma: no branch
                     (self.names.get(addr, False), addr)) for addr in bcc]
 
+                # Format our reply-to addresses to support the Name field
+                reply_to = [formataddr(  # pragma: no branch
+                    (self.names.get(addr, False), addr)) for addr in reply_to]
+
             self.logger.debug(
                 'Email From: {} <{}>'.format(from_name, self.from_addr))
             self.logger.debug('Email To: {}'.format(to_addr))
@@ -670,45 +725,29 @@ class NotifyEmail(NotifyBase):
                 self.logger.debug('Email Cc: {}'.format(', '.join(cc)))
             if bcc:
                 self.logger.debug('Email Bcc: {}'.format(', '.join(bcc)))
+            if reply_to:
+                self.logger.debug(
+                    'Email Reply-To: {}'.format(', '.join(reply_to))
+                )
             self.logger.debug('Login ID: {}'.format(self.user))
             self.logger.debug(
                 'Delivery: {}:{}'.format(self.smtp_host, self.port))
 
             # Prepare Email Message
             if self.notify_format == NotifyFormat.HTML:
-                content = MIMEText(body, 'html', 'utf-8')
+                base = MIMEMultipart("alternative")
-
+                base.attach(MIMEText(
+                    convert_between(
+                        NotifyFormat.HTML, NotifyFormat.TEXT, body),
+                    'plain', 'utf-8')
+                )
+                base.attach(MIMEText(body, 'html', 'utf-8'))
             else:
-                content = MIMEText(body, 'plain', 'utf-8')
+                base = MIMEText(body, 'plain', 'utf-8')
-
-            base = MIMEMultipart() if attach else content
-
-            # Apply any provided custom headers
-            for k, v in self.headers.items():
-                base[k] = Header(v, 'utf-8')
-
-            base['Subject'] = Header(title, 'utf-8')
-            try:
-                base['From'] = formataddr(
-                    (from_name if from_name else False, self.from_addr),
-                    charset='utf-8')
-                base['To'] = formataddr((to_name, to_addr), charset='utf-8')
-
-            except TypeError:
-                # Python v2.x Support (no charset keyword)
-                base['From'] = formataddr(
-                    (from_name if from_name else False, self.from_addr))
-                base['To'] = formataddr((to_name, to_addr))
-
-            base['Cc'] = ','.join(cc)
-            base['Date'] = \
-                datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
-            base['X-Application'] = self.app_id
 
             if attach:
-                # First attach our body to our content as the first element
+                mixed = MIMEMultipart("mixed")
-                base.attach(content)
+                mixed.attach(base)
-
                 # Now store our attachments
                 for attachment in attach:
                     if not attachment:
@@ -735,8 +774,41 @@ class NotifyEmail(NotifyBase):
                             'attachment; filename="{}"'.format(
                                 Header(attachment.name, 'utf-8')),
                         )
+                        mixed.attach(app)
+                base = mixed
 
-                        base.attach(app)
+            # Apply any provided custom headers
+            for k, v in self.headers.items():
+                base[k] = Header(v, self._get_charset(v))
+
+            base['Subject'] = Header(title, self._get_charset(title))
+            try:
+                base['From'] = formataddr(
+                    (from_name if from_name else False, self.from_addr),
+                    charset='utf-8')
+                base['To'] = formataddr((to_name, to_addr), charset='utf-8')
+
+            except TypeError:
+                # Python v2.x Support (no charset keyword)
+                base['From'] = formataddr(
+                    (from_name if from_name else False, self.from_addr))
+                base['To'] = formataddr((to_name, to_addr))
+
+            try:
+                base['Message-ID'] = make_msgid(domain=self.smtp_host)
+
+            except TypeError:
+                # Python v2.x Support (no domain keyword)
+                base['Message-ID'] = make_msgid()
+
+            base['Date'] = \
+                datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+            base['X-Application'] = self.app_id
+
+            if cc:
+                base['Cc'] = ','.join(cc)
+
+            base['Reply-To'] = ','.join(reply_to)
 
             # bind the socket variable to the current namespace
             socket = None
@@ -829,6 +901,12 @@ class NotifyEmail(NotifyBase):
                     '' if not e not in self.names
                     else '{}:'.format(self.names[e]), e) for e in self.bcc])
 
+        # Handle our Reply-To Addresses
+        params['reply'] = ','.join(
+            ['{}{}'.format(
+                '' if not e not in self.names
+                else '{}:'.format(self.names[e]), e) for e in self.reply_to])
+
         # pull email suffix from username (if present)
         user = None if not self.user else self.user.split('@')[0]
 
@@ -923,6 +1001,10 @@ class NotifyEmail(NotifyBase):
         if 'bcc' in results['qsd'] and len(results['qsd']['bcc']):
             results['bcc'] = results['qsd']['bcc']
 
+        # Handle Reply To Addresses
+        if 'reply' in results['qsd'] and len(results['qsd']['reply']):
+            results['reply_to'] = results['qsd']['reply']
+
         results['from_addr'] = from_addr
         results['smtp_host'] = smtp_host
 

test/test_plugin_email.py

@@ -160,6 +160,20 @@ TEST_URLS = (
             'instance': plugins.NotifyEmail,
         },
     ),
+    (
+        # Test Reply To
+        'mailtos://user:pass@example.com?smtp=smtp.example.com'
+        '&name=l2g&reply=test@example.com,test2@example.com', {
+            'instance': plugins.NotifyEmail,
+        },
+    ),
+    (
+        # Test Reply To with bad email
+        'mailtos://user:pass@example.com?smtp=smtp.example.com'
+        '&name=l2g&reply=test@example.com,@', {
+            'instance': plugins.NotifyEmail,
+        },
+    ),
     # headers
     ('mailto://user:pass@localhost.localdomain'
         '?+X-Customer-Campaign-ID=Apprise', {
@@ -233,6 +247,9 @@ TEST_URLS = (
     ('mailto://user:pass@localhost/?cc=test2@,$@!/', {
         'instance': plugins.NotifyEmail,
     }),
+    ('mailto://user:pass@localhost/?reply=test2@,$@!/', {
+        'instance': plugins.NotifyEmail,
+    }),
 )
 
 
@@ -788,7 +805,7 @@ def test_plugin_email_url_parsing(mock_smtp, mock_smtp_ssl):
     assert isinstance(_to, list)
     assert len(_to) == 1
     assert _to[0] == 'user2@yahoo.com'
-    assert _msg.endswith('test')
+    assert _msg.split('\n')[-3] == 'test'
 
     # Our URL port was over-ridden (on template) to use 444
     # We can verify that this was correctly saved
@@ -835,7 +852,7 @@ def test_plugin_email_url_parsing(mock_smtp, mock_smtp_ssl):
     assert isinstance(_to, list)
     assert len(_to) == 1
     assert _to[0] == 'user2@yahoo.com'
-    assert _msg.endswith('test')
+    assert _msg.split('\n')[-3] == 'test'
 
     assert obj.url().startswith(
         'mailtos://user:pass123@hotmail.com/user2%40yahoo.com')