From 89eaffa2862d0ee0555c06013c4dfd3d89893433 Mon Sep 17 00:00:00 2001 From: Chris Caron Date: Tue, 6 Oct 2020 10:43:46 -0400 Subject: [PATCH] Mailgun cc, bcc, batch processing, and attachment support (#308) --- apprise/plugins/NotifyMailgun.py | 375 ++++++++++++++++++++++++++++--- test/test_mailgun.py | 163 ++++++++++++++ test/test_rest_plugins.py | 46 +++- 3 files changed, 556 insertions(+), 28 deletions(-) create mode 100644 test/test_mailgun.py diff --git a/apprise/plugins/NotifyMailgun.py b/apprise/plugins/NotifyMailgun.py index e876a5bd..7af3f954 100644 --- a/apprise/plugins/NotifyMailgun.py +++ b/apprise/plugins/NotifyMailgun.py @@ -52,10 +52,12 @@ # then it will also become the 'to' address as well. # import requests - +from email.utils import formataddr from .NotifyBase import NotifyBase from ..common import NotifyType -from ..utils import parse_list +from ..common import NotifyFormat +from ..utils import parse_emails +from ..utils import parse_bool from ..utils import is_email from ..utils import validate_regex from ..AppriseLocale import gettext_lazy as _ @@ -111,9 +113,16 @@ class NotifyMailgun(NotifyBase): # A URL that takes you to the setup/help of the specific protocol setup_url = 'https://github.com/caronc/apprise/wiki/Notify_mailgun' + # Default Notify Format + notify_format = NotifyFormat.HTML + # The default region to use if one isn't otherwise specified mailgun_default_region = MailgunRegion.US + # The maximum amount of emails that can reside within a single + # batch transfer + default_batch_size = 2000 + # Define object templates templates = ( '{schema}://{user}@{host}:{apikey}/', @@ -161,9 +170,35 @@ class NotifyMailgun(NotifyBase): 'to': { 'alias_of': 'targets', }, + 'cc': { + 'name': _('Carbon Copy'), + 'type': 'list:string', + }, + 'bcc': { + 'name': _('Blind Carbon Copy'), + 'type': 'list:string', + }, + 'batch': { + 'name': _('Batch Mode'), + 'type': 'bool', + 'default': False, + }, }) - def __init__(self, apikey, targets, from_name=None, region_name=None, + # Define any kwargs we're using + template_kwargs = { + 'headers': { + 'name': _('HTTP Header'), + 'prefix': '+', + }, + 'tokens': { + 'name': _('Template Tokens'), + 'prefix': ':', + }, + } + + def __init__(self, apikey, targets, cc=None, bcc=None, from_name=None, + region_name=None, headers=None, tokens=None, batch=False, **kwargs): """ Initialize Mailgun Object @@ -184,8 +219,30 @@ class NotifyMailgun(NotifyBase): self.logger.warning(msg) raise TypeError(msg) - # Parse our targets - self.targets = parse_list(targets) + # Acquire Email 'To' + self.targets = list() + + # Acquire Carbon Copies + self.cc = set() + + # Acquire Blind Carbon Copies + self.bcc = set() + + # For tracking our email -> name lookups + self.names = {} + + self.headers = {} + if headers: + # Store our extra headers + self.headers.update(headers) + + self.tokens = {} + if tokens: + # Store our template tokens + self.tokens.update(tokens) + + # Prepare Batch Mode Flag + self.batch = batch # Store our region try: @@ -214,29 +271,146 @@ class NotifyMailgun(NotifyBase): self.logger.warning(msg) raise TypeError(msg) - def send(self, body, title='', notify_type=NotifyType.INFO, **kwargs): + if targets: + # Validate recipients (to:) and drop bad ones: + for recipient in parse_emails(targets): + result = is_email(recipient) + if result: + self.targets.append( + (result['name'] if result['name'] else False, + result['full_email'])) + continue + + self.logger.warning( + 'Dropped invalid To email ' + '({}) specified.'.format(recipient), + ) + + else: + # If our target email list is empty we want to add ourselves to it + self.targets.append( + (self.from_name if self.from_name else False, self.from_addr)) + + # Validate recipients (cc:) and drop bad ones: + for recipient in parse_emails(cc): + email = is_email(recipient) + if email: + self.cc.add(email['full_email']) + + # Index our name (if one exists) + self.names[email['full_email']] = \ + email['name'] if email['name'] else False + continue + + self.logger.warning( + 'Dropped invalid Carbon Copy email ' + '({}) specified.'.format(recipient), + ) + + # Validate recipients (bcc:) and drop bad ones: + for recipient in parse_emails(bcc): + email = is_email(recipient) + if email: + self.bcc.add(email['full_email']) + + # Index our name (if one exists) + self.names[email['full_email']] = \ + email['name'] if email['name'] else False + continue + + self.logger.warning( + 'Dropped invalid Blind Carbon Copy email ' + '({}) specified.'.format(recipient), + ) + + def send(self, body, title='', notify_type=NotifyType.INFO, attach=None, + **kwargs): """ Perform Mailgun Notification """ + if not self.targets: + # There is no one to email; we're done + self.logger.warning( + 'There are no Email recipients to notify') + return False + # error tracking (used for function return) has_error = False + # Send in batches if identified to do so + batch_size = 1 if not self.batch else self.default_batch_size + # Prepare our headers headers = { 'User-Agent': self.app_id, 'Accept': 'application/json', } + # Track our potential files + files = {} + + if attach: + for idx, attachment in enumerate(attach): + # Perform some simple error checking + if not attachment: + # We could not access the attachment + self.logger.error( + 'Could not access attachment {}.'.format( + attachment.url(privacy=True))) + return False + + self.logger.debug( + 'Preparing Mailgun attachment {}'.format( + attachment.url(privacy=True))) + try: + files['attachment[{}]'.format(idx)] = \ + (attachment.name, open(attachment.path, 'rb')) + + except (OSError, IOError) as e: + self.logger.warning( + 'An I/O error occurred while opening {}.'.format( + attachment.name if attachment + else 'attachment')) + self.logger.debug('I/O Exception: %s' % str(e)) + + # tidy up any open files before we make our early + # return + for entry in files.values(): + self.logger.trace( + 'Closing attachment {}'.format(entry[0])) + entry[1].close() + + return False + + try: + reply_to = formataddr( + (self.from_name if self.from_name else False, + self.from_addr), charset='utf-8') + + except TypeError: + # Python v2.x Support (no charset keyword) + # Format our cc addresses to support the Name field + reply_to = formataddr( + (self.from_name if self.from_name else False, + self.from_addr)) + # Prepare our payload payload = { - 'from': '{name} <{addr}>'.format( - name=self.app_id if not self.from_name else self.from_name, - addr=self.from_addr), + # pass skip-verification switch upstream too + 'o:skip-verification': not self.verify_certificate, + + # Base payload options + 'from': reply_to, 'subject': title, - 'text': body, } + if self.notify_format == NotifyFormat.HTML: + payload['html'] = body + + else: + payload['text'] = body + # Prepare our URL as it's based on our hostname url = '{}{}/messages'.format( MAILGUN_API_LOOKUP[self.region_name], self.host) @@ -244,22 +418,106 @@ class NotifyMailgun(NotifyBase): # Create a copy of the targets list emails = list(self.targets) - if len(emails) == 0: - # No email specified; use the from - emails.append(self.from_addr) + for index in range(0, len(emails), batch_size): + # Initialize our cc list + cc = (self.cc - self.bcc) + + # Initialize our bcc list + bcc = set(self.bcc) - while len(emails): - # Get our email to notify - email = emails.pop(0) + # Initialize our to list + to = list() - # Prepare our user - payload['to'] = '{} <{}>'.format(email, email) + # Ensure we're pointed to the head of the attachment; this doesn't + # do much for the first iteration through this loop as we're + # already pointing there..., but it allows us to re-use the + # attachment over and over again without closing and then + # re-opening the same file again and again + for entry in files.values(): + try: + self.logger.trace( + 'Seeking to head of attachment {}'.format(entry[0])) + entry[1].seek(0) + + except (OSError, IOError) as e: + self.logger.warning( + 'An I/O error occurred seeking to head of attachment ' + '{}.'.format(entry[0])) + self.logger.debug('I/O Exception: %s' % str(e)) + + # tidy up any open files before we make our early + # return + for entry in files.values(): + self.logger.trace( + 'Closing attachment {}'.format(entry[0])) + entry[1].close() + + return False + + for to_addr in self.targets[index:index + batch_size]: + # Strip target out of cc list if in To + cc = (cc - set([to_addr[1]])) + + # Strip target out of bcc list if in To + bcc = (bcc - set([to_addr[1]])) + + try: + # Prepare our to + to.append(formataddr(to_addr, charset='utf-8')) + + except TypeError: + # Python v2.x Support (no charset keyword) + # Format our cc addresses to support the Name field + + # Prepare our to + to.append(formataddr(to_addr)) + + # Prepare our To + payload['to'] = ','.join(to) + + if cc: + try: + # Format our cc addresses to support the Name field + payload['cc'] = ','.join([formataddr( + (self.names.get(addr, False), addr), charset='utf-8') + for addr in cc]) + + except TypeError: + # Python v2.x Support (no charset keyword) + # Format our cc addresses to support the Name field + payload['cc'] = ','.join([formataddr( + (self.names.get(addr, False), addr)) + for addr in cc]) + + # Format our bcc addresses to support the Name field + if bcc: + payload['bcc'] = ','.join(bcc) + + # Store our token entries; users can reference these as %value% + # in their email message. + if self.tokens: + payload.update( + {'v:{}'.format(k): v for k, v in self.tokens.items()}) + + # Store our header entries if defined into the payload + # in their payload + if self.headers: + payload.update( + {'h:{}'.format(k): v for k, v in self.headers.items()}) # Some Debug Logging self.logger.debug('Mailgun POST URL: {} (cert_verify={})'.format( url, self.verify_certificate)) self.logger.debug('Mailgun Payload: {}' .format(payload)) + # For logging output of success and errors; we get a head count + # of our outbound details: + verbose_dest = ', '.join( + [x[1] for x in self.targets[index:index + batch_size]]) \ + if len(self.targets[index:index + batch_size]) <= 3 \ + else '{} recipients'.format( + len(self.targets[index:index + batch_size])) + # Always call throttle before any remote server i/o is made self.throttle() try: @@ -268,6 +526,7 @@ class NotifyMailgun(NotifyBase): auth=("api", self.apikey), data=payload, headers=headers, + files=None if not files else files, verify=self.verify_certificate, timeout=self.request_timeout, ) @@ -281,7 +540,7 @@ class NotifyMailgun(NotifyBase): self.logger.warning( 'Failed to send Mailgun notification to {}: ' '{}{}error={}.'.format( - email, + verbose_dest, status_str, ', ' if status_str else '', r.status_code)) @@ -295,12 +554,13 @@ class NotifyMailgun(NotifyBase): else: self.logger.info( - 'Sent Mailgun notification to {}.'.format(email)) + 'Sent Mailgun notification to {}.'.format( + verbose_dest)) except requests.RequestException as e: self.logger.warning( 'A Connection error occurred sending Mailgun:%s ' % ( - email) + 'notification.' + verbose_dest) + 'notification.' ) self.logger.debug('Socket Exception: %s' % str(e)) @@ -308,6 +568,21 @@ class NotifyMailgun(NotifyBase): has_error = True continue + except (OSError, IOError) as e: + self.logger.warning( + 'An I/O error occurred while reading attachments') + self.logger.debug('I/O Exception: %s' % str(e)) + + # Mark our failure + has_error = True + continue + + # Close any potential attachments that are still open + for entry in files.values(): + self.logger.trace( + 'Closing attachment {}'.format(entry[0])) + entry[1].close() + return not has_error def url(self, privacy=False, *args, **kwargs): @@ -318,8 +593,15 @@ class NotifyMailgun(NotifyBase): # Define any URL parameters params = { 'region': self.region_name, + 'batch': 'yes' if self.batch else 'no', } + # Append our headers into our parameters + params.update({'+{}'.format(k): v for k, v in self.headers.items()}) + + # Append our template tokens into our parameters + params.update({':{}'.format(k): v for k, v in self.tokens.items()}) + # Extend our parameters params.update(self.url_parameters(privacy=privacy, *args, **kwargs)) @@ -327,13 +609,32 @@ class NotifyMailgun(NotifyBase): # from_name specified; pass it back on the url params['name'] = self.from_name + if self.cc: + # Handle our Carbon Copy Addresses + params['cc'] = ','.join( + ['{}{}'.format( + '' if not e not in self.names + else '{}:'.format(self.names[e]), e) for e in self.cc]) + + if self.bcc: + # Handle our Blind Carbon Copy Addresses + params['bcc'] = ','.join(self.bcc) + + # a simple boolean check as to whether we display our target emails + # or not + has_targets = \ + not (len(self.targets) == 1 + and self.targets[0][1] == self.from_addr) + return '{schema}://{user}@{host}/{apikey}/{targets}/?{params}'.format( schema=self.secure_protocol, host=self.host, user=NotifyMailgun.quote(self.user, safe=''), apikey=self.pprint(self.apikey, privacy, safe=''), - targets='/'.join( - [NotifyMailgun.quote(x, safe='') for x in self.targets]), + targets='' if not has_targets else '/'.join( + [NotifyMailgun.quote('{}{}'.format( + '' if not e[0] else '{}:'.format(e[0]), e[1]), + safe='') for e in self.targets]), params=NotifyMailgun.urlencode(params)) @staticmethod @@ -370,10 +671,30 @@ class NotifyMailgun(NotifyBase): results['region_name'] = \ NotifyMailgun.unquote(results['qsd']['region']) - # Support the 'to' variable so that we can support targets this way too - # The 'to' makes it easier to use yaml configuration + # Handle 'to' email address if 'to' in results['qsd'] and len(results['qsd']['to']): - results['targets'] += \ - NotifyMailgun.parse_list(results['qsd']['to']) + results['targets'].append(results['qsd']['to']) + + # Handle Carbon Copy Addresses + if 'cc' in results['qsd'] and len(results['qsd']['cc']): + results['cc'] = results['qsd']['cc'] + + # Handle Blind Carbon Copy Addresses + if 'bcc' in results['qsd'] and len(results['qsd']['bcc']): + results['bcc'] = results['qsd']['bcc'] + + # Add our Meta Headers that the user can provide with their outbound + # emails + results['headers'] = {NotifyBase.unquote(x): NotifyBase.unquote(y) + for x, y in results['qsd+'].items()} + + # Add our template tokens (if defined) + results['tokens'] = {NotifyBase.unquote(x): NotifyBase.unquote(y) + for x, y in results['qsd:'].items()} + + # Get Batch Mode Flag + results['batch'] = \ + parse_bool(results['qsd'].get( + 'batch', NotifyMailgun.template_args['batch']['default'])) return results diff --git a/test/test_mailgun.py b/test/test_mailgun.py new file mode 100644 index 00000000..4920ab5d --- /dev/null +++ b/test/test_mailgun.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2020 Chris Caron +# All rights reserved. +# +# This code is licensed under the MIT License. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files(the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions : +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import os +import sys +import mock +import requests +from apprise import plugins +from apprise import Apprise +from apprise import AppriseAttachment +from apprise import NotifyType + +# Disable logging for a cleaner testing output +import logging +logging.disable(logging.CRITICAL) + +# Attachment Directory +TEST_VAR_DIR = os.path.join(os.path.dirname(__file__), 'var') + + +@mock.patch('requests.post') +def test_notify_mailgun_plugin_attachments(mock_post): + """ + API: NotifyMailgun() Attachments + + """ + # Disable Throttling to speed testing + plugins.NotifyBase.request_rate_per_sec = 0 + + okay_response = requests.Request() + okay_response.status_code = requests.codes.ok + okay_response.content = "" + + # Assign our mock object our return value + mock_post.return_value = okay_response + + # API Key + apikey = 'abc123' + + obj = Apprise.instantiate( + 'mailgun://user@localhost.localdomain/{}'.format(apikey)) + assert isinstance(obj, plugins.NotifyMailgun) + + # Test Valid Attachment + path = os.path.join(TEST_VAR_DIR, 'apprise-test.gif') + attach = AppriseAttachment(path) + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is True + + # Test invalid attachment + path = os.path.join(TEST_VAR_DIR, '/invalid/path/to/an/invalid/file.jpg') + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=path) is False + + mock_post.return_value = None + mock_post.side_effect = OSError() + # We can't send the message if we can't read the attachment + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is False + + # Get a appropriate "builtin" module name for pythons 2/3. + if sys.version_info.major >= 3: + builtin_open_function = 'builtins.open' + + else: + builtin_open_function = '__builtin__.open' + + # Test Valid Attachment (load 3) + path = ( + os.path.join(TEST_VAR_DIR, 'apprise-test.gif'), + os.path.join(TEST_VAR_DIR, 'apprise-test.gif'), + os.path.join(TEST_VAR_DIR, 'apprise-test.gif'), + ) + attach = AppriseAttachment(path) + + # Return our good configuration + mock_post.side_effect = None + mock_post.return_value = okay_response + with mock.patch(builtin_open_function, side_effect=OSError()): + # We can't send the message we can't open the attachment for reading + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is False + + # Do it again, but fail on the third file + with mock.patch( + builtin_open_function, + side_effect=(mock.Mock(), mock.Mock(), OSError())): + + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is False + + with mock.patch(builtin_open_function) as mock_open: + mock_fp = mock.Mock() + mock_fp.seek.side_effect = OSError() + mock_open.return_value = mock_fp + + # We can't send the message we can't seek through it + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is False + + mock_post.reset_mock() + # Fail on the third file; this tests the for-loop inside the seek() + # section of the code that calls close() on previously opened files + mock_fp.seek.side_effect = (None, None, OSError()) + mock_open.return_value = mock_fp + # We can't send the message we can't seek through it + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is False + + # test the handling of our batch modes + obj = Apprise.instantiate( + 'mailgun://no-reply@example.com/{}/' + 'user1@example.com/user2@example.com?batch=yes'.format(apikey)) + assert isinstance(obj, plugins.NotifyMailgun) + + # Force our batch to break into separate messages + obj.default_batch_size = 1 + # We'll send 2 messages + mock_post.reset_mock() + + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is True + assert mock_post.call_count == 2 + + # single batch + mock_post.reset_mock() + # We'll send 1 message + obj.default_batch_size = 2 + + assert obj.notify( + body='body', title='title', notify_type=NotifyType.INFO, + attach=attach) is True + assert mock_post.call_count == 1 diff --git a/test/test_rest_plugins.py b/test/test_rest_plugins.py index d1e99e82..2133d4ba 100644 --- a/test/test_rest_plugins.py +++ b/test/test_rest_plugins.py @@ -1458,6 +1458,18 @@ TEST_URLS = ( 'a' * 32, 'b' * 8, 'c' * 8), { 'instance': plugins.NotifyMailgun, }), + ('mailgun://user@localhost.localdomain/{}-{}-{}?format=markdown'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + }), + ('mailgun://user@localhost.localdomain/{}-{}-{}?format=html'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + }), + ('mailgun://user@localhost.localdomain/{}-{}-{}?format=text'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + }), # valid url with region specified (case insensitve) ('mailgun://user@localhost.localdomain/{}-{}-{}?region=uS'.format( 'a' * 32, 'b' * 8, 'c' * 8), { @@ -1473,6 +1485,24 @@ TEST_URLS = ( 'a' * 32, 'b' * 8, 'c' * 8), { 'instance': TypeError, }), + # headers + ('mailgun://user@localhost.localdomain/{}-{}-{}' + '?+X-Customer-Campaign-ID=Apprise'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + }), + # template tokens + ('mailgun://user@localhost.localdomain/{}-{}-{}' + '?:name=Chris&:status=admin'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + }), + # bcc and cc + ('mailgun://user@localhost.localdomain/{}-{}-{}' + '?bcc=user@example.com&cc=user2@example.com'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + }), # One To Email address ('mailgun://user@localhost.localdomain/{}-{}-{}/test@example.com'.format( 'a' * 32, 'b' * 8, 'c' * 8), { @@ -1482,12 +1512,26 @@ TEST_URLS = ( '{}-{}-{}?to=test@example.com'.format( 'a' * 32, 'b' * 8, 'c' * 8), { 'instance': plugins.NotifyMailgun}), - # One To Email address, a from name specified too ('mailgun://user@localhost.localdomain/{}-{}-{}/' 'test@example.com?name="Frodo"'.format( 'a' * 32, 'b' * 8, 'c' * 8), { 'instance': plugins.NotifyMailgun}), + # Invalid 'To' Email address + ('mailgun://user@localhost.localdomain/{}-{}-{}/invalid'.format( + 'a' * 32, 'b' * 8, 'c' * 8), { + 'instance': plugins.NotifyMailgun, + # Expected notify() response + 'notify_response': False, + }), + # Multiple 'To', 'Cc', and 'Bcc' addresses (with invalid ones) + ('mailgun://user@example.com/{}-{}-{}/{}?bcc={}&cc={}'.format( + 'a' * 32, 'b' * 8, 'c' * 8, + '/'.join(('user1@example.com', 'invalid', 'User2:user2@example.com')), + ','.join(('user3@example.com', 'i@v', 'User1:user1@example.com')), + ','.join(('user4@example.com', 'g@r@b', 'Da:user5@example.com'))), { + 'instance': plugins.NotifyMailgun, + }), ('mailgun://user@localhost.localdomain/{}-{}-{}'.format( 'a' * 32, 'b' * 8, 'c' * 8), { 'instance': plugins.NotifyMailgun,