Config file support added for http & file (yaml); refs #55

pull/79/head
Chris Caron 2019-03-03 23:12:57 -05:00
parent 0ab86c2115
commit 1f9daeaa2b
7 changed files with 745 additions and 11 deletions

View File

@ -23,9 +23,11 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE. # THE SOFTWARE.
import os
import re import re
import six import six
import logging import logging
import yaml
from .. import plugins from .. import plugins
from ..AppriseAsset import AppriseAsset from ..AppriseAsset import AppriseAsset
@ -184,6 +186,18 @@ class ConfigBase(URLBase):
Optionally associate an asset with the notification. Optionally associate an asset with the notification.
The file syntax is:
#
# pound/hashtag allow for line comments
#
# One or more tags can be idenified using comma's (,) to separate
# them.
<Tag(s)>=<URL>
# Or you can use this format (no tags associated)
<URL>
""" """
# For logging, track the line number # For logging, track the line number
line = 0 line = 0
@ -196,8 +210,14 @@ class ConfigBase(URLBase):
r'(\s*(?P<tags>[^=]+)=|=)?\s*' r'(\s*(?P<tags>[^=]+)=|=)?\s*'
r'(?P<url>[a-z0-9]{2,9}://.*))?$', re.I) r'(?P<url>[a-z0-9]{2,9}://.*))?$', re.I)
# split our content up to read line by line try:
content = re.split(r'\r*\n', content) # split our content up to read line by line
content = re.split(r'\r*\n', content)
except TypeError:
# content was not expected string type
logger.error('Invalid apprise text data specified')
return list()
for entry in content: for entry in content:
# Increment our line count # Increment our line count
@ -244,7 +264,7 @@ class ConfigBase(URLBase):
# containing all of the information parsed from our URL # containing all of the information parsed from our URL
results = plugins.SCHEMA_MAP[schema].parse_url(_url) results = plugins.SCHEMA_MAP[schema].parse_url(_url)
if not results: if results is None:
# Failed to parse the server URL # Failed to parse the server URL
logger.warning( logger.warning(
'Unparseable URL {} on line {}.'.format(url, line)) 'Unparseable URL {} on line {}.'.format(url, line))
@ -288,7 +308,234 @@ class ConfigBase(URLBase):
""" """
response = list() response = list()
# TODO try:
# Load our data
result = yaml.load(content)
except (AttributeError, yaml.error.MarkedYAMLError) as e:
# Invalid content
logger.error('Invalid apprise yaml data specified.')
logger.debug('YAML Exception:{}{}'.format(os.linesep, e))
return list()
if not isinstance(result, dict):
# Invalid content
logger.error('Invalid apprise yaml structure specified')
return list()
# YAML Version
version = result.get('version', 1)
if version != 1:
# Invalid syntax
logger.error(
'Invalid apprise yaml version specified {}.'.format(version))
return list()
#
# global asset object
#
asset = asset if isinstance(asset, AppriseAsset) else AppriseAsset()
tokens = result.get('asset', None)
if tokens and isinstance(tokens, dict):
for k, v in tokens.items():
if k.startswith('_') or k.endswith('_'):
# Entries are considered reserved if they start or end
# with an underscore
logger.warning('Ignored asset key "{}".'.format(k))
continue
if not (hasattr(asset, k) and
isinstance(getattr(asset, k), six.string_types)):
# We can't set a function or non-string set value
logger.warning('Invalid asset key "{}".'.format(k))
continue
if v is None:
# Convert to an empty string
v = ''
if not isinstance(v, six.string_types):
# we must set strings with a string
logger.warning('Invalid asset value to "{}".'.format(k))
continue
# Set our asset object with the new value
setattr(asset, k, v.strip())
#
# global tag root directive
#
global_tags = set()
tags = result.get('tag', None)
if tags and isinstance(tags, (list, tuple, six.string_types)):
# Store any preset tags
global_tags = set(parse_list(tags))
#
# urls root directive
#
urls = result.get('urls', None)
if not isinstance(urls, (list, tuple)):
# Unsupported
logger.error('Missing "urls" directive in apprise yaml.')
return list()
# Iterate over each URL
for no, url in enumerate(urls):
# Our results object is what we use to instantiate our object if
# we can. Reset it to None on each iteration
results = list()
if isinstance(url, six.string_types):
# We're just a simple URL string
# swap hash (#) tag values with their html version
_url = url.replace('/#', '/%23')
# Attempt to acquire the schema at the very least to allow our
# plugins to determine if they can make a better
# interpretation of a URL geared for them
schema = GET_SCHEMA_RE.match(_url)
if schema is None:
logger.warning(
'Unsupported schema in urls entry #{}'.format(no))
continue
# Ensure our schema is always in lower case
schema = schema.group('schema').lower()
# Some basic validation
if schema not in plugins.SCHEMA_MAP:
logger.warning(
'Unsupported schema {} in urls entry #{}'.format(
schema, no))
continue
# Parse our url details of the server object as dictionary
# containing all of the information parsed from our URL
_results = plugins.SCHEMA_MAP[schema].parse_url(_url)
if _results is None:
logger.warning(
'Unparseable {} based url; entry #{}'.format(
schema, no))
continue
# add our results to our global set
results.append(_results)
elif isinstance(url, dict):
# We are a url string with additional unescaped options
if six.PY2:
_url, tokens = next(url.iteritems())
else: # six.PY3
_url, tokens = next(iter(url.items()))
# swap hash (#) tag values with their html version
_url = _url.replace('/#', '/%23')
# Get our schema
schema = GET_SCHEMA_RE.match(_url)
if schema is None:
logger.warning(
'Unsupported schema in urls entry #{}'.format(no))
continue
# Ensure our schema is always in lower case
schema = schema.group('schema').lower()
# Some basic validation
if schema not in plugins.SCHEMA_MAP:
logger.warning(
'Unsupported schema {} in urls entry #{}'.format(
schema, no))
continue
# Parse our url details of the server object as dictionary
# containing all of the information parsed from our URL
_results = plugins.SCHEMA_MAP[schema].parse_url(_url)
if _results is None:
# Setup dictionary
_results = {
# Minimum requirements
'schema': schema,
}
if tokens is not None:
# populate and/or override any results populated by
# parse_url()
for entries in tokens:
# Copy ourselves a template of our parsed URL as a base
# to work with
r = _results.copy()
# We are a url string with additional unescaped options
if isinstance(entries, dict):
if six.PY2:
_url, tokens = next(url.iteritems())
else: # six.PY3
_url, tokens = next(iter(url.items()))
# Tags you just can't over-ride
if 'schema' in entries:
del entries['schema']
# Extend our dictionary with our new entries
r.update(entries)
# add our results to our global set
results.append(r)
else:
# add our results to our global set
results.append(_results)
else:
# Unsupported
logger.warning(
'Unsupported apprise yaml entry #{}'.format(no))
continue
# Track our entries
entry = 0
while len(results):
# Increment our entry count
entry += 1
# Grab our first item
_results = results.pop(0)
# tag is a special keyword that is managed by apprise object.
# The below ensures our tags are set correctly
if 'tag' in _results:
# Tidy our list up
_results['tag'] = \
set(parse_list(_results['tag'])) | global_tags
else:
# Just use the global settings
_results['tag'] = global_tags
# Prepare our Asset Object
_results['asset'] = asset
try:
# Attempt to create an instance of our plugin using the
# parsed URL information
plugin = plugins.SCHEMA_MAP[_results['schema']](**_results)
except Exception:
# the arguments are invalid or can not be used.
logger.warning(
'Could not load apprise yaml entry #{}, item #{}'
.format(no, entry))
continue
# if we reach here, we successfully loaded our data
response.append(plugin)
return response return response

View File

@ -36,6 +36,11 @@ from ..common import ConfigFormat
# application/x-yaml # application/x-yaml
MIME_IS_YAML = re.compile('(text|application)/(x-)?yaml', re.I) MIME_IS_YAML = re.compile('(text|application)/(x-)?yaml', re.I)
# Support TEXT formats
# text/plain
# text/html
MIME_IS_TEXT = re.compile('text/(plain|html)', re.I)
class ConfigHTTP(ConfigBase): class ConfigHTTP(ConfigBase):
""" """
@ -224,12 +229,21 @@ class ConfigHTTP(ConfigBase):
# Detect config format based on mime if the format isn't # Detect config format based on mime if the format isn't
# already enforced # already enforced
if self.config_format is None \ content_type = r.headers.get(
and MIME_IS_YAML.match(r.headers.get( 'Content-Type', 'application/octet-stream')
'Content-Type', 'text/plain')) is not None: if self.config_format is None and content_type:
if MIME_IS_YAML.match(content_type) is not None:
# YAML data detected based on header content # YAML data detected based on header content
self.default_config_format = ConfigFormat.YAML self.default_config_format = ConfigFormat.YAML
elif MIME_IS_TEXT.match(content_type) is not None:
# TEXT data detected based on header content
self.default_config_format = ConfigFormat.TEXT
# else do nothing; fall back to whatever default is
# already set.
except requests.RequestException as e: except requests.RequestException as e:
self.logger.warning( self.logger.warning(

View File

@ -136,7 +136,11 @@ def is_email(address):
and False if it isn't. and False if it isn't.
""" """
return GET_EMAIL_RE.match(address) is not None try:
return GET_EMAIL_RE.match(address) is not None
except TypeError:
# invalid syntax
return False
def tidy_path(path): def tidy_path(path):

View File

@ -6,3 +6,4 @@ urllib3
six six
click >= 5.0 click >= 5.0
markdown markdown
PyYAML

View File

@ -23,6 +23,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE. # THE SOFTWARE.
import six
from apprise.AppriseAsset import AppriseAsset from apprise.AppriseAsset import AppriseAsset
from apprise.config.ConfigBase import ConfigBase from apprise.config.ConfigBase import ConfigBase
@ -107,6 +108,12 @@ def test_config_base_config_parse_text():
""" """
# Garbage Handling
assert isinstance(ConfigBase.config_parse_text(object()), list)
assert isinstance(ConfigBase.config_parse_text(None), list)
assert isinstance(ConfigBase.config_parse_text(''), list)
# Valid Configuration
result = ConfigBase.config_parse_text(""" result = ConfigBase.config_parse_text("""
# A comment line over top of a URL # A comment line over top of a URL
mailto://userb:pass@gmail.com mailto://userb:pass@gmail.com
@ -166,3 +173,421 @@ def test_config_base_config_parse_text():
# We expect to parse 0 entries from the above # We expect to parse 0 entries from the above
assert isinstance(result, list) assert isinstance(result, list)
assert len(result) == 0 assert len(result) == 0
def test_config_base_config_parse_yaml():
"""
API: ConfigBase.config_parse_yaml object
"""
# general reference used below
asset = AppriseAsset()
# Garbage Handling
assert isinstance(ConfigBase.config_parse_yaml(object()), list)
assert isinstance(ConfigBase.config_parse_yaml(None), list)
assert isinstance(ConfigBase.config_parse_yaml(''), list)
# Invalid Version
result = ConfigBase.config_parse_yaml("version: 2a", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid Syntax (throws a ScannerError)
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
urls
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Missing url token
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# No urls defined
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
urls:
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url defined
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
# Invalid URL definition; yet the answer to life at the same time
urls: 43
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url/schema
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
urls:
- invalid://
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url/schema
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
urls:
- invalid://:
- a: b
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url/schema
result = ConfigBase.config_parse_yaml("""
urls:
- just some free text that isn't valid:
- a garbage entry to go with it
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url/schema
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
urls:
- not even a proper url
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url/schema
result = ConfigBase.config_parse_yaml("""
# no lists... just no
urls: [milk, pumpkin pie, eggs, juice]
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Invalid url/schema
result = ConfigBase.config_parse_yaml("""
urls:
# a very invalid sns entry
- sns://T1JJ3T3L2/
- sns://:@/:
- invalid: test
- sns://T1JJ3T3L2/:
- invalid: test
# some strangness
-
-
- test
""", asset=asset)
# Invalid data gets us an empty result set
assert isinstance(result, list)
assert len(result) == 0
# Valid Configuration
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed
version: 1
#
# Define your notification urls:
#
urls:
- pbul://o.gn5kj6nfhv736I7jC3cj3QLRiyhgl98b
- mailto://test:password@gmail.com
""", asset=asset)
# We expect to parse 2 entries from the above
assert isinstance(result, list)
assert len(result) == 2
assert len(result[0].tags) == 0
# Valid Configuration
result = ConfigBase.config_parse_yaml("""
urls:
- json://localhost:
- tag: my-custom-tag, my-other-tag
# How to stack multiple entries:
- mailto://:
- user: jeff
pass: 123abc
from: jeff@yahoo.ca
- user: jack
pass: pass123
from: jack@hotmail.com
# This is an illegal entry; the schema can not be changed
schema: json
# accidently left a colon at the end of the url; no problem
# we'll accept it
- mailto://oscar:pass@gmail.com:
# A telegram entry (returns a None in parse_url())
- tgram://invalid
""", asset=asset)
# We expect to parse 4 entries from the above because the tgram:// entry
# would have failed to be loaded
assert isinstance(result, list)
assert len(result) == 4
assert len(result[0].tags) == 2
# Global Tags
result = ConfigBase.config_parse_yaml("""
# Global Tags stacked as a list
tag:
- admin
- devops
urls:
- json://localhost
- dbus://
""", asset=asset)
# We expect to parse 3 entries from the above
assert isinstance(result, list)
assert len(result) == 2
# all entries will have our global tags defined in them
for entry in result:
assert 'admin' in entry.tags
assert 'devops' in entry.tags
# Global Tags
result = ConfigBase.config_parse_yaml("""
# Global Tags
tag: admin, devops
urls:
# The following tags will get added to the global set
- json://localhost:
- tag: string-tag, my-other-tag, text
# Tags can be presented in this list format too:
- dbus://:
- tag:
- list-tag
- dbus
""", asset=asset)
# all entries will have our global tags defined in them
for entry in result:
assert 'admin' in entry.tags
assert 'devops' in entry.tags
# We expect to parse 3 entries from the above
assert isinstance(result, list)
assert len(result) == 2
# json:// has 2 globals + 3 defined
assert len(result[0].tags) == 5
assert 'text' in result[0].tags
# json:// has 2 globals + 2 defined
assert len(result[1].tags) == 4
assert 'list-tag' in result[1].tags
# An invalid set of entries
result = ConfigBase.config_parse_yaml("""
urls:
# The following tags will get added to the global set
- json://localhost:
-
-
- entry
""", asset=asset)
# We expect to parse 3 entries from the above
assert isinstance(result, list)
assert len(result) == 0
# An asset we'll manipulate
asset = AppriseAsset()
# Global Tags
result = ConfigBase.config_parse_yaml("""
# Test the creation of our apprise asset object
asset:
app_id: AppriseTest
app_desc: Apprise Test Notifications
app_url: http://nuxref.com
# Support setting empty values
image_url_mask:
image_url_logo:
image_path_mask: tmp/path
# invalid entry
theme:
-
-
- entry
# Now for some invalid entries
invalid: entry
__init__: can't be over-ridden
nolists:
- we don't support these entries
- in the apprise object
urls:
- json://localhost:
""", asset=asset)
# We expect to parse 3 entries from the above
assert isinstance(result, list)
assert len(result) == 1
assert asset.app_id == "AppriseTest"
assert asset.app_desc == "Apprise Test Notifications"
assert asset.app_url == "http://nuxref.com"
# the theme was not updated and remains the same as it was
assert asset.theme == AppriseAsset().theme
# Empty string assignment
assert isinstance(asset.image_url_mask, six.string_types) is True
assert asset.image_url_mask == ""
assert isinstance(asset.image_url_logo, six.string_types) is True
assert asset.image_url_logo == ""
# For on-lookers looking through this file; here is a perfectly formatted
# YAML configuration file for your reference so you can see it without
# all of the errors like the ones identified above
result = ConfigBase.config_parse_yaml("""
# if no version is specified then version 1 is presumed. Thus this is a
# completely optional field. It's a good idea to just add this line because it
# will help with future ambiguity (if it ever occurs).
version: 1
# Define an Asset object if you wish (Optional)
asset:
app_id: AppriseTest
app_desc: Apprise Test Notifications
app_url: http://nuxref.com
# Optionally define some global tags to associate with ALL of your
# urls below.
tag: admin, devops
# Define your URLs (Mandatory!)
urls:
# Either on-line each entry like this:
- json://localhost
# Or add a colon to the end of the URL where you can optionally provide
# over-ride entries. One of the most likely entry to be used here
# is the tag entry. This gets extended to the global tag (if defined)
# above
- xml://localhost:
- tag: customer
# The more elements you specify under a URL the more times the URL will
# get replicated and used. Hence this entry actually could be considered
# 2 URLs being called with just the destination email address changed:
- mailto://george:password@gmail.com:
- to: jason@hotmail.com
- to: fred@live.com
# Again... to re-iterate, the above mailto:// would actually fire two (2)
# separate emails each with a different destination address specified.
# Be careful when defining your arguments and differentiating between
# when to use the dash (-) and when not to. Each time you do, you will
# cause another instance to be created.
# Defining more then 1 element to a muti-set is easy, it looks like this:
- mailto://jackson:abc123@hotmail.com:
- to: jeff@gmail.com
tag: jeff, customer
- to: chris@yahoo.com
tag: chris, customer
""", asset=asset)
# okay, here is how we get our total based on the above (read top-down)
# +1 json:// entry
# +1 xml:// entry
# +2 mailto:// entry to jason@hotmail.com and fred@live.com
# +2 mailto:// entry to jeff@gmail.com and chris@yahoo.com
# = 6
assert len(result) == 6
# all six entries will have our global tags defined in them
for entry in result:
assert 'admin' in entry.tags
assert 'devops' in entry.tags
# Entries can be directly accessed as they were added
# our json:// had no additional tags added; so just the global ones
# So just 2; admin and devops (these were already validated above in the
# for loop
assert len(result[0].tags) == 2
# our xml:// object has 1 tag added (customer)
assert len(result[1].tags) == 3
assert 'customer' in result[1].tags
# You get the idea, here is just a direct mapping to the remaining entries
# in the same order they appear above
assert len(result[2].tags) == 2
assert len(result[3].tags) == 2
assert len(result[4].tags) == 4
assert 'customer' in result[4].tags
assert 'jeff' in result[4].tags
assert len(result[5].tags) == 4
assert 'customer' in result[5].tags
assert 'chris' in result[5].tags

View File

@ -94,18 +94,27 @@ def test_config_http(mock_post, mock_get):
assert isinstance(ch.url(), six.string_types) is True assert isinstance(ch.url(), six.string_types) is True
assert isinstance(ch.read(), six.string_types) is True assert isinstance(ch.read(), six.string_types) is True
# one entry added
assert len(ch) == 1
results = ConfigHTTP.parse_url('http://localhost:8080/path/') results = ConfigHTTP.parse_url('http://localhost:8080/path/')
assert isinstance(results, dict) assert isinstance(results, dict)
ch = ConfigHTTP(**results) ch = ConfigHTTP(**results)
assert isinstance(ch.url(), six.string_types) is True assert isinstance(ch.url(), six.string_types) is True
assert isinstance(ch.read(), six.string_types) is True assert isinstance(ch.read(), six.string_types) is True
# one entry added
assert len(ch) == 1
results = ConfigHTTP.parse_url('http://user@localhost?format=text') results = ConfigHTTP.parse_url('http://user@localhost?format=text')
assert isinstance(results, dict) assert isinstance(results, dict)
ch = ConfigHTTP(**results) ch = ConfigHTTP(**results)
assert isinstance(ch.url(), six.string_types) is True assert isinstance(ch.url(), six.string_types) is True
assert isinstance(ch.read(), six.string_types) is True assert isinstance(ch.read(), six.string_types) is True
# one entry added
assert len(ch) == 1
results = ConfigHTTP.parse_url('https://localhost') results = ConfigHTTP.parse_url('https://localhost')
assert isinstance(results, dict) assert isinstance(results, dict)
ch = ConfigHTTP(**results) ch = ConfigHTTP(**results)
@ -153,10 +162,41 @@ def test_config_http(mock_post, mock_get):
for st in yaml_supported_types: for st in yaml_supported_types:
dummy_request.headers['Content-Type'] = st dummy_request.headers['Content-Type'] = st
ch.default_config_format = ConfigFormat.TEXT ch.default_config_format = None
assert isinstance(ch.read(), six.string_types) is True assert isinstance(ch.read(), six.string_types) is True
# Set to YAML
assert ch.default_config_format == ConfigFormat.YAML assert ch.default_config_format == ConfigFormat.YAML
# Test TEXT detection
text_supported_types = ('text/plain', 'text/html')
for st in text_supported_types:
dummy_request.headers['Content-Type'] = st
ch.default_config_format = None
assert isinstance(ch.read(), six.string_types) is True
# Set to TEXT
assert ch.default_config_format == ConfigFormat.TEXT
# The type is never adjusted to mime types we don't understand
ukwn_supported_types = ('text/css', 'application/zip')
for st in ukwn_supported_types:
dummy_request.headers['Content-Type'] = st
ch.default_config_format = None
assert isinstance(ch.read(), six.string_types) is True
# Remains unchanged
assert ch.default_config_format is None
# When the entry is missing; we handle this too
del dummy_request.headers['Content-Type']
ch.default_config_format = None
assert isinstance(ch.read(), six.string_types) is True
# Remains unchanged
assert ch.default_config_format is None
# Restore our content type object for lower tests
dummy_request.headers['Content-Type'] = 'text/plain'
ch.max_buffer_size = len(dummy_request.content) - 1 ch.max_buffer_size = len(dummy_request.content) - 1
assert ch.read() is None assert ch.read() is None

View File

@ -398,9 +398,12 @@ def test_is_email():
""" """
# Valid Emails # Valid Emails
assert utils.is_email('test@gmail.com') is True assert utils.is_email('test@gmail.com') is True
assert utils.is_email('tag+test@gmail.com') is True
# Invalid Emails # Invalid Emails
assert utils.is_email('invalid.com') is False assert utils.is_email('invalid.com') is False
assert utils.is_email(object()) is False
assert utils.is_email(None) is False
def test_parse_list(): def test_parse_list():