From 35dc0b11f0b13d7cfb3fef430f3c0f861b1e5770 Mon Sep 17 00:00:00 2001 From: Buddhika Chathuranga Date: Fri, 21 Aug 2020 07:06:39 +0530 Subject: [PATCH] YAML support for extension (#638) * Yaml support for extensions * Update RunawayProcessKillerTest.cs * Update yaml_extension_support Add unit test * Update Yaml extension support * Upate extension yaml support Remove old object query class and add methods to extension which can take yaml objects as aruments. Add unit test for runawayprocesskiller for yaml support * Use stong types instead object to manage extension confiuration with YAML * Add unit test for SharedDirectoryMapper yaml config * Update yaml extension support Improve error messages improve boolean parser remove unncessary logs --- .../Configuration/DefaultSettings.cs | 2 + .../Configuration/IWinSWConfiguration.cs | 2 + .../Configuration/YamlConfiguration.cs | 41 ++++++++++-- .../YamlExtensionConfiguration.cs | 51 ++++++++++++++ .../Extensions/AbstractWinSWExtension.cs | 5 ++ .../WinSWCore/Extensions/IWinSWExtension.cs | 7 ++ .../Extensions/WinSWExtensionDescriptor.cs | 10 +++ .../Extensions/WinSWExtensionManager.cs | 66 ++++++++++++++++++- src/Core/WinSWCore/ServiceDescriptor.cs | 2 + src/Core/WinSWCore/ServiceDescriptorYaml.cs | 4 +- src/Core/WinSWCore/Util/ConfigHelper.cs | 12 ++++ .../RunawayProcessKillerExtension.cs | 19 ++++++ .../SharedDirectoryMapper.cs | 19 ++++++ .../SharedDirectoryMapperConfig.cs | 19 +++++- .../Extensions/RunawayProcessKillerTest.cs | 42 +++++++++++- .../Extensions/SharedDirectoryMapperTest.cs | 50 +++++++++++++- 16 files changed, 339 insertions(+), 12 deletions(-) create mode 100644 src/Core/WinSWCore/Configuration/YamlExtensionConfiguration.cs diff --git a/src/Core/WinSWCore/Configuration/DefaultSettings.cs b/src/Core/WinSWCore/Configuration/DefaultSettings.cs index aa45c01..6c6de51 100644 --- a/src/Core/WinSWCore/Configuration/DefaultSettings.cs +++ b/src/Core/WinSWCore/Configuration/DefaultSettings.cs @@ -135,6 +135,8 @@ namespace WinSW.Configuration // Extensions public XmlNode? ExtensionsConfiguration => null; + public List? YamlExtensionsConfiguration => new List(0); + public string BaseName { get diff --git a/src/Core/WinSWCore/Configuration/IWinSWConfiguration.cs b/src/Core/WinSWCore/Configuration/IWinSWConfiguration.cs index 1b21450..7c6080f 100644 --- a/src/Core/WinSWCore/Configuration/IWinSWConfiguration.cs +++ b/src/Core/WinSWCore/Configuration/IWinSWConfiguration.cs @@ -76,6 +76,8 @@ namespace WinSW.Configuration // Extensions XmlNode? ExtensionsConfiguration { get; } + List? YamlExtensionsConfiguration { get; } + List ExtensionIds { get; } // Service Account diff --git a/src/Core/WinSWCore/Configuration/YamlConfiguration.cs b/src/Core/WinSWCore/Configuration/YamlConfiguration.cs index 2686d7b..4f6432b 100644 --- a/src/Core/WinSWCore/Configuration/YamlConfiguration.cs +++ b/src/Core/WinSWCore/Configuration/YamlConfiguration.cs @@ -106,9 +106,6 @@ namespace WinSW.Configuration [YamlMember(Alias = "securityDescriptor")] public string? SecurityDescriptorYaml { get; set; } - [YamlMember(Alias = "extensions")] - public List? YamlExtensionIds { get; set; } - public class YamlEnv { [YamlMember(Alias = "name")] @@ -658,10 +655,42 @@ namespace WinSW.Configuration public string LogMode => this.Log.Mode is null ? this.Defaults.LogMode : this.Log.Mode; - // TODO - Extensions - XmlNode? IWinSWConfiguration.ExtensionsConfiguration => throw new NotImplementedException(); + public XmlNode? ExtensionsConfiguration => null; - public List ExtensionIds => this.YamlExtensionIds ?? this.Defaults.ExtensionIds; + // YAML Extension + [YamlMember(Alias = "extensions")] + public List? YamlExtensionsConfiguration { get; set; } + + public List ExtensionIds + { + get + { + int extensionNumber = 1; + + if (this.YamlExtensionsConfiguration is null) + { + return new List(0); + } + + var result = new List(this.YamlExtensionsConfiguration.Count); + + foreach (var item in this.YamlExtensionsConfiguration) + { + try + { + result.Add(item.GetId()); + } + catch (InvalidDataException) + { + throw new InvalidDataException("Id is null in Extension " + extensionNumber); + } + + extensionNumber++; + } + + return result; + } + } public string BaseName { get; set; } diff --git a/src/Core/WinSWCore/Configuration/YamlExtensionConfiguration.cs b/src/Core/WinSWCore/Configuration/YamlExtensionConfiguration.cs new file mode 100644 index 0000000..9ea32f6 --- /dev/null +++ b/src/Core/WinSWCore/Configuration/YamlExtensionConfiguration.cs @@ -0,0 +1,51 @@ +using System.Collections.Generic; +using System.IO; +using YamlDotNet.Serialization; + +namespace WinSW.Configuration +{ + public class YamlExtensionConfiguration + { + [YamlMember(Alias = "id")] + public string? ExtensionId { get; set; } + + [YamlMember(Alias = "className")] + public string? ExtensionClassName { get; set; } + + [YamlMember(Alias = "enabled")] + public bool Enabled { get; set; } + + [YamlMember(Alias = "settings")] + public Dictionary? Settings { get; set; } + + public string GetId() + { + if (this.ExtensionId is null) + { + throw new InvalidDataException(); + } + + return this.ExtensionId; + } + + public string GetClassName() + { + if (this.ExtensionClassName is null) + { + throw new InvalidDataException($@"Extension ClassName is empty in extension {this.GetId()}"); + } + + return this.ExtensionClassName; + } + + public Dictionary GetSettings() + { + if (this.Settings is null) + { + throw new InvalidDataException(@$"Extension settings is empty in extension {this.GetId()}"); + } + + return this.Settings; + } + } +} diff --git a/src/Core/WinSWCore/Extensions/AbstractWinSWExtension.cs b/src/Core/WinSWCore/Extensions/AbstractWinSWExtension.cs index a3b4733..141f2d2 100644 --- a/src/Core/WinSWCore/Extensions/AbstractWinSWExtension.cs +++ b/src/Core/WinSWCore/Extensions/AbstractWinSWExtension.cs @@ -16,6 +16,11 @@ namespace WinSW.Extensions // Do nothing } + public virtual void Configure(IWinSWConfiguration descriptor, YamlExtensionConfiguration config) + { + // Do nothing + } + public virtual void OnWrapperStarted() { // Do nothing diff --git a/src/Core/WinSWCore/Extensions/IWinSWExtension.cs b/src/Core/WinSWCore/Extensions/IWinSWExtension.cs index d187be8..fb683e4 100644 --- a/src/Core/WinSWCore/Extensions/IWinSWExtension.cs +++ b/src/Core/WinSWCore/Extensions/IWinSWExtension.cs @@ -30,6 +30,13 @@ namespace WinSW.Extensions /// Configuration node void Configure(IWinSWConfiguration descriptor, XmlNode node); + /// + /// Configure the extension from Yaml configuration + /// + /// YamlConfiguration + /// Configuration Node + void Configure(IWinSWConfiguration descriptor, YamlExtensionConfiguration config); + /// /// Start handler. Called during startup of the service before the child process. /// diff --git a/src/Core/WinSWCore/Extensions/WinSWExtensionDescriptor.cs b/src/Core/WinSWCore/Extensions/WinSWExtensionDescriptor.cs index e8794f1..838555b 100644 --- a/src/Core/WinSWCore/Extensions/WinSWExtensionDescriptor.cs +++ b/src/Core/WinSWCore/Extensions/WinSWExtensionDescriptor.cs @@ -1,4 +1,5 @@ using System.Xml; +using WinSW.Configuration; using WinSW.Util; namespace WinSW.Extensions @@ -40,5 +41,14 @@ namespace WinSW.Extensions string id = XmlHelper.SingleAttribute(node, "id"); return new WinSWExtensionDescriptor(id, className, enabled); } + + public static WinSWExtensionDescriptor FromYaml(YamlExtensionConfiguration config) + { + bool enabled = config.Enabled; + string className = config.GetClassName(); + string id = config.GetId(); + + return new WinSWExtensionDescriptor(id, className, enabled); + } } } diff --git a/src/Core/WinSWCore/Extensions/WinSWExtensionManager.cs b/src/Core/WinSWCore/Extensions/WinSWExtensionManager.cs index 3b0bf6a..d440950 100644 --- a/src/Core/WinSWCore/Extensions/WinSWExtensionManager.cs +++ b/src/Core/WinSWCore/Extensions/WinSWExtensionManager.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Xml; using log4net; using WinSW.Configuration; @@ -127,6 +128,18 @@ namespace WinSW.Extensions throw new ExtensionException(id, "Extension has been already loaded"); } + if (this.ServiceDescriptor.GetType() == typeof(ServiceDescriptor)) + { + this.LoadExtensionFromXml(id); + } + else + { + this.LoadExtensionFromYaml(id); + } + } + + private void LoadExtensionFromXml(string id) + { XmlNode? extensionsConfig = this.ServiceDescriptor.ExtensionsConfiguration; XmlElement? configNode = extensionsConfig is null ? null : extensionsConfig.SelectSingleNode("extension[@id='" + id + "'][1]") as XmlElement; if (configNode is null) @@ -135,6 +148,7 @@ namespace WinSW.Extensions } var descriptor = WinSWExtensionDescriptor.FromXml(configNode); + if (descriptor.Enabled) { IWinSWExtension extension = this.CreateExtensionInstance(descriptor.Id, descriptor.ClassName); @@ -146,7 +160,7 @@ namespace WinSW.Extensions catch (Exception ex) { // Consider any unexpected exception as fatal Log.Fatal("Failed to configure the extension " + id, ex); - throw ex; + throw new ExtensionException(id, "Failed to configure the extension"); } this.Extensions.Add(id, extension); @@ -158,6 +172,56 @@ namespace WinSW.Extensions } } + private void LoadExtensionFromYaml(string id) + { + var extensionConfigList = this.ServiceDescriptor.YamlExtensionsConfiguration; + + if (extensionConfigList is null) + { + throw new ExtensionException(id, "Cannot get the configuration entry"); + } + + var configNode = GetYamlonfigById(extensionConfigList, id); + + var descriptor = WinSWExtensionDescriptor.FromYaml(configNode); + + if (descriptor.Enabled) + { + IWinSWExtension extension = this.CreateExtensionInstance(descriptor.Id, descriptor.ClassName); + extension.Descriptor = descriptor; + + try + { + extension.Configure(this.ServiceDescriptor, configNode); + } + catch (Exception ex) + { // Consider any unexpected exception as fatal + Log.Fatal("Failed to configure the extension " + id, ex); + throw new ExtensionException(id, "Failed to configure the extension"); + } + + this.Extensions.Add(id, extension); + Log.Info("Extension loaded: " + id); + } + else + { + Log.Warn("Extension is disabled: " + id); + } + + YamlExtensionConfiguration GetYamlonfigById(List configs, string id) + { + foreach (var item in configs) + { + if (item.GetId().Equals(id)) + { + return item; + } + } + + throw new ExtensionException(id, $@"Can't find extension with id: ""{id}"" "); + } + } + private IWinSWExtension CreateExtensionInstance(string id, string className) { object created; diff --git a/src/Core/WinSWCore/ServiceDescriptor.cs b/src/Core/WinSWCore/ServiceDescriptor.cs index 78d8872..8782307 100644 --- a/src/Core/WinSWCore/ServiceDescriptor.cs +++ b/src/Core/WinSWCore/ServiceDescriptor.cs @@ -703,5 +703,7 @@ namespace WinSW return environment; } + + public List? YamlExtensionsConfiguration => Defaults.YamlExtensionsConfiguration; } } diff --git a/src/Core/WinSWCore/ServiceDescriptorYaml.cs b/src/Core/WinSWCore/ServiceDescriptorYaml.cs index 332b78b..d731879 100644 --- a/src/Core/WinSWCore/ServiceDescriptorYaml.cs +++ b/src/Core/WinSWCore/ServiceDescriptorYaml.cs @@ -18,7 +18,7 @@ namespace WinSW using (var reader = new StreamReader(basepath + ".yml")) { var file = reader.ReadToEnd(); - var deserializer = new DeserializerBuilder().Build(); + var deserializer = new DeserializerBuilder().IgnoreUnmatchedProperties().Build(); this.Configurations = deserializer.Deserialize(file); } @@ -47,7 +47,7 @@ namespace WinSW public static ServiceDescriptorYaml FromYaml(string yaml) { - var deserializer = new DeserializerBuilder().Build(); + var deserializer = new DeserializerBuilder().IgnoreUnmatchedProperties().Build(); var configs = deserializer.Deserialize(yaml); return new ServiceDescriptorYaml(configs); } diff --git a/src/Core/WinSWCore/Util/ConfigHelper.cs b/src/Core/WinSWCore/Util/ConfigHelper.cs index 8ca87fa..8579a2b 100644 --- a/src/Core/WinSWCore/Util/ConfigHelper.cs +++ b/src/Core/WinSWCore/Util/ConfigHelper.cs @@ -33,5 +33,17 @@ namespace WinSW.Util { "day", 1000L * 60L * 60L * 24L }, { "days", 1000L * 60L * 60L * 24L } }; + + public static bool YamlBoolParse(string value) + { + value = value.ToLower(); + + if (value.Equals("true") || value.Equals("yes") || value.Equals("on") || value.Equals("y") || value.Equals("1")) + { + return true; + } + + return false; + } } } diff --git a/src/Plugins/RunawayProcessKiller/RunawayProcessKillerExtension.cs b/src/Plugins/RunawayProcessKiller/RunawayProcessKillerExtension.cs index 2f495cf..13a82f5 100644 --- a/src/Plugins/RunawayProcessKiller/RunawayProcessKillerExtension.cs +++ b/src/Plugins/RunawayProcessKiller/RunawayProcessKillerExtension.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Collections.Specialized; using System.Diagnostics; using System.IO; @@ -193,6 +194,24 @@ namespace WinSW.Plugins.RunawayProcessKiller this.CheckWinSWEnvironmentVariable = checkWinSWEnvironmentVariable is null ? true : bool.Parse(checkWinSWEnvironmentVariable); } + public override void Configure(IWinSWConfiguration descriptor, YamlExtensionConfiguration config) + { + var dict = config.GetSettings(); + + this.Pidfile = (string)dict["pidfile"]; + this.StopTimeout = TimeSpan.FromMilliseconds(int.Parse((string)dict["stopTimeOut"])); + this.StopParentProcessFirst = bool.Parse((string)dict["StopParentFirst"]); + + try + { + this.CheckWinSWEnvironmentVariable = bool.Parse((string)dict["checkWinSWEnvironmentVariable"]); + } + catch + { + this.CheckWinSWEnvironmentVariable = true; + } + } + /// /// This method checks if the PID file is stored on the disk and then terminates runaway processes if they exist. /// diff --git a/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapper.cs b/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapper.cs index 4813ebb..f638e5c 100644 --- a/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapper.cs +++ b/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapper.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; using System.Xml; using log4net; using WinSW.Configuration; @@ -43,6 +44,24 @@ namespace WinSW.Plugins.SharedDirectoryMapper } } + public override void Configure(IWinSWConfiguration descriptor, YamlExtensionConfiguration config) + { + var dict = config.GetSettings(); + + var mappingNode = dict["mapping"]; + + if (!(mappingNode is List mappings)) + { + throw new InvalidDataException("SharedDirectoryMapper mapping should be a list"); + } + + foreach (var map in mappings) + { + var mapConfig = SharedDirectoryMapperConfig.FromYaml(map); + this._entries.Add(mapConfig); + } + } + public override void OnWrapperStarted() { foreach (SharedDirectoryMapperConfig config in this._entries) diff --git a/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapperConfig.cs b/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapperConfig.cs index 273e207..0a9402c 100644 --- a/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapperConfig.cs +++ b/src/Plugins/SharedDirectoryMapper/SharedDirectoryMapperConfig.cs @@ -1,4 +1,6 @@ -using System.Xml; +using System.Collections.Generic; +using System.IO; +using System.Xml; using WinSW.Util; namespace WinSW.Plugins.SharedDirectoryMapper @@ -26,5 +28,20 @@ namespace WinSW.Plugins.SharedDirectoryMapper string uncPath = XmlHelper.SingleAttribute(node, "uncpath"); return new SharedDirectoryMapperConfig(enableMapping, label, uncPath); } + + public static SharedDirectoryMapperConfig FromYaml(object yamlObject) + { + if (!(yamlObject is Dictionary dict)) + { + // TODO : throw ExtensionExeption + throw new InvalidDataException("SharedDirectoryMapperConfig config error"); + } + + bool enableMapping = ConfigHelper.YamlBoolParse((string)dict["enabled"]); + string label = (string)dict["label"]; + string uncPath = (string)dict["uncpath"]; + + return new SharedDirectoryMapperConfig(enableMapping, label, uncPath); + } } } diff --git a/src/Test/winswTests/Extensions/RunawayProcessKillerTest.cs b/src/Test/winswTests/Extensions/RunawayProcessKillerTest.cs index b92bcfb..dea027a 100644 --- a/src/Test/winswTests/Extensions/RunawayProcessKillerTest.cs +++ b/src/Test/winswTests/Extensions/RunawayProcessKillerTest.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.IO; using NUnit.Framework; using WinSW; +using WinSW.Configuration; using WinSW.Extensions; using WinSW.Plugins.RunawayProcessKiller; using WinSW.Util; @@ -14,7 +15,8 @@ namespace winswTests.Extensions [TestFixture] class RunawayProcessKillerExtensionTest : ExtensionTestBase { - ServiceDescriptor _testServiceDescriptor; + IWinSWConfiguration _testServiceDescriptor; + IWinSWConfiguration _testServiceDescriptorYaml; readonly string testExtension = GetExtensionClassNameWithAssembly(typeof(RunawayProcessKillerExtension)); @@ -38,6 +40,29 @@ $@" "; this._testServiceDescriptor = ServiceDescriptor.FromXML(seedXml); + + string seedYaml = $@"--- +id: jenkins +name: Jenkins +description: This service runs Jenkins automation server. +env: + - + name: JENKINS_HOME + value: '%LocalAppData%\Jenkins.jenkins' +executable: java +arguments: >- + -Xrs -Xmx256m -Dhudson.lifecycle=hudson.lifecycle.WindowsServiceLifecycle + -jar E:\Winsw Test\yml6\jenkins.war --httpPort=8081 +extensions: + - id: killRunawayProcess + enabled: yes + className: ""{this.testExtension}"" + settings: + pidfile: 'foo/bar/pid.txt' + stopTimeOut: 5000 + StopParentFirst: true"; + + this._testServiceDescriptorYaml = ServiceDescriptorYaml.FromYaml(seedYaml).Configurations; } [Test] @@ -55,6 +80,21 @@ $@" Assert.AreEqual(true, extension.StopParentProcessFirst, "Loaded StopParentFirst is not equal to the expected one"); } + [Test] + public void LoadExtensionsYaml() + { + WinSWExtensionManager manager = new WinSWExtensionManager(this._testServiceDescriptorYaml); + manager.LoadExtensions(); + Assert.AreEqual(1, manager.Extensions.Count, "One extension should be loaded"); + + // Check the file is correct + var extension = manager.Extensions["killRunawayProcess"] as RunawayProcessKillerExtension; + Assert.IsNotNull(extension, "RunawayProcessKillerExtension should be loaded"); + Assert.AreEqual("foo/bar/pid.txt", extension.Pidfile, "Loaded PID file path is not equal to the expected one"); + Assert.AreEqual(5000, extension.StopTimeout.TotalMilliseconds, "Loaded Stop Timeout is not equal to the expected one"); + Assert.AreEqual(true, extension.StopParentProcessFirst, "Loaded StopParentFirst is not equal to the expected one"); + } + [Test] public void StartStopExtension() { diff --git a/src/Test/winswTests/Extensions/SharedDirectoryMapperTest.cs b/src/Test/winswTests/Extensions/SharedDirectoryMapperTest.cs index e0217a4..3b0136c 100644 --- a/src/Test/winswTests/Extensions/SharedDirectoryMapperTest.cs +++ b/src/Test/winswTests/Extensions/SharedDirectoryMapperTest.cs @@ -1,5 +1,6 @@ using NUnit.Framework; using WinSW; +using WinSW.Configuration; using WinSW.Extensions; using WinSW.Plugins.SharedDirectoryMapper; @@ -8,7 +9,8 @@ namespace winswTests.Extensions [TestFixture] class SharedDirectoryMapperTest : ExtensionTestBase { - ServiceDescriptor _testServiceDescriptor; + IWinSWConfiguration _testServiceDescriptor; + IWinSWConfiguration _testServiceDescriptorYaml; readonly string testExtension = GetExtensionClassNameWithAssembly(typeof(SharedDirectoryMapper)); @@ -39,6 +41,44 @@ $@" "; this._testServiceDescriptor = ServiceDescriptor.FromXML(seedXml); + + string seedYaml = $@"--- +id: jenkins +name: Jenkins +description: This service runs Jenkins automation server. +env: + - + name: JENKINS_HOME + value: '%LocalAppData%\Jenkins.jenkins' +executable: java +arguments: >- + -Xrs -Xmx256m -Dhudson.lifecycle=hudson.lifecycle.WindowsServiceLifecycle + -jar E:\Winsw Test\yml6\jenkins.war --httpPort=8081 +extensions: + - id: mapNetworDirs + className: ""{this.testExtension}"" + enabled: true + settings: + mapping: + - enabled: false + label: N + uncpath: \\UNC + - enabled: false + label: M + uncpath: \\UNC2 + - id: mapNetworDirs2 + className: ""{this.testExtension}"" + enabled: true + settings: + mapping: + - enabled: false + label: X + uncpath: \\UNC + - enabled: false + label: Y + uncpath: \\UNC2"; + + this._testServiceDescriptorYaml = ServiceDescriptorYaml.FromYaml(seedYaml).Configurations; } [Test] @@ -49,6 +89,14 @@ $@" Assert.AreEqual(2, manager.Extensions.Count, "Two extensions should be loaded"); } + [Test] + public void LoadExtensionsYaml() + { + WinSWExtensionManager manager = new WinSWExtensionManager(this._testServiceDescriptorYaml); + manager.LoadExtensions(); + Assert.AreEqual(2, manager.Extensions.Count, "Two extensions should be loaded"); + } + [Test] public void StartStopExtension() {