Added Support for the log appender roll-by-size-time to zip older files (#259)

* Introduced the following new elements.
1. logname - you can override the name of the log file rather than using the EXE name, this means you don't have to call your EXE a different name, just name the winsw exe different. Default's the name to the EXE as before.
2. outfiledisabled - you can disable writing to the out file. Defaults to false.
3. errfiledisabled - you can disable writing to the error file. Defaults to false.
4. outfilepattern - you can choose the pattern of the out file. Defaults to .out.log.
5. errfilepattern - you can choos the pattern of the error file. Defaults to .err.log.

* Downgraded from C#7.0 syntax.

* Applied reviewers comment

* not required

* removed the key

* Added unit test for new fields logname, outfiledisabled, errfiledisabled and errfilepattern.

Created a new appender called roll-by-size-time see class RollingSizeTimeLogAppender, this appender supports rolling by time and size and rolling at a specific time each day.

Added unit test for the new appender.

Added a new option testwait which is similar to test but waits for the user to press any key before calling the stop method.

* Update loggingAndErrorReporting.md

* Cannot use $ string.format syntax, downgraded code to string.format.

* Another syntax found of $

* Fixed a unit tests

* Added support to zip files.

* Added error handling

* Removed the zip call at startup.

* Fix issue with UTC

* Update loggingAndErrorReporting.md

Documented the new fields zipolderthannumdays and zipdateformat

* Update loggingAndErrorReporting.md

* Applied Code review

* Fixed a BST bug

* Added zip lib

.gitignore

@@ -8,3 +8,4 @@ obj
 /winsw_key.pfx
 /src/.vs/winsw/v15/sqlite3/storage.ide
 /src/Core/WinSWCore/WinSWCore.csproj.DotSettings
+/src/.vs/winsw/v15/Server/sqlite3

doc/loggingAndErrorReporting.md

@@ -56,6 +56,8 @@ Works in a combination of rotate size mode and rotate time mode, if the log file
       <sizeThreshold>10240</sizeThreshold>
       <pattern>yyyyMMdd</pattern>
       <autoRollAtTime>00:00:00</autoRollAtTime>
+      <zipOlderThanNumDays>5</zipOlderThanNumDays>
+      <zipDateFormat>yyyyMM</zipDateFormat>
     </log>
 ```
 
@@ -65,6 +67,21 @@ For example, in the above example, the log of Jan 1, 2013 gets written to `myapp
 The syntax of the autoRollAtTime is specified by [TimeSpan.ToString()](https://msdn.microsoft.com/en-us/library/1ecy8h51(v=vs.110).aspx).
 For example, in the above example, at the start of the day it will roll the file over.
 
+The zipOlderThanNumDays can only be used in conjection with autoRollAtTime, provide the number of days of files to keep.
+```
+    <log mode="roll-by-size-time">
+      <autoRollAtTime>00:00:00</autoRollAtTime>
+      <zipOlderThanNumDays>5</zipOlderThanNumDays>
+    </log>
+```
+The zipDateFormat can only be used in conjection with autoRollAtTime, provide the zip file format using the [DateTime.ToString()](http://msdn.microsoft.com/en-us/library/zdtaw1bw.aspx).
+```
+    <log mode="roll-by-size-time">
+      <autoRollAtTime>00:00:00</autoRollAtTime>
+      <zipDateFormat>yyyyMM</zipDateFormat>
+    </log>
+```
+
 ### Error reporting
 
 Winsw uses WMI underneath, and as such it uses its error code as the exit code. 

src/Core/ServiceWrapper/packages.config

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="ICSharpCode.SharpZipLib.dll" version="0.85.4.369" targetFramework="net20" />
   <package id="ILMerge" version="2.14.1208" targetFramework="net20" />
   <package id="log4net" version="2.0.8" targetFramework="net20" />
   <package id="MSBuildTasks" version="1.4.0.88" targetFramework="net20" />

src/Core/ServiceWrapper/winsw.csproj

@@ -62,6 +62,9 @@
     </DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ICSharpCode.SharpZipLib, Version=0.85.4.369, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\ICSharpCode.SharpZipLib.dll.0.85.4.369\lib\net20\ICSharpCode.SharpZipLib.dll</HintPath>
+    </Reference>
     <Reference Include="log4net">
       <HintPath>..\..\packages\log4net.2.0.8\lib\net20-full\log4net.dll</HintPath>
     </Reference>

src/Core/ServiceWrapper_dotNET4/packages.config

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="ICSharpCode.SharpZipLib.dll" version="0.85.4.369" targetFramework="net40" />
   <package id="ILMerge" version="2.14.1208" targetFramework="net20" />
   <package id="log4net" version="2.0.8" targetFramework="net20" requireReinstallation="True" />
   <package id="MSBuildTasks" version="1.4.0.88" targetFramework="net20" />
-</packages>
+</packages>

src/Core/ServiceWrapper_dotNET4/winsw_dotNET4.csproj

@@ -63,6 +63,10 @@
     </DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ICSharpCode.SharpZipLib, Version=0.85.4.369, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\ICSharpCode.SharpZipLib.dll.0.85.4.369\lib\net20\ICSharpCode.SharpZipLib.dll</HintPath>
+      <EmbedInteropTypes>False</EmbedInteropTypes>
+    </Reference>
     <Reference Include="log4net">
       <HintPath>..\..\packages\log4net.2.0.8\lib\net20-full\log4net.dll</HintPath>
     </Reference>

src/Core/WinSWCore/LogAppenders.cs

@@ -3,6 +3,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Threading;
 using System.Timers;
+using ICSharpCode.SharpZipLib.Zip;
 
 namespace winsw
 {
@@ -328,13 +329,17 @@ namespace winsw
         public int SizeTheshold { private set; get; }
         public string FilePattern { private set; get; }
         public TimeSpan? AutoRollAtTime { private set; get; }
+        public int? ZipOlderThanNumDays { private set; get; }
+        public string ZipDateFormat { private set; get; }
 
-        public RollingSizeTimeLogAppender(string logDirectory, string baseName, bool outFileDisabled, bool errFileDisabled, string outFilePattern, string errFilePattern, int sizeThreshold, string filePattern, TimeSpan? autoRollAtTime)
+        public RollingSizeTimeLogAppender(string logDirectory, string baseName, bool outFileDisabled, bool errFileDisabled, string outFilePattern, string errFilePattern, int sizeThreshold, string filePattern, TimeSpan? autoRollAtTime, int? zipolderthannumdays, string zipdateformat)
             : base(logDirectory, baseName, outFileDisabled, errFileDisabled, outFilePattern, errFilePattern)
         {
             SizeTheshold = sizeThreshold;
             FilePattern = filePattern;
             AutoRollAtTime = autoRollAtTime;
+            ZipOlderThanNumDays = zipolderthannumdays;
+            ZipDateFormat = zipdateformat;
         }
 
         public override void log(Stream outputStream, Stream errorStream)
@@ -360,6 +365,7 @@ namespace winsw
             // We auto roll at time is configured then we need to create a timer and wait until time is elasped and roll the file over
             if (AutoRollAtTime != null)
             {
+                // Run at start
                 var tickTime = SetupRollTimer();
                 var timer = new System.Timers.Timer(tickTime);
                 timer.Elapsed += (s, e) =>
@@ -371,13 +377,17 @@ namespace winsw
                         {
                             w.Close();
 
-                            var nextFileNumber = GetNextFileNumber(ext, baseDirectory, baseFileName);
-                            var nextFileName =  Path.Combine(baseDirectory, string.Format("{0}.{1}.#{2:D4}{3}", baseFileName, DateTime.UtcNow.ToString(FilePattern), nextFileNumber, ext));
+                            var now = DateTime.Now.AddDays(-1);
+                            var nextFileNumber = GetNextFileNumber(ext, baseDirectory, baseFileName, now);
+                            var nextFileName =  Path.Combine(baseDirectory, string.Format("{0}.{1}.#{2:D4}{3}", baseFileName, now.ToString(FilePattern), nextFileNumber, ext));
                             File.Move(logFile, nextFileName);
 
                             w = new FileStream(logFile, FileMode.Create);
                             sz = new FileInfo(logFile).Length;
                         }
+
+                        // Next day so check if file can be zipped
+                        ZipFiles(baseDirectory, ext, baseFileName);
                     }
                     catch (Exception et)
                     {
@@ -422,10 +432,11 @@ namespace winsw
                                 s = i + 1;
 
                                 // rotate file
-                                var nextFileNumber = GetNextFileNumber(ext, baseDirectory, baseFileName);
+                                var now = DateTime.Now;
+                                var nextFileNumber = GetNextFileNumber(ext, baseDirectory, baseFileName, now);
                                 var nextFileName =
                                     Path.Combine(baseDirectory,
-                                        string.Format("{0}.{1}.#{2:D4}{3}", baseFileName, DateTime.UtcNow.ToString(FilePattern), nextFileNumber, ext));
+                                        string.Format("{0}.{1}.#{2:D4}{3}", baseFileName, now.ToString(FilePattern), nextFileNumber, ext));
                                 File.Move(logFile, nextFileName);
 
                                 // even if the log rotation fails, create a new one, or else
@@ -446,22 +457,100 @@ namespace winsw
             w.Close();
         }
 
+        private void ZipFiles(string path, string fileExt, string baseZipfilename)
+        {
+            if (ZipOlderThanNumDays == null || !(ZipOlderThanNumDays > 0)) return;
+
+            try
+            {
+                var files = Directory.GetFiles(path, "*" + fileExt);
+                foreach (var file in files)
+                {
+                    var fi = new FileInfo(file);
+                    if (fi.LastWriteTimeUtc >= DateTime.UtcNow.AddDays(-ZipOlderThanNumDays.Value)) continue;
+
+                    // lets archive this bugger
+                    ZipTheFile(file, path, fi.LastWriteTimeUtc.ToString(ZipDateFormat), baseZipfilename);
+                    File.Delete(file);
+                }
+            }
+            catch (Exception e)
+            {
+                EventLogger.LogEvent(string.Format("Failed to Zip File. Error {0}", e.Message));
+            }
+        }
+
+        private void ZipTheFile(string filename, string zipPath, string zipFilePattern, string baseZipfilename)
+        {
+            var zipfilename = Path.Combine(zipPath, string.Format("{0}.{1}.zip", baseZipfilename, zipFilePattern));
+            ZipFile zipFile = null;
+            bool commited = false;
+            try
+            {
+
+                if (File.Exists(zipfilename))
+                {
+                    zipFile = new ZipFile(zipfilename);
+                    TestZipfile(zipFile, zipfilename);
+                }
+                else
+                {
+                    zipFile = ZipFile.Create(zipfilename);
+                }
+
+                zipFile.BeginUpdate();
+                zipFile.NameTransform = new ZipNameTransform(zipPath);
+                var relFile = Path.GetFileName(filename);
+                if (zipFile.FindEntry(relFile, true) == -1)
+                {
+                    zipFile.Add(filename);
+                }
+
+                zipFile.CommitUpdate();
+                commited = true;
+                TestZipfile(zipFile, zipfilename);
+            }
+            catch (Exception e)
+            {
+                EventLogger.LogEvent(string.Format("Failed to Zip the File {0}. Error {1}", filename, e.Message));
+                if (zipFile != null && !commited)
+                    zipFile.AbortUpdate();
+            }
+            finally
+            {
+                if (zipFile != null)
+                {
+                    zipFile.Close();
+                }
+            }
+        }
+
+        static void TestZipfile(ZipFile zipFile, string zipArchive)
+        {
+            var testResult = zipFile.TestArchive(true);
+            if (!testResult)
+            {
+                var em = string.Format("Bad zip file \"{0}\"", zipArchive);
+                throw new ApplicationException(em);
+            }
+        }
+
         private double SetupRollTimer()
         {
-            var nowTime = DateTime.Now.ToUniversalTime();
+            var nowTime = DateTime.Now;
             var scheduledTime = new DateTime(nowTime.Year, nowTime.Month, nowTime.Day, AutoRollAtTime.Value.Hours,
-                AutoRollAtTime.Value.Minutes, AutoRollAtTime.Value.Seconds, 0).ToUniversalTime(); //Specify your time HH,MM,SS
+                AutoRollAtTime.Value.Minutes, AutoRollAtTime.Value.Seconds, 0); //Specify your time HH,MM,SS
             if (nowTime > scheduledTime)
                 scheduledTime = scheduledTime.AddDays(1);
 
-            double tickTime = (double) (scheduledTime - DateTime.Now.ToUniversalTime()).TotalMilliseconds;
+            double tickTime = (double) (scheduledTime - DateTime.Now).TotalMilliseconds;
             return tickTime;
         }
 
-        private int GetNextFileNumber(string ext, string baseDirectory, string baseFileName)
+        private int GetNextFileNumber(string ext, string baseDirectory, string baseFileName, DateTime now)
         {
             var nextFileNumber = 0;
-            var files = Directory.GetFiles(baseDirectory, String.Format("{0}.{1}.#*{2}", baseFileName, DateTime.UtcNow.ToString(FilePattern), ext));
+            var files = Directory.GetFiles(baseDirectory, String.Format("{0}.{1}.#*{2}", baseFileName, now.ToString(FilePattern), ext));
             if (files.Length == 0)
             {
                 nextFileNumber = 1;

src/Core/WinSWCore/ServiceDescriptor.cs

@@ -466,8 +466,29 @@ namespace winsw
                                 throw new InvalidDataException("Roll-Size-Time Based rolling policy is specified but autoRollAtTime does not match the TimeSpan format HH:mm:ss found in configuration XML.");
                             autoRollAtTime = autoRollAtTimeValue;
                         }
+                        XmlNode zipolderthannumdaysNode = e.SelectSingleNode("zipOlderThanNumDays");
+                        int? zipolderthannumdays = null;
+                        if (zipolderthannumdaysNode != null)
+                        {
+                            int zipolderthannumdaysValue;
+                            // validate it
+                            if (!int.TryParse(zipolderthannumdaysNode.InnerText, out zipolderthannumdaysValue))
+                                throw new InvalidDataException("Roll-Size-Time Based rolling policy is specified but zipOlderThanNumDays does not match the int format found in configuration XML.");
+                            zipolderthannumdays = zipolderthannumdaysValue;
+                        }
 
-                        return new RollingSizeTimeLogAppender(LogDirectory, LogName, OutFileDisabled, ErrFileDisabled, OutFilePattern, ErrFilePattern, sizeThreshold, filePatternNode.InnerText, autoRollAtTime);
+                        XmlNode zipdateformatNode = e.SelectSingleNode("zipDateFormat");
+                        string zipdateformat = null;
+                        if (zipdateformatNode == null)
+                        {
+                            zipdateformat = "yyyyMM";
+                        }
+                        else
+                        {
+                            zipdateformat = zipdateformatNode.InnerText;
+                        }
+
+                        return new RollingSizeTimeLogAppender(LogDirectory, LogName, OutFileDisabled, ErrFileDisabled, OutFilePattern, ErrFilePattern, sizeThreshold, filePatternNode.InnerText, autoRollAtTime, zipolderthannumdays, zipdateformat);
 
                     default:
                         throw new InvalidDataException("Undefined logging mode: " + LogMode);

src/Core/WinSWCore/WinSWCore.csproj

@@ -36,6 +36,9 @@
     <AssemblyOriginatorKeyFile>$(SolutionDir)..\winsw_key.snk</AssemblyOriginatorKeyFile>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="ICSharpCode.SharpZipLib, Version=0.85.4.369, Culture=neutral, PublicKeyToken=1b03e6acf1164f73, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\ICSharpCode.SharpZipLib.dll.0.85.4.369\lib\net20\ICSharpCode.SharpZipLib.dll</HintPath>
+    </Reference>
     <Reference Include="log4net">
       <HintPath>..\..\packages\log4net.2.0.8\lib\net20-full\log4net.dll</HintPath>
     </Reference>

src/Core/WinSWCore/packages.config

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="ICSharpCode.SharpZipLib.dll" version="0.85.4.369" targetFramework="net20" />
   <package id="log4net" version="2.0.8" targetFramework="net20" />
 </packages>