From 5683c0b087653e53c18f41c6afe2d6d6a4a71d9c Mon Sep 17 00:00:00 2001
From: Ryan Hitchman <rmmh@google.com>
Date: Mon, 11 Apr 2016 15:31:25 -0700
Subject: [PATCH] Make test-history generator create pages for individual
 suites

This makes it easier to determine which tests cause particular suites to
fail.

All static HTML pages are now generated by one invocation of gen_html.py.

- make index include good/flake/fail numbers for each link
- consistently use % for string interpolation
---
 hack/jenkins/test-history/gen_history         |  12 +-
 hack/jenkins/test-history/gen_html.py         | 143 ++++++++++++++----
 hack/jenkins/test-history/gen_html_test.py    |  73 +++++++++
 hack/jenkins/test-history/index_template.html |  15 --
 hack/jenkins/test-history/static/style.css    |  22 ++-
 5 files changed, 202 insertions(+), 63 deletions(-)
 create mode 100755 hack/jenkins/test-history/gen_html_test.py
 delete mode 100644 hack/jenkins/test-history/index_template.html

diff --git a/hack/jenkins/test-history/gen_history b/hack/jenkins/test-history/gen_history
index fb5cd4bd2a..8694704d5c 100755
--- a/hack/jenkins/test-history/gen_history
+++ b/hack/jenkins/test-history/gen_history
@@ -26,16 +26,8 @@ readonly datestr=$(date +"%Y-%m-%d")
 # Create JSON report
 time python gen_json.py "${jenkins}" kubernetes
 
-# Create static HTML report out of the JSON
-python gen_html.py > static/tests.html
-python gen_html.py kubernetes-e2e > static/tests-e2e.html
-python gen_html.py kubernetes-soak > static/tests-soak.html
-python gen_html.py kubernetes-e2e-gce > static/tests-e2e-gce.html
-python gen_html.py kubernetes-e2e-gke > static/tests-e2e-gke.html
-python gen_html.py kubernetes-upgrade > static/tests-upgrade.html
-
-# Fill in the last updated time into the template.
-cat index_template.html | sed -e "s/TIME/Last updated: ${datestr}/" > static/index.html
+# Create static HTML reports out of the JSON
+python gen_html.py --suites --prefixes ,e2e,soak,e2e-gce,e2e-gke,upgrade --output-dir static --input tests.json
 
 # Upload to GCS
 readonly bucket="kubernetes-test-history"
diff --git a/hack/jenkins/test-history/gen_html.py b/hack/jenkins/test-history/gen_html.py
index a461095194..06f622b9d0 100755
--- a/hack/jenkins/test-history/gen_html.py
+++ b/hack/jenkins/test-history/gen_html.py
@@ -27,20 +27,24 @@ JSON. That would allow custom filtering and stuff like that.
 
 from __future__ import print_function
 
+import argparse
 import json
+import os
 import string
 import sys
+import time
 
-def gen_tests(data, prefix):
+def gen_tests(data, prefix, exact_match):
     """Creates the HTML for all test cases.
 
     Args:
         data: Parsed JSON data that was created by gen_json.py.
         prefix: Considers Jenkins jobs that start with this.
+        exact_match: Only match Jenkins jobs with name equal to prefix.
 
     Returns:
-        The HTML as a list of elements along with the number of passing,
-        unstable, failing, and skipped tests.
+        The HTML as a list of elements along with a tuple of the number of
+        passing, unstable, failing, and skipped tests.
     """
     html = ['<ul class="test">']
     total_okay = 0
@@ -55,6 +59,8 @@ def gen_tests(data, prefix):
         for suite in sorted(data[test]):
             if not suite.startswith(prefix):
                 continue
+            if exact_match and suite != prefix:
+                continue
             has_test = True
             num_failed = 0
             num_builds = 0
@@ -78,10 +84,11 @@ def gen_tests(data, prefix):
             else:
                 status = 'okay'
             test_html.append('<li class="suite">')
-            test_html.append('<span class="{}">{}/{}</span>'.format(status, str(num_builds - num_failed), str(num_builds)))
-            test_html.append('<span class="time">{}</span>'.format(str(int(avg_time)) + unit))
+            test_html.append('<span class="%s">%d/%d</span>' % (status, num_builds - num_failed, num_builds))
+            test_html.append('<span class="time">%.0f%s</span>' % (avg_time, unit))
             test_html.append(suite)
             test_html.append('</li>')
+        test_html.append('</ul>')
         if has_failed:
             status = 'failed'
             total_failed += 1
@@ -94,42 +101,112 @@ def gen_tests(data, prefix):
         else:
             status = 'skipped'
             total_skipped += 1
-        html.append('<li class="test {}">{}'.format(status, test))
-        html.extend(test_html)
-        html.append('</ul>')
+        html.append('<li class="test %s">' % status)
+        if exact_match and len(test_html) > 2:
+            if not (test_html[2].startswith('<span') and test_html[3].startswith('<span')):
+                raise ValueError("couldn't extract suite results for prepending")
+            html.extend(test_html[2:4])
+            html.append(test)
+        else:
+            html.append(test)
+            html.extend(test_html)
         html.append('</li>')
     html.append('</ul>')
-    return html, total_okay, total_unstable, total_failed, total_skipped
+    return '\n'.join(html), (total_okay, total_unstable, total_failed, total_skipped)
 
-def gen_html(data, prefix):
-    """Creates the HTML for the entire page.
-
-    Args: Same as gen_tests.
-    Returns: Just the list of HTML elements.
-    """
-    tests_html, okay, unstable, failed, skipped = gen_tests(data, prefix)
+def html_header():
     html = ['<html>', '<head>']
     html.append('<link rel="stylesheet" type="text/css" href="style.css" />')
     html.append('<script src="script.js"></script>')
     html.append('</head>')
     html.append('<body>')
-    if len(prefix) > 0:
-        html.append('<div id="header">Suites starting with {}:'.format(prefix))
-    else:
-        html.append('<div id="header">All suites:')
-    html.append('<span class="total okay" onclick="toggle(\'okay\');">{}</span>'.format(str(okay)))
-    html.append('<span class="total unstable" onclick="toggle(\'unstable\');">{}</span>'.format(str(unstable)))
-    html.append('<span class="total failed" onclick="toggle(\'failed\');">{}</span>'.format(str(failed)))
-    html.append('<span class="total skipped" onclick="toggle(\'skipped\');">{}</span>'.format(str(skipped)))
-    html.append('</div>')
-    html.extend(tests_html)
-    html.append('</body>')
-    html.append('</html>')
     return html
 
+def gen_html(data, prefix, exact_match=False):
+    """Creates the HTML for the entire page.
+
+    Args: Same as gen_tests.
+    Returns: Same as gen_tests.
+    """
+    tests_html, (okay, unstable, failed, skipped) = gen_tests(data, prefix, exact_match)
+    html = html_header()
+    if exact_match:
+        html.append('<div id="header">Suite %s' % prefix)
+    elif len(prefix) > 0:
+        html.append('<div id="header">Suites starting with %s:' % prefix)
+    else:
+        html.append('<div id="header">All suites:')
+    html.append('<span class="total okay" onclick="toggle(\'okay\');">%s</span>' % okay)
+    html.append('<span class="total unstable" onclick="toggle(\'unstable\');">%d</span>' % unstable)
+    html.append('<span class="total failed" onclick="toggle(\'failed\');">%d</span>' % failed)
+    html.append('<span class="total skipped" onclick="toggle(\'skipped\');">%d</span>' % skipped)
+    html.append('</div>')
+    html.append(tests_html)
+    html.append('</body>')
+    html.append('</html>')
+    return '\n'.join(html), (okay, unstable, failed, skipped)
+
+def gen_metadata_links(suites):
+    html = []
+    for (name, target), (okay, unstable, failed, skipped) in sorted(suites.iteritems()):
+        html.append('<a class="suite-link" href="%s">' % target)
+        html.append('<span class="total okay">%d</span>' % okay)
+        html.append('<span class="total unstable">%d</span>' % unstable)
+        html.append('<span class="total failed">%d</span>' % failed)
+        html.append(name)
+        html.append('</a>')
+    return html
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--suites', action='store_true',
+                        help='output test results for each suite')
+    parser.add_argument('--prefixes',
+                        help='comma-separated list of suite prefixes to create pages for')
+    parser.add_argument('--output-dir', required=True,
+                        help='where to write output pages')
+    parser.add_argument('--input', required=True,
+                        help='JSON test data to read for input')
+    options=parser.parse_args(args)
+
+    with open(options.input) as f:
+        data = json.load(f)
+
+    if options.prefixes:
+        # the empty prefix means "all tests"
+        options.prefixes = options.prefixes.split(',')
+        prefix_metadata = {}
+        for prefix in options.prefixes:
+            if prefix:
+                path = 'tests-%s.html' % prefix
+                prefix = 'kubernetes-%s' % prefix
+            else:
+                path = 'tests.html'
+            html, prefix_metadata[prefix or 'kubernetes', path] = gen_html(data, prefix, False)
+            with open(os.path.join(options.output_dir, path), 'w') as f:
+                f.write(html)
+    if options.suites:
+        suites_set = set()
+        for test, suites in data.iteritems():
+            suites_set.update(suites.keys())
+        suite_metadata = {}
+        for suite in sorted(suites_set):
+            path = 'suite-%s.html' % suite
+            html, suite_metadata[suite, path] = gen_html(data, suite, True)
+            with open(os.path.join(options.output_dir, path), 'w') as f:
+                f.write(html)
+    html = html_header()
+    html.append('<h1>Kubernetes Tests</h1>')
+    html.append('Last updated %s' % time.strftime('%F'))
+    if options.prefixes:
+        html.append('<h2>All suites starting with:</h2>')
+        html.extend(gen_metadata_links(prefix_metadata))
+    if options.suites:
+        html.append('<h2>Specific suites:</h2>')
+        html.extend(gen_metadata_links(suite_metadata))
+    html.extend(['</body>', '</html>'])
+    with open(os.path.join(options.output_dir, 'index.html'), 'w') as f:
+        f.write('\n'.join(html))
+
 if __name__ == '__main__':
-    prefix = ''
-    if len(sys.argv) == 2:
-        prefix = sys.argv[1]
-    with open('tests.json', 'r') as f:
-        print('\n'.join(gen_html(json.load(f), prefix)))
+    main(sys.argv[1:])
diff --git a/hack/jenkins/test-history/gen_html_test.py b/hack/jenkins/test-history/gen_html_test.py
new file mode 100755
index 0000000000..60fbc03d4f
--- /dev/null
+++ b/hack/jenkins/test-history/gen_html_test.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for gen_html."""
+
+import json
+import os
+import shutil
+import tempfile
+import unittest
+
+import gen_html
+
+TEST_DATA = {
+  "test1":
+      {"kubernetes-release": [{"build": 3, "failed": False, "time": 3.52},
+                              {"build": 4, "failed": True, "time": 63.21}],
+       "kubernetes-debug": [{"build": 5, "failed": False, "time": 7.56},
+                            {"build": 6, "failed": False, "time": 8.43}],
+      },
+  "test2":
+      {"kubernetes-debug": [{"build": 6, "failed": True, "time": 3.53}]},
+}
+
+class GenHtmlTest(unittest.TestCase):
+    def gen_html(self, *args):
+        return gen_html.gen_html(TEST_DATA, *args)[0]
+
+    def testGenHtml(self):
+        html = self.gen_html('')
+        self.assertIn("test1", html)
+        self.assertIn("test2", html)
+        self.assertIn("release", html)
+        self.assertIn("debug", html)
+
+    def testGenHtmlFilter(self):
+        html = self.gen_html('release')
+        self.assertIn("release", html)
+        self.assertIn('skipped">\ntest2', html)
+        self.assertNotIn("debug", html)
+
+    def testGenHtmlFilterExact(self):
+        html = self.gen_html('release', True)
+        self.assertNotIn('debug', html)
+
+    def testMain(self):
+        temp_dir = tempfile.mkdtemp(prefix='kube-test-hist-')
+        try:
+            tests_json = os.path.join(temp_dir, 'tests.json')
+            with open(tests_json, 'w') as f:
+                json.dump(TEST_DATA, f)
+            gen_html.main(['--suites', '--prefixes', ',rel,deb',
+                           '--output-dir', temp_dir, '--input', tests_json])
+            for page in ('index', 'suite-kubernetes-debug', 'tests', 'tests-rel', 'tests-deb'):
+                self.assertTrue(os.path.exists('%s/%s.html' % (temp_dir, page)))
+        finally:
+            shutil.rmtree(temp_dir)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/hack/jenkins/test-history/index_template.html b/hack/jenkins/test-history/index_template.html
deleted file mode 100644
index f54a0b6b90..0000000000
--- a/hack/jenkins/test-history/index_template.html
+++ /dev/null
@@ -1,15 +0,0 @@
-<html>
-<body>
-<h2>All tests starting with:</h2>
-<ul>
-<li><a href="tests.html">kubernetes</a></li>
-<li><a href="tests-e2e.html">kubernetes-e2e</a></li>
-<li><a href="tests-e2e-gce.html">kubernetes-e2e-gce</a></li>
-<li><a href="tests-e2e-gke.html">kubernetes-e2e-gke</a></li>
-<li><a href="tests-upgrade.html">kubernetes-upgrade</a></li>
-<li><a href="tests-soak.html">kubernetes-soak</a></li>
-</ul>
-TIME
-</body>
-</html>
-
diff --git a/hack/jenkins/test-history/static/style.css b/hack/jenkins/test-history/static/style.css
index 1a23685eeb..c03bc2da37 100644
--- a/hack/jenkins/test-history/static/style.css
+++ b/hack/jenkins/test-history/static/style.css
@@ -11,7 +11,14 @@ body {
     padding: 10px;
 }
 
-#header span{
+a.suite-link {
+    font-weight: bold;
+    font-size: large;
+    display: block;
+    text-decoration: none;
+}
+
+span.total {
     display: inline-block;
     width: 60px;
     padding-left: 30px;
@@ -91,19 +98,24 @@ li.suite span {
     font-weight: bold;
 }
 
-li.suite span.time {
+li.test span.time {
     width: 50px;
     font-weight: normal;
 }
 
-li.suite span.okay {
+li.test span.okay {
     color: green;
 }
 
-li.suite span.unstable {
+li.test span.unstable {
     color: orange;
 }
 
-li.suite span.failed {
+li.test span.failed {
     color: red;
 }
+
+li.test>span {
+    display: inline-block;
+    text-align: right;
+}