Feat 4612 real time metrics for kube nodes (#4708)

* feat(k8s/node): display realtime node metrics GH#4612

* feat(k8s): show observation timestamp instead of real timestamp GH#4612

Co-authored-by: Simon Meng <simon.meng@portainer.io>

app/constants.js

@@ -30,3 +30,5 @@ angular
   .constant('PREDEFINED_NETWORKS', ['host', 'bridge', 'none'])
   .constant('KUBERNETES_DEFAULT_NAMESPACE', 'default')
   .constant('KUBERNETES_SYSTEM_NAMESPACES', ['kube-system', 'kube-public', 'kube-node-lease', 'portainer']);
+
+export const PORTAINER_FADEOUT = 1500;

app/kubernetes/__module.js

@@ -182,6 +182,16 @@ angular.module('portainer.kubernetes', ['portainer.app']).config([
       },
     };
 
+    const nodeStats = {
+      name: 'kubernetes.cluster.node.stats',
+      url: '/stats',
+      views: {
+        'content@': {
+          component: 'kubernetesNodeStatsView',
+        },
+      },
+    };
+
     const dashboard = {
       name: 'kubernetes.dashboard',
       url: '/dashboard',
@@ -280,6 +290,7 @@ angular.module('portainer.kubernetes', ['portainer.app']).config([
     $stateRegistryProvider.register(dashboard);
     $stateRegistryProvider.register(deploy);
     $stateRegistryProvider.register(node);
+    $stateRegistryProvider.register(nodeStats);
     $stateRegistryProvider.register(resourcePools);
     $stateRegistryProvider.register(resourcePoolCreation);
     $stateRegistryProvider.register(resourcePool);

app/kubernetes/components/datatables/nodes-datatable/nodesDatatable.html

@@ -107,6 +107,9 @@
                   <i class="fa fa-sort-alpha-up" aria-hidden="true" ng-if="$ctrl.state.orderBy === 'IPAddress' && $ctrl.state.reverseOrder"></i>
                 </a>
               </th>
+              <th ng-if="$ctrl.useServerMetrics">
+                Actions
+              </th>
             </tr>
           </thead>
           <tbody>
@@ -128,6 +131,9 @@
               <td>{{ item.Memory | humansize }}</td>
               <td>{{ item.Version }}</td>
               <td>{{ item.IPAddress }}</td>
+              <td ng-if="$ctrl.useServerMetrics">
+                <a ui-sref="kubernetes.cluster.node.stats({ name: item.Name })" style="cursor: pointer;"> <i class="fa fa-chart-area" aria-hidden="true"></i> Stats </a>
+              </td>
             </tr>
             <tr ng-if="!$ctrl.dataset">
               <td colspan="7" class="text-center text-muted">Loading...</td>

app/kubernetes/components/datatables/nodes-datatable/nodesDatatable.js

@@ -9,5 +9,6 @@ angular.module('portainer.kubernetes').component('kubernetesNodesDatatable', {
     orderBy: '@',
     refreshCallback: '<',
     isAdmin: '<',
+    useServerMetrics: '<',
   },
 });

app/kubernetes/metrics/metrics.js

@@ -10,6 +10,7 @@ class KubernetesMetricsService {
 
     this.capabilitiesAsync = this.capabilitiesAsync.bind(this);
     this.getPodAsync = this.getPodAsync.bind(this);
+    this.getNodeAsync = this.getNodeAsync.bind(this);
   }
 
   /**
@@ -27,6 +28,26 @@ class KubernetesMetricsService {
     return this.$async(this.capabilitiesAsync, endpointID);
   }
 
+  /**
+   * Stats of Node
+   *
+   * @param {string} nodeName
+   */
+  async getNodeAsync(nodeName) {
+    try {
+      const params = new KubernetesCommonParams();
+      params.id = nodeName;
+      const data = await this.KubernetesMetrics().getNode(params).$promise;
+      return data;
+    } catch (err) {
+      throw new PortainerError('Unable to retrieve node stats', err);
+    }
+  }
+
+  getNode(nodeName) {
+    return this.$async(this.getNodeAsync, nodeName);
+  }
+
   /**
    * Stats
    *

app/kubernetes/metrics/rest.js

@@ -20,6 +20,10 @@ angular.module('portainer.kubernetes').factory('KubernetesMetrics', [
             method: 'GET',
             url: podUrl,
           },
+          getNode: {
+            method: 'GET',
+            url: `${url}/nodes/:id`,
+          },
         }
       );
     };

app/kubernetes/views/cluster/cluster.html

@@ -89,6 +89,7 @@
         order-by="Name"
         refresh-callback="ctrl.getNodes"
         is-admin="ctrl.isAdmin"
+        use-server-metrics="ctrl.state.useServerMetrics"
       ></kubernetes-nodes-datatable>
     </div>
   </div>

app/kubernetes/views/cluster/clusterController.js

@@ -15,7 +15,8 @@ class KubernetesClusterController {
     KubernetesNodeService,
     KubernetesApplicationService,
     KubernetesComponentStatusService,
-    KubernetesEndpointService
+    KubernetesEndpointService,
+    EndpointProvider
   ) {
     this.$async = $async;
     this.$state = $state;
@@ -26,6 +27,7 @@ class KubernetesClusterController {
     this.KubernetesApplicationService = KubernetesApplicationService;
     this.KubernetesComponentStatusService = KubernetesComponentStatusService;
     this.KubernetesEndpointService = KubernetesEndpointService;
+    this.EndpointProvider = EndpointProvider;
 
     this.onInit = this.onInit.bind(this);
     this.getNodes = this.getNodes.bind(this);
@@ -132,6 +134,7 @@ class KubernetesClusterController {
     }
 
     this.state.viewReady = true;
+    this.state.useServerMetrics = this.EndpointProvider.currentEndpoint().Kubernetes.Configuration.UseServerMetrics;
   }
 
   $onInit() {

app/kubernetes/views/cluster/node/stats/stats.html

@@ -0,0 +1,71 @@
+<kubernetes-view-header title="Node stats" state="kubernetes.cluster.node.stats" view-ready="ctrl.state.viewReady">
+  <a ui-sref="kubernetes.cluster">Cluster</a> &gt; <a ui-sref="kubernetes.cluster.node({name: ctrl.state.transition.nodeName})"> {{ ctrl.state.transition.nodeName }} </a> >
+  {{ ctrl.state.transition.nodeName }}
+</kubernetes-view-header>
+
+<kubernetes-view-loading view-ready="ctrl.state.viewReady"></kubernetes-view-loading>
+
+<div ng-if="ctrl.state.viewReady">
+  <information-panel ng-if="!ctrl.state.getMetrics" title-text="Unable to retrieve node metrics">
+    <span class="small text-muted">
+      <i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
+      Portainer was unable to retrieve any metrics associated to that node. Please contact your administrator to ensure that the Kubernetes metrics feature is properly configured.
+    </span>
+  </information-panel>
+  <div class="row" ng-if="ctrl.state.getMetrics">
+    <div class="col-md-12">
+      <rd-widget>
+        <rd-widget-header icon="fa-info-circle" title-text="About statistics"> </rd-widget-header>
+        <rd-widget-body>
+          <form class="form-horizontal">
+            <div class="form-group">
+              <div class="col-sm-12">
+                <span class="small text-muted">
+                  This view displays real-time statistics about the node <b>{{ ctrl.state.transition.nodeName }}</b
+                  >.
+                </span>
+              </div>
+            </div>
+            <div class="form-group">
+              <label for="refreshRate" class="col-sm-3 col-md-2 col-lg-2 margin-sm-top control-label text-left">
+                Refresh rate
+              </label>
+              <div class="col-sm-3 col-md-2">
+                <select id="refreshRate" ng-model="ctrl.state.refreshRate" ng-change="ctrl.changeUpdateRepeater()" class="form-control">
+                  <option value="30">30s</option>
+                  <option value="60">60s</option>
+                </select>
+              </div>
+              <span>
+                <i id="refreshRateChange" class="fa fa-check green-icon" aria-hidden="true" style="margin-top: 7px; display: none;"></i>
+              </span>
+            </div>
+          </form>
+        </rd-widget-body>
+      </rd-widget>
+    </div>
+  </div>
+
+  <div class="row" ng-show="ctrl.state.getMetrics">
+    <div class="col-lg-6 col-md-12 col-sm-12">
+      <rd-widget>
+        <rd-widget-header icon="fa-chart-area" title-text="Memory usage"></rd-widget-header>
+        <rd-widget-body>
+          <div class="chart-node" style="position: relative;">
+            <canvas id="memoryChart" width="770" height="300"></canvas>
+          </div>
+        </rd-widget-body>
+      </rd-widget>
+    </div>
+    <div class="col-lg-6 col-md-12 col-sm-12">
+      <rd-widget>
+        <rd-widget-header icon="fa-chart-area" title-text="CPU usage"></rd-widget-header>
+        <rd-widget-body>
+          <div class="chart-node" style="position: relative;">
+            <canvas id="cpuChart" width="770" height="300"></canvas>
+          </div>
+        </rd-widget-body>
+      </rd-widget>
+    </div>
+  </div>
+</div>

app/kubernetes/views/cluster/node/stats/stats.js

@@ -0,0 +1,8 @@
+angular.module('portainer.kubernetes').component('kubernetesNodeStatsView', {
+  templateUrl: './stats.html',
+  controller: 'KubernetesNodeStatsController',
+  controllerAs: 'ctrl',
+  bindings: {
+    $transition$: '<',
+  },
+});

app/kubernetes/views/cluster/node/stats/statsController.js

@@ -0,0 +1,144 @@
+import angular from 'angular';
+import moment from 'moment';
+import filesizeParser from 'filesize-parser';
+import KubernetesResourceReservationHelper from 'Kubernetes/helpers/resourceReservationHelper';
+import { PORTAINER_FADEOUT } from '@/constants';
+
+class KubernetesNodeStatsController {
+  /* @ngInject */
+  constructor($async, $state, $interval, $document, Notifications, KubernetesNodeService, KubernetesMetricsService, ChartService) {
+    this.$async = $async;
+    this.$state = $state;
+    this.$interval = $interval;
+    this.$document = $document;
+    this.Notifications = Notifications;
+    this.KubernetesNodeService = KubernetesNodeService;
+    this.KubernetesMetricsService = KubernetesMetricsService;
+    this.ChartService = ChartService;
+
+    this.onInit = this.onInit.bind(this);
+  }
+
+  changeUpdateRepeater() {
+    var cpuChart = this.cpuChart;
+    var memoryChart = this.memoryChart;
+
+    this.stopRepeater();
+    this.setUpdateRepeater(cpuChart, memoryChart);
+    $('#refreshRateChange').show();
+    $('#refreshRateChange').fadeOut(PORTAINER_FADEOUT);
+  }
+
+  updateCPUChart() {
+    const label = moment(this.stats.read).format('HH:mm:ss');
+    this.ChartService.UpdateCPUChart(label, this.stats.CPUUsage, this.cpuChart);
+  }
+
+  updateMemoryChart() {
+    const label = moment(this.stats.read).format('HH:mm:ss');
+    this.ChartService.UpdateMemoryChart(label, this.stats.MemoryUsage, 0, this.memoryChart);
+  }
+
+  stopRepeater() {
+    var repeater = this.repeater;
+    if (angular.isDefined(repeater)) {
+      this.$interval.cancel(repeater);
+      this.repeater = undefined;
+    }
+  }
+
+  setUpdateRepeater() {
+    const refreshRate = this.state.refreshRate;
+
+    this.repeater = this.$interval(async () => {
+      try {
+        await this.getStats();
+        this.updateCPUChart();
+        this.updateMemoryChart();
+      } catch (error) {
+        this.stopRepeater();
+        this.Notifications.error('Failure', error);
+      }
+    }, refreshRate * 1000);
+  }
+
+  initCharts() {
+    const cpuChartCtx = $('#cpuChart');
+    const cpuChart = this.ChartService.CreateCPUChart(cpuChartCtx);
+    this.cpuChart = cpuChart;
+
+    const memoryChartCtx = $('#memoryChart');
+    const memoryChart = this.ChartService.CreateMemoryChart(memoryChartCtx);
+    this.memoryChart = memoryChart;
+
+    this.updateCPUChart();
+    this.updateMemoryChart();
+    this.setUpdateRepeater();
+  }
+
+  getStats() {
+    return this.$async(async () => {
+      try {
+        const stats = await this.KubernetesMetricsService.getNode(this.state.transition.nodeName);
+        if (stats) {
+          const memory = filesizeParser(stats.usage.memory);
+          const cpu = KubernetesResourceReservationHelper.parseCPU(stats.usage.cpu);
+          this.stats = {
+            read: stats.creationTimestamp,
+            MemoryUsage: memory,
+            CPUUsage: (cpu / this.nodeCPU) * 100,
+          };
+        }
+      } catch (err) {
+        this.Notifications.error('Failure', err, 'Unable to retrieve node stats');
+      }
+    });
+  }
+
+  $onDestroy() {
+    this.stopRepeater();
+  }
+
+  async onInit() {
+    this.state = {
+      autoRefresh: false,
+      refreshRate: '30',
+      viewReady: false,
+      transition: {
+        nodeName: this.$transition$.params().name,
+      },
+      getMetrics: true,
+    };
+
+    try {
+      const nodeMetrics = await this.KubernetesMetricsService.getNode(this.state.transition.nodeName);
+
+      if (nodeMetrics) {
+        const node = await this.KubernetesNodeService.get(this.state.transition.nodeName);
+        this.nodeCPU = node.CPU || 1;
+
+        await this.getStats();
+
+        if (this.state.getMetrics) {
+          this.$document.ready(() => {
+            this.initCharts();
+          });
+        }
+      } else {
+        this.state.getMetrics = false;
+      }
+    } catch (err) {
+      this.state.getMetrics = false;
+      this.Notifications.error('Failure', err, 'Unable to retrieve node stats');
+    } finally {
+      this.state.viewReady = true;
+    }
+  }
+
+  $onInit() {
+    return this.$async(this.onInit);
+  }
+}
+
+export default KubernetesNodeStatsController;
+angular.module('portainer.kubernetes').controller('KubernetesNodeStatsController', KubernetesNodeStatsController);