mirror of https://github.com/portainer/portainer
				
				
				
			Feat 4612 real time metrics for kube nodes (#4708)
* feat(k8s/node): display realtime node metrics GH#4612 * feat(k8s): show observation timestamp instead of real timestamp GH#4612 Co-authored-by: Simon Meng <simon.meng@portainer.io>pull/5047/head^2
							parent
							
								
									45ceece1a9
								
							
						
					
					
						commit
						dc180d85c5
					
				| 
						 | 
				
			
			@ -30,3 +30,5 @@ angular
 | 
			
		|||
  .constant('PREDEFINED_NETWORKS', ['host', 'bridge', 'none'])
 | 
			
		||||
  .constant('KUBERNETES_DEFAULT_NAMESPACE', 'default')
 | 
			
		||||
  .constant('KUBERNETES_SYSTEM_NAMESPACES', ['kube-system', 'kube-public', 'kube-node-lease', 'portainer']);
 | 
			
		||||
 | 
			
		||||
export const PORTAINER_FADEOUT = 1500;
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -182,6 +182,16 @@ angular.module('portainer.kubernetes', ['portainer.app']).config([
 | 
			
		|||
      },
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    const nodeStats = {
 | 
			
		||||
      name: 'kubernetes.cluster.node.stats',
 | 
			
		||||
      url: '/stats',
 | 
			
		||||
      views: {
 | 
			
		||||
        'content@': {
 | 
			
		||||
          component: 'kubernetesNodeStatsView',
 | 
			
		||||
        },
 | 
			
		||||
      },
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    const dashboard = {
 | 
			
		||||
      name: 'kubernetes.dashboard',
 | 
			
		||||
      url: '/dashboard',
 | 
			
		||||
| 
						 | 
				
			
			@ -280,6 +290,7 @@ angular.module('portainer.kubernetes', ['portainer.app']).config([
 | 
			
		|||
    $stateRegistryProvider.register(dashboard);
 | 
			
		||||
    $stateRegistryProvider.register(deploy);
 | 
			
		||||
    $stateRegistryProvider.register(node);
 | 
			
		||||
    $stateRegistryProvider.register(nodeStats);
 | 
			
		||||
    $stateRegistryProvider.register(resourcePools);
 | 
			
		||||
    $stateRegistryProvider.register(resourcePoolCreation);
 | 
			
		||||
    $stateRegistryProvider.register(resourcePool);
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -107,6 +107,9 @@
 | 
			
		|||
                  <i class="fa fa-sort-alpha-up" aria-hidden="true" ng-if="$ctrl.state.orderBy === 'IPAddress' && $ctrl.state.reverseOrder"></i>
 | 
			
		||||
                </a>
 | 
			
		||||
              </th>
 | 
			
		||||
              <th ng-if="$ctrl.useServerMetrics">
 | 
			
		||||
                Actions
 | 
			
		||||
              </th>
 | 
			
		||||
            </tr>
 | 
			
		||||
          </thead>
 | 
			
		||||
          <tbody>
 | 
			
		||||
| 
						 | 
				
			
			@ -128,6 +131,9 @@
 | 
			
		|||
              <td>{{ item.Memory | humansize }}</td>
 | 
			
		||||
              <td>{{ item.Version }}</td>
 | 
			
		||||
              <td>{{ item.IPAddress }}</td>
 | 
			
		||||
              <td ng-if="$ctrl.useServerMetrics">
 | 
			
		||||
                <a ui-sref="kubernetes.cluster.node.stats({ name: item.Name })" style="cursor: pointer;"> <i class="fa fa-chart-area" aria-hidden="true"></i> Stats </a>
 | 
			
		||||
              </td>
 | 
			
		||||
            </tr>
 | 
			
		||||
            <tr ng-if="!$ctrl.dataset">
 | 
			
		||||
              <td colspan="7" class="text-center text-muted">Loading...</td>
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -9,5 +9,6 @@ angular.module('portainer.kubernetes').component('kubernetesNodesDatatable', {
 | 
			
		|||
    orderBy: '@',
 | 
			
		||||
    refreshCallback: '<',
 | 
			
		||||
    isAdmin: '<',
 | 
			
		||||
    useServerMetrics: '<',
 | 
			
		||||
  },
 | 
			
		||||
});
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -10,6 +10,7 @@ class KubernetesMetricsService {
 | 
			
		|||
 | 
			
		||||
    this.capabilitiesAsync = this.capabilitiesAsync.bind(this);
 | 
			
		||||
    this.getPodAsync = this.getPodAsync.bind(this);
 | 
			
		||||
    this.getNodeAsync = this.getNodeAsync.bind(this);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  /**
 | 
			
		||||
| 
						 | 
				
			
			@ -27,6 +28,26 @@ class KubernetesMetricsService {
 | 
			
		|||
    return this.$async(this.capabilitiesAsync, endpointID);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  /**
 | 
			
		||||
   * Stats of Node
 | 
			
		||||
   *
 | 
			
		||||
   * @param {string} nodeName
 | 
			
		||||
   */
 | 
			
		||||
  async getNodeAsync(nodeName) {
 | 
			
		||||
    try {
 | 
			
		||||
      const params = new KubernetesCommonParams();
 | 
			
		||||
      params.id = nodeName;
 | 
			
		||||
      const data = await this.KubernetesMetrics().getNode(params).$promise;
 | 
			
		||||
      return data;
 | 
			
		||||
    } catch (err) {
 | 
			
		||||
      throw new PortainerError('Unable to retrieve node stats', err);
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  getNode(nodeName) {
 | 
			
		||||
    return this.$async(this.getNodeAsync, nodeName);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  /**
 | 
			
		||||
   * Stats
 | 
			
		||||
   *
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -20,6 +20,10 @@ angular.module('portainer.kubernetes').factory('KubernetesMetrics', [
 | 
			
		|||
            method: 'GET',
 | 
			
		||||
            url: podUrl,
 | 
			
		||||
          },
 | 
			
		||||
          getNode: {
 | 
			
		||||
            method: 'GET',
 | 
			
		||||
            url: `${url}/nodes/:id`,
 | 
			
		||||
          },
 | 
			
		||||
        }
 | 
			
		||||
      );
 | 
			
		||||
    };
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -89,6 +89,7 @@
 | 
			
		|||
        order-by="Name"
 | 
			
		||||
        refresh-callback="ctrl.getNodes"
 | 
			
		||||
        is-admin="ctrl.isAdmin"
 | 
			
		||||
        use-server-metrics="ctrl.state.useServerMetrics"
 | 
			
		||||
      ></kubernetes-nodes-datatable>
 | 
			
		||||
    </div>
 | 
			
		||||
  </div>
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -15,7 +15,8 @@ class KubernetesClusterController {
 | 
			
		|||
    KubernetesNodeService,
 | 
			
		||||
    KubernetesApplicationService,
 | 
			
		||||
    KubernetesComponentStatusService,
 | 
			
		||||
    KubernetesEndpointService
 | 
			
		||||
    KubernetesEndpointService,
 | 
			
		||||
    EndpointProvider
 | 
			
		||||
  ) {
 | 
			
		||||
    this.$async = $async;
 | 
			
		||||
    this.$state = $state;
 | 
			
		||||
| 
						 | 
				
			
			@ -26,6 +27,7 @@ class KubernetesClusterController {
 | 
			
		|||
    this.KubernetesApplicationService = KubernetesApplicationService;
 | 
			
		||||
    this.KubernetesComponentStatusService = KubernetesComponentStatusService;
 | 
			
		||||
    this.KubernetesEndpointService = KubernetesEndpointService;
 | 
			
		||||
    this.EndpointProvider = EndpointProvider;
 | 
			
		||||
 | 
			
		||||
    this.onInit = this.onInit.bind(this);
 | 
			
		||||
    this.getNodes = this.getNodes.bind(this);
 | 
			
		||||
| 
						 | 
				
			
			@ -132,6 +134,7 @@ class KubernetesClusterController {
 | 
			
		|||
    }
 | 
			
		||||
 | 
			
		||||
    this.state.viewReady = true;
 | 
			
		||||
    this.state.useServerMetrics = this.EndpointProvider.currentEndpoint().Kubernetes.Configuration.UseServerMetrics;
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  $onInit() {
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -0,0 +1,71 @@
 | 
			
		|||
<kubernetes-view-header title="Node stats" state="kubernetes.cluster.node.stats" view-ready="ctrl.state.viewReady">
 | 
			
		||||
  <a ui-sref="kubernetes.cluster">Cluster</a> > <a ui-sref="kubernetes.cluster.node({name: ctrl.state.transition.nodeName})"> {{ ctrl.state.transition.nodeName }} </a> >
 | 
			
		||||
  {{ ctrl.state.transition.nodeName }}
 | 
			
		||||
</kubernetes-view-header>
 | 
			
		||||
 | 
			
		||||
<kubernetes-view-loading view-ready="ctrl.state.viewReady"></kubernetes-view-loading>
 | 
			
		||||
 | 
			
		||||
<div ng-if="ctrl.state.viewReady">
 | 
			
		||||
  <information-panel ng-if="!ctrl.state.getMetrics" title-text="Unable to retrieve node metrics">
 | 
			
		||||
    <span class="small text-muted">
 | 
			
		||||
      <i class="fa fa-exclamation-circle orange-icon" aria-hidden="true" style="margin-right: 2px;"></i>
 | 
			
		||||
      Portainer was unable to retrieve any metrics associated to that node. Please contact your administrator to ensure that the Kubernetes metrics feature is properly configured.
 | 
			
		||||
    </span>
 | 
			
		||||
  </information-panel>
 | 
			
		||||
  <div class="row" ng-if="ctrl.state.getMetrics">
 | 
			
		||||
    <div class="col-md-12">
 | 
			
		||||
      <rd-widget>
 | 
			
		||||
        <rd-widget-header icon="fa-info-circle" title-text="About statistics"> </rd-widget-header>
 | 
			
		||||
        <rd-widget-body>
 | 
			
		||||
          <form class="form-horizontal">
 | 
			
		||||
            <div class="form-group">
 | 
			
		||||
              <div class="col-sm-12">
 | 
			
		||||
                <span class="small text-muted">
 | 
			
		||||
                  This view displays real-time statistics about the node <b>{{ ctrl.state.transition.nodeName }}</b
 | 
			
		||||
                  >.
 | 
			
		||||
                </span>
 | 
			
		||||
              </div>
 | 
			
		||||
            </div>
 | 
			
		||||
            <div class="form-group">
 | 
			
		||||
              <label for="refreshRate" class="col-sm-3 col-md-2 col-lg-2 margin-sm-top control-label text-left">
 | 
			
		||||
                Refresh rate
 | 
			
		||||
              </label>
 | 
			
		||||
              <div class="col-sm-3 col-md-2">
 | 
			
		||||
                <select id="refreshRate" ng-model="ctrl.state.refreshRate" ng-change="ctrl.changeUpdateRepeater()" class="form-control">
 | 
			
		||||
                  <option value="30">30s</option>
 | 
			
		||||
                  <option value="60">60s</option>
 | 
			
		||||
                </select>
 | 
			
		||||
              </div>
 | 
			
		||||
              <span>
 | 
			
		||||
                <i id="refreshRateChange" class="fa fa-check green-icon" aria-hidden="true" style="margin-top: 7px; display: none;"></i>
 | 
			
		||||
              </span>
 | 
			
		||||
            </div>
 | 
			
		||||
          </form>
 | 
			
		||||
        </rd-widget-body>
 | 
			
		||||
      </rd-widget>
 | 
			
		||||
    </div>
 | 
			
		||||
  </div>
 | 
			
		||||
 | 
			
		||||
  <div class="row" ng-show="ctrl.state.getMetrics">
 | 
			
		||||
    <div class="col-lg-6 col-md-12 col-sm-12">
 | 
			
		||||
      <rd-widget>
 | 
			
		||||
        <rd-widget-header icon="fa-chart-area" title-text="Memory usage"></rd-widget-header>
 | 
			
		||||
        <rd-widget-body>
 | 
			
		||||
          <div class="chart-node" style="position: relative;">
 | 
			
		||||
            <canvas id="memoryChart" width="770" height="300"></canvas>
 | 
			
		||||
          </div>
 | 
			
		||||
        </rd-widget-body>
 | 
			
		||||
      </rd-widget>
 | 
			
		||||
    </div>
 | 
			
		||||
    <div class="col-lg-6 col-md-12 col-sm-12">
 | 
			
		||||
      <rd-widget>
 | 
			
		||||
        <rd-widget-header icon="fa-chart-area" title-text="CPU usage"></rd-widget-header>
 | 
			
		||||
        <rd-widget-body>
 | 
			
		||||
          <div class="chart-node" style="position: relative;">
 | 
			
		||||
            <canvas id="cpuChart" width="770" height="300"></canvas>
 | 
			
		||||
          </div>
 | 
			
		||||
        </rd-widget-body>
 | 
			
		||||
      </rd-widget>
 | 
			
		||||
    </div>
 | 
			
		||||
  </div>
 | 
			
		||||
</div>
 | 
			
		||||
| 
						 | 
				
			
			@ -0,0 +1,8 @@
 | 
			
		|||
angular.module('portainer.kubernetes').component('kubernetesNodeStatsView', {
 | 
			
		||||
  templateUrl: './stats.html',
 | 
			
		||||
  controller: 'KubernetesNodeStatsController',
 | 
			
		||||
  controllerAs: 'ctrl',
 | 
			
		||||
  bindings: {
 | 
			
		||||
    $transition$: '<',
 | 
			
		||||
  },
 | 
			
		||||
});
 | 
			
		||||
| 
						 | 
				
			
			@ -0,0 +1,144 @@
 | 
			
		|||
import angular from 'angular';
 | 
			
		||||
import moment from 'moment';
 | 
			
		||||
import filesizeParser from 'filesize-parser';
 | 
			
		||||
import KubernetesResourceReservationHelper from 'Kubernetes/helpers/resourceReservationHelper';
 | 
			
		||||
import { PORTAINER_FADEOUT } from '@/constants';
 | 
			
		||||
 | 
			
		||||
class KubernetesNodeStatsController {
 | 
			
		||||
  /* @ngInject */
 | 
			
		||||
  constructor($async, $state, $interval, $document, Notifications, KubernetesNodeService, KubernetesMetricsService, ChartService) {
 | 
			
		||||
    this.$async = $async;
 | 
			
		||||
    this.$state = $state;
 | 
			
		||||
    this.$interval = $interval;
 | 
			
		||||
    this.$document = $document;
 | 
			
		||||
    this.Notifications = Notifications;
 | 
			
		||||
    this.KubernetesNodeService = KubernetesNodeService;
 | 
			
		||||
    this.KubernetesMetricsService = KubernetesMetricsService;
 | 
			
		||||
    this.ChartService = ChartService;
 | 
			
		||||
 | 
			
		||||
    this.onInit = this.onInit.bind(this);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  changeUpdateRepeater() {
 | 
			
		||||
    var cpuChart = this.cpuChart;
 | 
			
		||||
    var memoryChart = this.memoryChart;
 | 
			
		||||
 | 
			
		||||
    this.stopRepeater();
 | 
			
		||||
    this.setUpdateRepeater(cpuChart, memoryChart);
 | 
			
		||||
    $('#refreshRateChange').show();
 | 
			
		||||
    $('#refreshRateChange').fadeOut(PORTAINER_FADEOUT);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  updateCPUChart() {
 | 
			
		||||
    const label = moment(this.stats.read).format('HH:mm:ss');
 | 
			
		||||
    this.ChartService.UpdateCPUChart(label, this.stats.CPUUsage, this.cpuChart);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  updateMemoryChart() {
 | 
			
		||||
    const label = moment(this.stats.read).format('HH:mm:ss');
 | 
			
		||||
    this.ChartService.UpdateMemoryChart(label, this.stats.MemoryUsage, 0, this.memoryChart);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  stopRepeater() {
 | 
			
		||||
    var repeater = this.repeater;
 | 
			
		||||
    if (angular.isDefined(repeater)) {
 | 
			
		||||
      this.$interval.cancel(repeater);
 | 
			
		||||
      this.repeater = undefined;
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  setUpdateRepeater() {
 | 
			
		||||
    const refreshRate = this.state.refreshRate;
 | 
			
		||||
 | 
			
		||||
    this.repeater = this.$interval(async () => {
 | 
			
		||||
      try {
 | 
			
		||||
        await this.getStats();
 | 
			
		||||
        this.updateCPUChart();
 | 
			
		||||
        this.updateMemoryChart();
 | 
			
		||||
      } catch (error) {
 | 
			
		||||
        this.stopRepeater();
 | 
			
		||||
        this.Notifications.error('Failure', error);
 | 
			
		||||
      }
 | 
			
		||||
    }, refreshRate * 1000);
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  initCharts() {
 | 
			
		||||
    const cpuChartCtx = $('#cpuChart');
 | 
			
		||||
    const cpuChart = this.ChartService.CreateCPUChart(cpuChartCtx);
 | 
			
		||||
    this.cpuChart = cpuChart;
 | 
			
		||||
 | 
			
		||||
    const memoryChartCtx = $('#memoryChart');
 | 
			
		||||
    const memoryChart = this.ChartService.CreateMemoryChart(memoryChartCtx);
 | 
			
		||||
    this.memoryChart = memoryChart;
 | 
			
		||||
 | 
			
		||||
    this.updateCPUChart();
 | 
			
		||||
    this.updateMemoryChart();
 | 
			
		||||
    this.setUpdateRepeater();
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  getStats() {
 | 
			
		||||
    return this.$async(async () => {
 | 
			
		||||
      try {
 | 
			
		||||
        const stats = await this.KubernetesMetricsService.getNode(this.state.transition.nodeName);
 | 
			
		||||
        if (stats) {
 | 
			
		||||
          const memory = filesizeParser(stats.usage.memory);
 | 
			
		||||
          const cpu = KubernetesResourceReservationHelper.parseCPU(stats.usage.cpu);
 | 
			
		||||
          this.stats = {
 | 
			
		||||
            read: stats.creationTimestamp,
 | 
			
		||||
            MemoryUsage: memory,
 | 
			
		||||
            CPUUsage: (cpu / this.nodeCPU) * 100,
 | 
			
		||||
          };
 | 
			
		||||
        }
 | 
			
		||||
      } catch (err) {
 | 
			
		||||
        this.Notifications.error('Failure', err, 'Unable to retrieve node stats');
 | 
			
		||||
      }
 | 
			
		||||
    });
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  $onDestroy() {
 | 
			
		||||
    this.stopRepeater();
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  async onInit() {
 | 
			
		||||
    this.state = {
 | 
			
		||||
      autoRefresh: false,
 | 
			
		||||
      refreshRate: '30',
 | 
			
		||||
      viewReady: false,
 | 
			
		||||
      transition: {
 | 
			
		||||
        nodeName: this.$transition$.params().name,
 | 
			
		||||
      },
 | 
			
		||||
      getMetrics: true,
 | 
			
		||||
    };
 | 
			
		||||
 | 
			
		||||
    try {
 | 
			
		||||
      const nodeMetrics = await this.KubernetesMetricsService.getNode(this.state.transition.nodeName);
 | 
			
		||||
 | 
			
		||||
      if (nodeMetrics) {
 | 
			
		||||
        const node = await this.KubernetesNodeService.get(this.state.transition.nodeName);
 | 
			
		||||
        this.nodeCPU = node.CPU || 1;
 | 
			
		||||
 | 
			
		||||
        await this.getStats();
 | 
			
		||||
 | 
			
		||||
        if (this.state.getMetrics) {
 | 
			
		||||
          this.$document.ready(() => {
 | 
			
		||||
            this.initCharts();
 | 
			
		||||
          });
 | 
			
		||||
        }
 | 
			
		||||
      } else {
 | 
			
		||||
        this.state.getMetrics = false;
 | 
			
		||||
      }
 | 
			
		||||
    } catch (err) {
 | 
			
		||||
      this.state.getMetrics = false;
 | 
			
		||||
      this.Notifications.error('Failure', err, 'Unable to retrieve node stats');
 | 
			
		||||
    } finally {
 | 
			
		||||
      this.state.viewReady = true;
 | 
			
		||||
    }
 | 
			
		||||
  }
 | 
			
		||||
 | 
			
		||||
  $onInit() {
 | 
			
		||||
    return this.$async(this.onInit);
 | 
			
		||||
  }
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
export default KubernetesNodeStatsController;
 | 
			
		||||
angular.module('portainer.kubernetes').controller('KubernetesNodeStatsController', KubernetesNodeStatsController);
 | 
			
		||||
		Loading…
	
		Reference in New Issue