.::Undergrounds v3.3 Webshells::.

: / opt / cloudlinux / venv / lib / python3.11 / site-packages / cl_plus / collectors /
Filename : node_exporter.py
back
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2020 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENCE.TXT
#

import os
import platform
import re
from requests import PreparedRequest, RequestException
from prometheus_client.parser import text_string_to_metric_families
from typing import Dict, AnyStr, List
from itertools import groupby

from .collector_base import CollectorBase
from cl_plus.utils.unix_socket import UnixHTTPAdapter
from clcommon.utils import get_virt_type
from cl_plus.consts import METRIC_NAME, METRIC_LABELS, METRIC_VALUE

# Labels added in newer node_exporter versions that are not in the pushgateway schema
# These must be stripped to pass JSON validation
LABELS_TO_STRIP = {'device_error'}


class NodeExporterCollector(CollectorBase):
    def __init__(self, _logger):
        super(NodeExporterCollector, self).__init__(_logger)
        self._required_metrics = ['node_filesystem_avail_bytes',
                                  'node_filesystem_size_bytes',
                                  'node_cpu_seconds_total',
                                  'node_filesystem_free_bytes',
                                  'node_memory_MemTotal_bytes',
                                  'node_memory_MemAvailable_bytes',
                                  'node_boot_time_seconds',
                                  'node_context_switches_total',
                                  'node_disk_io_time_seconds_total',
                                  'node_disk_read_bytes_total',
                                  'node_disk_read_time_seconds_total',
                                  'node_disk_reads_completed_total',
                                  'node_disk_write_time_seconds_total',
                                  'node_disk_writes_completed_total',
                                  'node_disk_written_bytes_total',
                                  'node_network_receive_bytes_total',
                                  'node_network_transmit_bytes_total',
                                  'node_filefd_allocated',
                                  'node_load1',
                                  'node_load15',
                                  'node_load5',
                                  'node_hwmon_temp_celsius']
        self._is_ne_error = None
        self._socket_file = '/var/run/cl_node_exporter.socket'
        self._metrics_url = 'http://localhost/metrics'
        self.virt_type = get_virt_type()

    def init(self):
        """
        Initialize Node Exporter collector
        :return: None
        """
        self._aggregated_data = {}
        self._logger.info("Node Exporter collector init")

    def _get_data_from_socket(self):

        request = PreparedRequest()
        request.prepare(
            method='GET',
            url=self._metrics_url,
        )
        unix_http_adapter = UnixHTTPAdapter(self._socket_file)
        response = unix_http_adapter.send(request)
        try:
            response.raise_for_status()
        finally:
            unix_http_adapter.close()

        return response.text

    @staticmethod
    def _is_metric_value_exclude(labels):
        """
        Determines is metric value with supposed labels need exclude
        :param labels: labels for mount dictionary to check. Example:
            {'device': '/dev/sda1', 'fstype': 'xfs', 'mountpoint': '/usr/share/cagefs-skeleton'}
        :return: True/False - Exclude/not exclude metric value from collector output
        """
        if 'mountpoint' not in labels:
            return False
        # Check for special case if CageFs skeleton path (or it part) is symlink to some another dir
        path_to_check = os.path.realpath('/usr/share/cagefs-skeleton')
        if labels['mountpoint'].startswith(path_to_check) or '/virtfs/' in labels['mountpoint']:
            return True
        return False

    @staticmethod
    def _calculate_available_mem(metrics):
        """
        Calculates available mem metric
        according to article:
        https://access.redhat.com/solutions/406773
        """
        return metrics['node_memory_MemFree_bytes'] + \
            metrics['node_memory_SReclaimable_bytes'] + \
            metrics['node_memory_Buffers_bytes'] + \
            metrics['node_memory_Cached_bytes']

    def _prepare_node_disk_entries(self, raw_ne_output: AnyStr) -> AnyStr:
        """
        Different virtualization types display all devices
        in the /proc/diskstats with different titles
        Node exporter provides default filtering, but it doesn't work
        with non-kvm virtual machines (at least, XEN)

        Example on kvm /proc/diskstats output:
            vda ...
            vda1 ...
            sr0 ...
            loop0 ...

        And default NE behaviour will leave only vda and sr0 devices

        Example on XEN:
            xvda1 ...

        And default NE behaviour will ignore this entry

        So this method checks virt type and leaves only suitable
        devices (depending on vm type)

        :param raw_ne_output: node exporter output including all devices
        for node_disk metrics
        :return: output with skipped lines
        """
        # Set default regexp value (just like NE)
        # All lines with node_disk metrics and devices to filter
        # Example of raw output with kvm virtualization:
        # # HELP node_disk_io_now The number of I/Os currently in progress.
        # # TYPE node_disk_io_now gauge
        # node_disk_io_now{device="loop0"} 0
        # node_disk_io_now{device="sr0"} 0
        # node_disk_io_now{device="vda"} 0
        # node_disk_io_now{device="vda1"} 0
        # After the regexp substring:
        # # HELP node_disk_io_now The number of I/Os currently in progress.
        # # TYPE node_disk_io_now gauge
        # node_disk_io_now{device="sr0"} 0
        # node_disk_io_now{device="vda"} 0
        reg = "node_disk.*(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\\d+n\\d+p)\\d.*\n"
        if self.virt_type == "xen":
            # Regex, that will include `xvd*{num}` and `vd*{num}` devices
            reg = "node_disk.*(ram|loop|fd|(h|s)d[a-z]|nvme\\d+n\\d+p)\\d.*\n"
        return re.sub(reg, "", raw_ne_output)

    @staticmethod
    def filter_samples_for_xen(family) -> List:
        """
        Checks for presence of devices without numbers in a title and
        ignores all devices with numbers (if any)
        If there are only numbered devices - doesn't filter anything

        Example of received node_disk metric family:
        Name: node_disk_* Labels: {'device': 'xvda'} Value: 811395.84
        Name: node_disk_* Labels: {'device': 'xvda1'} Value: 1114.75
        Name: node_disk_* Labels: {'device': 'xvda2'} Value: 0.0
        Name: node_disk_* Labels: {'device': 'xvdb1'} Value: 813760.16
        Name: node_disk_* Labels: {'device': 'xvdb2'} Value: 872.06
        Name: node_disk_* Labels: {'device': 'xvdb3'} Value: 0.0
        Expected result of filtering:
        `xvda` group (since we have sample with device without num):
        Name: node_disk_* Labels: {'device': 'xvda'} Value: 811395.84
        `xvdb` group (since all samples devices are with num):
        Name: node_disk_* Labels: {'device': 'xvdb1'} Value: 813760.16
        Name: node_disk_* Labels: {'device': 'xvdb2'} Value: 872.06
        Name: node_disk_* Labels: {'device': 'xvdb3'} Value: 0.0

        :param family: parsed Metric object with all samples
        :return: list of needed samples
        """
        filtered_family_samples = []
        # Grouping samples of metric by device base
        # (device value without numbers)
        for device_base, group in groupby(
                family.samples, lambda x: re.sub("\\d+", "", x[1]["device"])):
            group = list(group)
            for sample in group:
                # If devices have xvd* title without numbers (`xvda`)
                if sample[METRIC_LABELS]["device"] == device_base:
                    # Leave only this sample from device group
                    filtered_family_samples.append(sample)
                    break
            else:
                # There is no device without a number in title
                # Add info about all devices in the group
                filtered_family_samples.extend(group)
        return filtered_family_samples

    def _get_data_from_ne(self):
        """
        Retrieve new data from Node Exporter
        :return: Dict with NE metrics
        """
        # Get data from NE
        try:
            node_exporter_output = self._get_data_from_socket()
        except RequestException as err:
            if not self._is_ne_error:
                self._logger.warn("[Node exporter collector] socket get_data error: %s", err)
            self._is_ne_error = True
            return None
        self._is_ne_error = False
        try:
            node_exporter_metrics = {}

            # metrics to calculate availableMEM for cl6
            memory_metrics = ['node_memory_MemFree_bytes', 'node_memory_SReclaimable_bytes',
                              'node_memory_Cached_bytes', 'node_memory_Buffers_bytes']
            memory_metrics_values = {}
            # Filter diskstats entries
            node_exporter_output = self._prepare_node_disk_entries(
                node_exporter_output)
            for family in text_string_to_metric_families(node_exporter_output):
                # Exclude all unneeded samples for node_disk and XEN VMs
                if family.name.startswith("node_disk") and self.virt_type == "xen":
                    family.samples = self.filter_samples_for_xen(family)
                for sample in family.samples:
                    # sample example
                    # Name: node_cpu_seconds_total Labels: {'cpu': '0', 'mode': 'idle'} Value: 811395.84
                    # Name: node_cpu_seconds_total Labels: {'cpu': '0', 'mode': 'iowait'} Value: 1114.75
                    # Name: node_cpu_seconds_total Labels: {'cpu': '0', 'mode': 'irq'} Value: 0.0
                    # Name: node_cpu_seconds_total Labels: {'cpu': '1', 'mode': 'idle'} Value: 813760.16
                    # Name: node_cpu_seconds_total Labels: {'cpu': '1', 'mode': 'iowait'} Value: 872.06
                    # Name: node_cpu_seconds_total Labels: {'cpu': '1', 'mode': 'irq'} Value: 0.0
                    name = sample[METRIC_NAME]            # metric name
                    labels = sample[METRIC_LABELS]        # labels dict
                    value = sample[METRIC_VALUE]          # metric value

                    # Strip labels not in pushgateway schema (e.g. device_error from v1.8.x)
                    if LABELS_TO_STRIP & labels.keys():
                        labels = {k: v for k, v in labels.items() if k not in LABELS_TO_STRIP}

                    if name in memory_metrics:
                        memory_metrics_values[name] = value

                    if name not in self._required_metrics:
                        continue
                    # Skip unneeded mounts
                    if self._is_metric_value_exclude(labels):
                        continue
                    if name not in node_exporter_metrics:
                        node_exporter_metrics[name] = []
                    metric_data = {"value": value}
                    if len(labels) != 0:
                        metric_data.update({"labels": labels})
                    node_exporter_metrics[name].append(metric_data)
            self._is_ne_error = False

            if 'el6' in platform.release() and \
                    node_exporter_metrics.get('node_memory_MemAvailable_bytes') is None:
                available_mem = self._calculate_available_mem(memory_metrics_values)
                node_exporter_metrics['node_memory_MemAvailable_bytes'] = [{'value': available_mem}]

            return node_exporter_metrics
        except Exception:
            if not self._is_ne_error:
                import traceback
                var = traceback.format_exc()
                self._logger.warn("[Node exporter collector] generic get_data error: %s" % var)
                self._is_ne_error = True
        return None

    def _collect_new_data(self, new_ne_data_dict: Dict):
        """
        Add new NE data
        :param new_ne_data_dict: New data from NE

        income node exporter dict:
        {
            'metric_name': [
                {'value': 1}
            ],
            'metric_name2': [
                {'value': 1, 'labels': {...}},
                {'value': 1, 'labels': {...}}
            ]
        }

        aggregated data:
        {
            'metric_name': [
                {'value': [1, 2, 3]}
            ],
            'metric_name2': [
                {'value': [1, 2, 3], 'labels': {...}},
                {'value': [1, 3, 5], 'labels': {...}}
            ]
        }
        """
        for metric_name, metric_values_list in new_ne_data_dict.items():
            for metric_values_dict in metric_values_list:
                if metric_name not in self._aggregated_data:
                    self._aggregated_data[metric_name] = []

                if 'labels' not in metric_values_dict:
                    if self._aggregated_data[metric_name]:
                        self._aggregated_data[metric_name][0]['value'].append(metric_values_dict['value'])
                    else:
                        self._aggregated_data[metric_name].append({
                            'value': [metric_values_dict['value']]
                        })
                else:
                    for aggregated_item_dict in self._aggregated_data[metric_name]:
                        if metric_values_dict['labels'] == aggregated_item_dict['labels']:
                            aggregated_item_dict['value'].append(metric_values_dict['value'])
                            break
                    else:
                        self._aggregated_data[metric_name].append({
                            'value': [metric_values_dict['value']],
                            'labels': metric_values_dict['labels']
                        })

    def aggregate_new_data(self):
        """
        Retrieve and aggregate new data
        :return None
        """
        json_dict = self._get_data_from_ne()
        # Exit if no data
        if not json_dict:
            return
        # New data present - aggregate
        self._collect_new_data(json_dict)

    def get_averages(self):
        """
        Get collector's averages data
        :return: dict
            {
             "node_exporter":
              {
                "go_gc_duration_seconds": [ { "labels":  {"quantile": "0"} , "value": "0" },
                                            { "labels":  {"quantile": "0.25"} , "value": "0" },
                                            { "labels":  {"quantile": "0.5"} , "value": "0" },
                                            { "labels":  {"quantile": "0.75"} , "value": "0" },
                                            { "labels":  {"quantile": "1"} , "value": "0" }
                                          ],
                "go_gc_duration_seconds_sum": [ { "value": "0" } ]
              }
            }
            or None if can't get data
        """
        if not self._aggregated_data:
            return None

        # calculate average for each metric
        for _, metric_values_list in self._aggregated_data.items():
            for metric_data in metric_values_list:
                # {'metric_name': [{'value': [1,2,3,...], 'labels': {}}, ...], ...} ->
                # {'metric_name': [{'value': average([1,2,3,...]), 'labels': {}}, ...], ...}
                metric_data['value'] = sum(metric_data['value']) // len(metric_data['value'])
        ret_dict = self._aggregated_data.copy()
        # Prepare to next collecting cycle - reset all needed variables
        self._aggregated_data = {}
        return {"node_exporter": ret_dict}