.::Undergrounds v3.3 Webshells::.

: / opt / cloudlinux / venv / lib64 / python3.11 / site-packages / ssa / modules /
Filename : storage.py
back
# -*- coding: utf-8 -*-

# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
Represents storage where ssa data is collected, stored and extracted
"""

import itertools
import random
from dataclasses import dataclass
from typing import List, Iterator, Tuple, Dict

import sqlalchemy
from sqlalchemy import func, cast, distinct

from ssa.db import session_scope, RequestResult

# One hot URL can accumulate an unbounded number of duration rows within the
# retention window, and the per-URL list is fed whole to np.percentile / mean /
# max downstream (they copy and sort it). Cap the per-URL sample so peak memory
# stays bounded regardless of stored-row count for a single path.
MAX_URL_DURATION_SAMPLES = 1_000_000


def _sample_durations(dur_group) -> List[int]:
    """
    Collect a per-path durations list bounded to ``MAX_URL_DURATION_SAMPLES``.

    Below the cap every duration is kept (order preserved). Above it, reservoir
    sampling (Algorithm R) keeps a uniform random sample so percentile/mean
    estimates stay statistically representative without letting one hot path
    materialise an arbitrarily large list.
    """
    reservoir: List[int] = []
    true_min = true_max = None
    total = 0
    for seen, (_, duration) in enumerate(dur_group):
        total = seen + 1
        if true_min is None or duration < true_min:
            true_min = duration
        if true_max is None or duration > true_max:
            true_max = duration
        if seen < MAX_URL_DURATION_SAMPLES:
            reservoir.append(duration)
        else:
            idx = random.randint(0, seen)
            if idx < MAX_URL_DURATION_SAMPLES:
                reservoir[idx] = duration
    # Reservoir sampling draws a uniform sample and does not preserve the
    # extremes, but the default autotracing gates evaluate np.percentile at
    # 0/100 (= per-URL min/max). Only when we actually subsampled (total above
    # the cap) force the true min/max back into two slots so those decisions
    # stay exact; below the cap every row is already kept in order.
    if total > MAX_URL_DURATION_SAMPLES:
        reservoir[0] = true_min
        reservoir[-1] = true_max
    return reservoir


@dataclass
class DomainData:
    domain_name: str
    domain_total_reqs: List[int]
    is_a_wordpress_domain: bool
    urls_number: int


def iter_domains_data(engine) -> Iterator[DomainData]:
    """
    Iterates data from database domain-by-domain.
    """
    with session_scope(engine) as db:
        results_by_hour = (
            db.query(
                RequestResult.domain,
                func.strftime('%H', RequestResult.created_at),
                func.Count(RequestResult.id),
                func.max(RequestResult.wordpress),
                func.count(distinct(RequestResult.path)),
            )
            .group_by(RequestResult.domain, func.strftime('%H', RequestResult.created_at))
            .order_by(RequestResult.domain, func.strftime('%H', RequestResult.created_at))
        )

        results_by_hour_grouped = itertools.groupby(results_by_hour, key=lambda item: item[0])

        for domain_name, group in results_by_hour_grouped:
            domain_results_by_hour = tuple(group)

            urls_number = 0
            # at some hours there may be no requests
            # so we must normalize data to match 24h data format
            requests_number_by_hour = [0] * 24
            for _, hour, requests_num, is_wordpress, urls in domain_results_by_hour:
                requests_number_by_hour[int(hour)] = requests_num
                urls_number = max(urls_number, urls)

            yield DomainData(
                domain_name=domain_name,
                domain_total_reqs=requests_number_by_hour,
                is_a_wordpress_domain=is_wordpress,
                urls_number=urls_number,
            )


def iter_urls_data(engine, domain_name, all_paths):
    """
    Iterates urls data from database url-by-url.
    """
    with session_scope(engine) as db:
        urls_data = (
            db.query(
                RequestResult.path,
                func.strftime('%H', RequestResult.created_at),
                func.Sum(cast(RequestResult.hitting_limits, sqlalchemy.Integer)).label('url_throttled_reqs'),
                func.Count(RequestResult.id).label('url_total_reqs'),
                func.Sum(cast(RequestResult.is_slow_request, sqlalchemy.Integer)).label('url_slow_reqs'),
            )
            .filter(RequestResult.domain == domain_name)
            .filter(RequestResult.path.in_(all_paths))
            .group_by(RequestResult.path, func.strftime('%H', RequestResult.created_at))
            .order_by(RequestResult.path, func.strftime('%H', RequestResult.created_at))
        )

        previous_path = None
        url_throttled_reqs, url_total_reqs, url_slow_reqs = [0] * 24, [0] * 24, [0] * 24

        for path, hour, url_throttled_req, url_total_req, url_slow_req in urls_data:
            if previous_path and previous_path != path:
                yield (
                    previous_path,
                    dict(
                        path=previous_path,
                        url_throttled_reqs=url_throttled_reqs,
                        url_total_reqs=url_total_reqs,
                        url_slow_reqs=url_slow_reqs,
                    ),
                )
                url_throttled_reqs, url_total_reqs, url_slow_reqs = [0] * 24, [0] * 24, [0] * 24

            url_throttled_reqs[int(hour)] = url_throttled_req
            url_total_reqs[int(hour)] = url_total_req
            url_slow_reqs[int(hour)] = url_slow_req

            previous_path = path

        yield (
            path,
            dict(
                path=path,
                url_throttled_reqs=url_throttled_reqs,
                url_total_reqs=url_total_reqs,
                url_slow_reqs=url_slow_reqs,
            ),
        )


def get_url_durations(engine, domain_name) -> Dict[str, Tuple[int]]:
    """
    Get information about durations of requests url-by-url.
    """
    with session_scope(engine) as db:
        urls_data = (
            db.query(RequestResult.path, RequestResult.duration)
            .filter(RequestResult.domain == domain_name)
            .order_by(RequestResult.path)
        )

        # Use iterator directly to avoid loading all data into RAM
        durations_by_path = itertools.groupby(urls_data, lambda item: item[0])
        for key, group in durations_by_path:
            yield key, [duration for _, duration in group]


def iter_domain_url_data(engine, domain_name):
    """
    Stream per-URL data for a single domain.

    Yields ``(path, durations, url_data)`` per path, where:
    - ``path`` is the URL path (str);
    - ``durations`` is a list of raw request durations for that path;
    - ``url_data`` is a dict with per-hour aggregates
      (``url_throttled_reqs``, ``url_total_reqs``, ``url_slow_reqs``
      as 24-element lists) plus ``path``.

    Merges two ``ORDER BY path`` streams in lock-step instead of
    materialising the per-domain ``{path: [durations...]}`` dict up
    front, which caused OOM on large databases (CLPRO-3077). A second
    benefit: this drops the redundant ``WHERE path IN (...)`` clause —
    both queries already filter by ``domain``, so the path list adds
    no selectivity, only memory pressure on the SQL parser.
    """
    with session_scope(engine) as db:
        agg_query = (
            db.query(
                RequestResult.path,
                func.strftime('%H', RequestResult.created_at),
                func.Sum(cast(RequestResult.hitting_limits, sqlalchemy.Integer)).label('url_throttled_reqs'),
                func.Count(RequestResult.id).label('url_total_reqs'),
                func.Sum(cast(RequestResult.is_slow_request, sqlalchemy.Integer)).label('url_slow_reqs'),
            )
            .filter(RequestResult.domain == domain_name)
            .group_by(RequestResult.path, func.strftime('%H', RequestResult.created_at))
            .order_by(RequestResult.path, func.strftime('%H', RequestResult.created_at))
        )

        dur_query = (
            db.query(RequestResult.path, RequestResult.duration)
            .filter(RequestResult.domain == domain_name)
            .order_by(RequestResult.path)
        )

        agg_by_path = itertools.groupby(agg_query, key=lambda r: r[0])
        dur_by_path = itertools.groupby(dur_query, key=lambda r: r[0])

        agg_item = next(agg_by_path, None)
        dur_item = next(dur_by_path, None)

        while agg_item is not None and dur_item is not None:
            agg_path, agg_group = agg_item
            dur_path, dur_group = dur_item

            if agg_path == dur_path:
                url_throttled_reqs = [0] * 24
                url_total_reqs = [0] * 24
                url_slow_reqs = [0] * 24
                for _, hour, throttled, total, slow in agg_group:
                    url_throttled_reqs[int(hour)] = throttled
                    url_total_reqs[int(hour)] = total
                    url_slow_reqs[int(hour)] = slow

                durations = _sample_durations(dur_group)

                yield (
                    agg_path,
                    durations,
                    dict(
                        path=agg_path,
                        url_throttled_reqs=url_throttled_reqs,
                        url_total_reqs=url_total_reqs,
                        url_slow_reqs=url_slow_reqs,
                    ),
                )

                agg_item = next(agg_by_path, None)
                dur_item = next(dur_by_path, None)
            elif agg_path < dur_path:
                # Defensive: both queries scan the same WHERE filter, so
                # path sets should match. If they ever diverge (concurrent
                # write between the two queries, indexed collation
                # mismatch, …), skip the unmatched side rather than crash.
                agg_item = next(agg_by_path, None)
            else:
                dur_item = next(dur_by_path, None)