Kubernetes HPA一定会减少资源使用吗？HPA可观测性实践分享！

毕海成，去哪儿旅行高级开发工程师, 主要负责网络平台，硬件平台，Kubernetes相关开发与运维。

create table tb_hpa_xxxx(  id SERIAL PRIMARY KEY,  appcode varchar(256),  uc int DEFAULT 0,  dc int DEFAULT 0,  maxuc int DEFAULT 0,  mindc int DEFAULT 0,  create_time timestamptz NOT NULL DEFAULT now(),  update_time timestamptz NOT NULL DEFAULT now());COMMENT ON TABLE tb_hpa_xxxx IS 'HPA指标收集';COMMENT ON COLUMN tb_hpa_xxxx.id IS '自增ID';COMMENT ON COLUMN tb_hpa_xxxx.appcode IS 'appcode';COMMENT ON COLUMN tb_hpa_xxxx.uc IS '扩容次数';COMMENT ON COLUMN tb_hpa_xxxx.dc IS '缩容次数';COMMENT ON COLUMN tb_hpa_xxxx.maxuc IS '最大副本数次数';COMMENT ON COLUMN tb_hpa_xxxx.mindc IS '最小副本数次数';COMMENT ON COLUMN tb_hpa_xxxx.create_time IS '创建时间';COMMENT ON COLUMN tb_hpa_xxxx.update_time IS '更新时间';

select G.*,    N.min_replicas,    N.max_replicasfrom (        select A.deployment_base as env,            A.appcode,            A.annotations as hpa,            coalesce(M.uc, 0) as uc,            coalesce(M.dc, 0) as dc,            coalesce(M.maxuc, 0) as maxuc,            coalesce(M.mindc, 0) as mindc        from (                select appcode,                    deployment_base,                    detail->'metadata'->'annotations' as annotations                from tb_k8s_hpaxxx                where dep_status = 0                    and status = 0                group by appcode,                    deployment_base,                    detail->'metadata'->'annotations'            ) A            left join (                select appcode,                    env_name,                    sum(uc) as uc,                    sum(dc) as dc,                    sum(maxuc) as maxuc,                    sum(mindc) as mindc                from tb_hpa_metrics                where create_time >= '2022-06-10'                    and create_time < '2022-06-11'                group by appcode,                    env_name            ) M on M.appcode = A.appcode            and M.env_name = A.deployment_base    ) G    left join tb_k8s_appcode_hpa N on G.appcode = N.appcode    and G.env = N.deployment_base;

select appcode,    avg(p50) as p50,    avg(p90) as p90,    avg(p99) as p99,    avg(mean) as meanfrom tb_cpu_usage_statxwhere sampling_point = 'day'    and stat_start >= '2022-06-08'    and stat_end < '2022-06-09'group by appcode;

select appcode,    percentile_cont(0.5) within group (        order by p50    ) as p50,    percentile_cont(0.9) within group (        order by p90    ) as p90,    percentile_cont(0.99) within group (        order by p99    ) as p99,    avg(mean) as meanfrom tb_cpu_usage_statwhere sampling_point = 'day'    and stat_start >= '2022-06-08'    and stat_end < '2022-06-09'group by appcode;

select appcode,    percentile_cont(0.5) within group (        order by cpu_usage    ) as p50,    percentile_cont(0.9) within group (        order by cpu_usage    ) as p90,    percentile_cont(0.99) within group (        order by cpu_usage    ) as p99,    avg(cpu_usage)from tb_container_cpu_usage_seconds_totalwhere collect_time >= '2022-06-08 08:00:00'    and collect_time <= '2022-06-08 22:59:59'group by appcode;

select appcode,    percentile_cont(0.5) within group (        order by cpu_usage    ) as p50,    percentile_cont(0.9) within group (        order by cpu_usage    ) as p90,    percentile_cont(0.99) within group (        order by cpu_usage    ) as p99,    avg(cpu_usage)from tb_container_cpu_xxxwhere collect_time >= '2022-06-08 23:00:00'    and collect_time < '2022-06-08 23:59:59'    or collect_time >= '2022-06-08 00:00:00'    and collect_time <= '2022-06-08 07:59:59'group by appcode;

select appcode,    round(sum(pod_replicas_avail) / 9.0, 2) as podsfrom tb_k8s_resourcewhere (        (            record_time >= '2022-06-09 23:00:00'            and record_time <= '2022-06-09 23:59:59'        )        or (            record_time >= '2022-06-09 00:00:00'            and record_time <= '2022-06-09 07:59:59'        )    )group by appcode;

select appcode,    round(sum(pod_replicas_avail) / 15.0, 2) as podsfrom tb_k8s_resourcewhere record_time >= '2022-06-09 08:00:00'    and record_time <= '2022-06-09 22:59:59'group by appcode;

create table tb_hpa_report_xxx(  id SERIAL PRIMARY KEY,  appcode varchar(256),  env_name varchar(256),  uc int DEFAULT 0,  dc int DEFAULT 0,  maxuc int DEFAULT 0,  mindc int DEFAULT 0,  cpu int DEFAULT 0,  mem int DEFAULT 0,  cname VARCHAR(512) DEFAULT '',  cval int DEFAULT 0,  min_replicas int DEFAULT 0,  max_replicas int DEFAULT 0,  hcpu_p50 numeric(10,4) DEFAULT 0,  hcpu_p90 numeric(10,4) DEFAULT 0,  hcpu_p99 numeric(10,4) DEFAULT 0,  hcpu_mean numeric(10,4) DEFAULT 0,  lcpu_p50 numeric(10,4)  DEFAULT 0,  lcpu_p90 numeric(10,4)  DEFAULT 0,  lcpu_p99 numeric(10,4)  DEFAULT 0,  lcpu_mean numeric(10,4) DEFAULT 0,  record_time timestamptz,  create_time timestamptz NOT NULL DEFAULT now(),  update_time timestamptz NOT NULL DEFAULT now());COMMENT ON TABLE tb_hpa_report_xxx IS 'HPA数据报表';COMMENT ON COLUMN tb_hpa_report_xxx.id IS '自增ID';COMMENT ON COLUMN tb_hpa_report_xxx.appcode IS 'appcode';COMMENT ON COLUMN tb_hpa_report_xxx.env_name IS 'env_name';COMMENT ON COLUMN tb_hpa_report_xxx.uc IS '扩容次数';COMMENT ON COLUMN tb_hpa_report_xxx.dc IS '缩容次数';COMMENT ON COLUMN tb_hpa_report_xxx.maxuc IS '最大副本数次数';COMMENT ON COLUMN tb_hpa_report_xxx.mindc IS '最小副本数次数';COMMENT ON COLUMN tb_hpa_report_xxx.cpu IS 'cpu阈值';COMMENT ON COLUMN tb_hpa_report_xxx.mem IS '内存阈值';COMMENT ON COLUMN tb_hpa_report_xxx.cname IS '自定义指标名';COMMENT ON COLUMN tb_hpa_report_xxx.cval IS '自定义指标阈值';COMMENT ON COLUMN tb_hpa_report_xxx.min_replicas IS '最小副本数';COMMENT ON COLUMN tb_hpa_report_xxx.max_replicas IS '最大副本数';COMMENT ON COLUMN tb_hpa_report_xxx.hcpu_p50 IS '高峰cpu p50使用率';COMMENT ON COLUMN tb_hpa_report_xxx.hcpu_p90 IS '高峰cpu p90使用率';COMMENT ON COLUMN tb_hpa_report_xxx.hcpu_p99 IS '高峰cpu p99使用率';COMMENT ON COLUMN tb_hpa_report_xxx.hcpu_mean IS '高峰cpu 平均使用率';COMMENT ON COLUMN tb_hpa_report_xxx.lcpu_p50 IS '低峰cpu p50使用率';COMMENT ON COLUMN tb_hpa_report_xxx.lcpu_p90 IS '低峰cpu p90使用率';COMMENT ON COLUMN tb_hpa_report_xxx.lcpu_p99 IS '低峰cpu p99使用率';COMMENT ON COLUMN tb_hpa_report_xxx.lcpu_mean IS '低峰cpu 平均使用率';COMMENT ON COLUMN tb_hpa_report_xxx.record_time IS '数据统计日期';COMMENT ON COLUMN tb_hpa_report_xxx.create_time IS '创建时间';COMMENT ON COLUMN tb_hpa_report_xxx.update_time IS '更新时间';

"""HPA统计相关"""from server.db.base import Basefrom server.conf.conf import CONFimport sentry_sdkimport datetimefrom sqlalchemy import textfrom server.db.model.meta import commit_on_success, db, try_catch_db_exceptionfrom server.db.model.model import HpaReportModelfrom server.db.hpa import HPAfrom server.libs.mail import SendMailfrom server.libs.qtalk import SendQtalkMsgfrom server.libs.decorators import statsd_indexfrom server.libs.error import Errorimport logging

LOG = logging.getLogger('gunicorn.access')
class HpaReport(Base):
    def __init__(self, *args, **kwargs):        super().__init__(*args, **kwargs)
    @try_catch_db_exception    @commit_on_success    def stats_hpa_updown(self, start_time, end_time):        rows = db.session.execute(text(            """            select G.*,                N.min_replicas,                N.max_replicas            from (                    select A.deployment_base as env,                        A.appcode,                        A.annotations as hpa,                        coalesce(M.uc, 0) as uc,                        coalesce(M.dc, 0) as dc,                        coalesce(M.maxuc, 0) as maxuc,                        coalesce(M.mindc, 0) as mindc                    from (                            select appcode,                                deployment_base,                                detail->'metadata'->'annotations' as annotations                            from tb_k8s_hpa_rec                            where dep_status = 0                                and status = 0                            group by appcode,                                deployment_base,                                detail->'metadata'->'annotations'                        ) A                        left join (                            select appcode,                                env_name,                                sum(uc) as uc,                                sum(dc) as dc,                                sum(maxuc) as maxuc,                                sum(mindc) as mindc                            from tb_hpa_metrics                            where create_time >= :start_time                                and create_time <= :end_time                            group by appcode,                                env_name                        ) M on M.appcode = A.appcode                        and M.env_name = A.deployment_base                ) G                left join tb_k8s_appcode_hpa N on G.appcode = N.appcode                and G.env = N.deployment_base;            """        ), {"start_time": start_time, "end_time": end_time})        LOG.info(f'stats_hpa_updown: {rows.rowcount}')        return self.rows_as_dicts(rows.cursor)
    @try_catch_db_exception    @commit_on_success    def stats_high_time_cpu(self, start_time, end_time):        LOG.info(f'stats_high_time_cpu: {start_time}, {end_time}')        rows = db.session.execute(text(            """            select appcode,                percentile_cont(0.5) within group (                    order by cpu_usage                ) as p50,                percentile_cont(0.9) within group (                    order by cpu_usage                ) as p90,                percentile_cont(0.99) within group (                    order by cpu_usage                ) as p99,                avg(cpu_usage)            from tb_container_cpu_usage_seconds_total            where collect_time >= :start_time                and collect_time <= :end_time            group by appcode            """        ), {"start_time": start_time, "end_time": end_time})        LOG.info(f'stats_high_time_cpu: {rows.rowcount}')        return self.rows_as_dicts(rows.cursor)
    @try_catch_db_exception    @commit_on_success    def stats_high_time_pods(self, start_time, end_time):        LOG.info(f'stats_high_time_pods: {start_time}, {end_time}')        rows = db.session.execute(text(            """            select appcode,                round(sum(pod_replicas_avail) / 15.0, 2) as pods            from tb_k8s_resource            where record_time >= :start_time                and record_time <= :end_time            group by appcode            """        ), {"start_time": start_time, "end_time": end_time})        LOG.info(f'stats_high_time_pods: {rows.rowcount}')        return self.rows_as_dicts(rows.cursor)
    @try_catch_db_exception    @commit_on_success    def stats_low_time_pods(self, s1, e1, s2, e2):        LOG.info(f'stats_low_time_pods: {s1}, {e1}, {s2}, {e2}')        """低峰期分两段（2022-06-08 23:00 - 2022-06-08 23:59:59， 2022-06-08 00:00 - 2022-06-08 07:59:59)
        @param s1 start_time1 低峰时段1开始时间        @param e1 end_time1 低峰时段1结束时间        @param s2 start_time2 低峰时段2开始时间        @param e2 end_time2 低峰时段2结束时间        """        rows = db.session.execute(text(            """            select appcode,                round(sum(pod_replicas_avail) / 9.0, 2) as pods            from tb_k8s_resource            where (                    (                        record_time >= :s1                        and record_time <= :e1                    )                    or (                        record_time >= :s2                        and record_time <= :e2                    )                )            group by appcode            """        ), {"s1": s1, "e1": e1, "s2": s2, "e2": e2})        LOG.info(f'stats_low_time_pods: {rows.rowcount}')        return self.rows_as_dicts(rows.cursor)
    @staticmethod    def rows_as_dicts(cursor):        """convert tuple result to dict with cursor"""        col_names = [i[0] for i in cursor.description]        return [dict(zip(col_names, row)) for row in cursor]
    @try_catch_db_exception    @commit_on_success    def stats_low_time_cpu(self, s1, e1, s2, e2):        """低峰期分两段（2022-06-08 23:00 - 2022-06-08 23:59:59， 2022-06-08 00:00 - 2022-06-08 07:59:59)
        @param s1 start_time1 低峰时段1开始时间        @param e1 end_time1 低峰时段1结束时间        @param s2 start_time2 低峰时段2开始时间        @param e2 end_time2 低峰时段2结束时间        """        LOG.info(f'stats_low_time_cpu: {s1}, {e1}, {s2}, {e2}')        rows = db.session.execute(text(            """            select appcode,                percentile_cont(0.5) within group (                    order by cpu_usage                ) as p50,                percentile_cont(0.9) within group (                    order by cpu_usage                ) as p90,                percentile_cont(0.99) within group (                    order by cpu_usage                ) as p99,                avg(cpu_usage)            from tb_container_cpu_usage_seconds_total            where collect_time >= :s1                and collect_time <= :e1                or collect_time >= :s2                and collect_time <= :e2            group by appcode            """        ), {"s1": s1, "e1": e1, "s2": s2, "e2": e2})        LOG.info(f'stats_low_time_cpu: {rows.rowcount}')        return self.rows_as_dicts(rows.cursor)
    @statsd_index('hpa_report.sendmail')    @commit_on_success    def send_report_form(self, day):        try:            start = datetime.datetime.combine(day, datetime.time(0,0,0))            end = datetime.datetime.combine(day, datetime.time(23,59,59))            q = HpaReportModel.query.filter(                HpaReportModel.record_time >= start,                HpaReportModel.record_time <= end            ).order_by(                HpaReportModel.uc.desc(),                HpaReportModel.dc.desc(),                HpaReportModel.maxuc.desc(),                HpaReportModel.mindc.desc()            )            count = q.count()            day_data = q.all()            cell = ""            if count > 0:                for stat in day_data:                    cell += f"""                        <tr>                            <td>{stat.appcode}</td>                            <td>{stat.env_name}</td>                            <td>{stat.min_replicas}</td>                            <td>{stat.max_replicas}</td>                            <td>{stat.cpu}</td>                            <td>{stat.mem}</td>                            <td>{stat.cname}:{stat.cval}</td>                            <td>{stat.uc}</td>                            <td>{stat.dc}</td>                            <td>{stat.maxuc}</td>                            <td>{stat.mindc}</td>                            <td>{round(stat.hpods, 2)}</td>                            <td>{round(stat.hcpu_mean, 2)}%</td>                            <td>{round(stat.lpods, 2)}</td>                            <td>{round(stat.lcpu_mean, 2)}%</td>                        </tr>"""                content = f"""                    <div>                        <h2>{day} 00:00:00至23:59:59</h2>                        <h3>高峰(08:00-23:00), 低锋(23:00-08:00)</h3>                        <table border='1' cellpadding='1' cellspacing='0'>                            <tr>                                <th>Appcode</th>                                <th>环境</th>                                <th>最小副本数</th>                                <th>最大副本数</th>                                <th>CPU扩容阈值</th>                                <th>内存扩容阈值</th>                                <th>自定义扩容阈值</th>                                <th>扩容次数</th>                                <th>缩容次数</th>                                <th>最大副本数次数</th>                                <th>最小副本数次数</th>                                <th>高峰副本数</th>                                <th>高峰CPU平均使用率</th>                                <th>低锋副本数</th>                                <th>低锋CPU平均使用率</th>                            </tr>                            {cell}                        </table>                    </div><br><br>"""                SendMail.send_mail(                    CONF.notice_user.users.split(','),                    "HPA阔缩容次数及CPU使用率相关统计",                    content)                SendQtalkMsg.send_msg(CONF.notice_user.users.split(','), 'HPA阔缩容次数及CPU使用率相关统计错误报表发送完成')        except Exception as ex:            sentry_sdk.capture_exception()            SendQtalkMsg.send_msg(['haicheng.bi'], f'HPA阔缩容次数及CPU使用率相关统计错误: {ex}')
    @try_catch_db_exception    @commit_on_success    @statsd_index('hpa_report.save_stats_result')    def save_stats_result(self, day):        """保存HPA和cpu统计的结果
        :param day date 统计日期        """        if not isinstance(day, datetime.date):            raise Error(f"param day is invalid type, we need datetime.date type.")        LOG.info(f'save_stats_result: {day}')        hpa_start = datetime.datetime.combine(day, datetime.time(0,0,0))        hpa_end = datetime.datetime.combine(day, datetime.time(23,59,59))        hpa_stats_rows = self.stats_hpa_updown(hpa_start, hpa_end)        # 08-23        h_start = datetime.datetime.combine(day, datetime.time(8,0,0))        h_end = datetime.datetime.combine(day, datetime.time(22,59,59))        # 23-00, 00-08        l_s1 = datetime.datetime.combine(day, datetime.time(23,0,0))        l_e1 = datetime.datetime.combine(day, datetime.time(23,59,59))        l_s2 = datetime.datetime.combine(day, datetime.time(0,0,0))        l_e2 = datetime.datetime.combine(day, datetime.time(7,59,59))        hcpu_stats_rows = self.stats_high_time_cpu(h_start, h_end)        lcpu_stats_rows = self.stats_low_time_cpu(l_s1,l_e1, l_s2, l_e2)        hpods_stats_rows = self.stats_high_time_pods(h_start, h_end)        lpods_stats_rows = self.stats_low_time_pods(l_s1,l_e1, l_s2, l_e2)        cpus_rows = {}        pods_rows = {}        report_rows = {}        for row in hpods_stats_rows:            appcode = row.get('appcode', '')            pods_rows[appcode] = {                'appcode': appcode,                'hpods': row.get('pods', 0)            }        for row in lpods_stats_rows:            appcode = row.get('appcode', '')            pods_rows[appcode].update({                'appcode': appcode,                'lpods': row.get('pods', 0)            })        for row in hcpu_stats_rows:            appcode = row.get('appcode', '')            cpus_rows[appcode] = {                'appcode': appcode,                'hcpu_p50': row.get('p50', 0),                'hcpu_p90': row.get('p90', 0),                'hcpu_p99': row.get('p99', 0),                'hcpu_mean': row.get('avg', 0),            }        for row in lcpu_stats_rows:            appcode = row.get('appcode', '')            cpus_rows[appcode].update({                'lcpu_p50': row.get('p50', 0),                'lcpu_p90': row.get('p90', 0),                'lcpu_p99': row.get('p99', 0),                'lcpu_mean': row.get('avg', 0),            })        for row in hpa_stats_rows:            appcode = row.get('appcode', '')            env_name = row.get('env', '')            hpa = row.get('hpa')            if not hpa:                continue            report_rows[f'{appcode}-{env_name}'] = {                'appcode': appcode,                'env_name': env_name,                'uc': row.get('uc', 0),                'dc': row.get('dc', 0),                'maxuc': row.get('maxuc', 0),                'mindc': row.get('mindc', 0),                'cpu': int(row.get('hpa', {}).get('cpuTargetUtilization', 0)),                'mem': int(row.get('hpa', {}).get('memoryTargetValue', 0)),                'cname': row.get('hpa', {}).get('customName', ''),                'cval': int(row.get('hpa', {}).get('customTargetValue', 0)),                'min_replicas': row.get('min_replicas', 0),                'max_replicas': row.get('max_replicas', 0),            }            report_rows[f'{appcode}-{env_name}'].update(cpus_rows.get(appcode, {}))            report_rows[f'{appcode}-{env_name}'].update(pods_rows.get(appcode, {}))        HpaReportModel.query.filter(            HpaReportModel.record_time == day        ).delete()        for value in report_rows.values():            model = HpaReportModel(record_time=day, **value)            db.session.add(model)