一文详解K8s环境下Job类日志采集方案

spec:  template:    spec:      containers:        - name: logtail          env:            - name: docker_config_update_interval  # 减小容器发现延时              value: "3"            - name: max_docker_config_update_times # 提高容器配置加载频率，减小开始采集延时              value: "60"

kubectl patch ds logtail-ds -n kube-system --patch-file ds-patch.yaml

{  "inputs": [    {      "detail": {        "FlushIntervalMs": 1000 # 加快插件轮询频率，减小开始采集延时      },      "type": "service_docker_stdout"    }  ]}

apiVersion: batch/v1kind: Jobmetadata:  name: sidecar-demo  namespace: defaultspec:  ttlSecondsAfterFinished: 90  template:    metadata:      name: sidecar-demo    spec:      restartPolicy: Never      containers:      - name: nginx-log-demo        image: registry.cn-hangzhou.aliyuncs.com/log-service/docker-log-test:latest        command: ["/bin/sh", "-c"]        args:        - /bin/mock_log --log-type=nginx --stdout=false --stderr=true --path=/var/log/nginx/access.log --total-count=10 --logs-per-sec=10;          retcode=$?;          touch /graveyard/tombstone;          exit $retcode        volumeMounts:        - name: nginx-log          mountPath: /var/log/nginx        - mountPath: /graveyard          name: graveyard      ##### logtail sidecar container      - name: logtail        # more info: https://cr.console.aliyun.com/repository/cn-hangzhou/log-service/logtail/detail        # this images is released for every region        image: registry.cn-huhehaote.aliyuncs.com/log-service/logtail:latest        command: ["/bin/sh", "-c"]        args:        - /etc/init.d/ilogtaild start;          sleep 10;          until [[ -f /graveyard/tombstone ]]; do sleep 3; done;          /etc/init.d/ilogtaild stop;        livenessProbe:          exec:            command:            - /etc/init.d/ilogtaild            - status          initialDelaySeconds: 30          periodSeconds: 30        resources:          limits:            memory: 512Mi          requests:            cpu: 10m            memory: 30Mi        env:          ##### base config          # user id          - name: ALIYUN_LOGTAIL_USER_ID            value: "1654218965343050"          # user defined id          - name: ALIYUN_LOGTAIL_USER_DEFINED_ID            value: hhht-ack-containerd-sidecar          # config file path in logtail's container          - name: ALIYUN_LOGTAIL_CONFIG            value: /etc/ilogtail/conf/cn-huhehaote/ilogtail_config.json          ##### env tags config          - name: ALIYUN_LOG_ENV_TAGS            value: _pod_name_|_pod_ip_|_namespace_|_node_name_|_node_ip_          - name: _pod_name_            valueFrom:              fieldRef:                fieldPath: metadata.name          - name: _pod_ip_            valueFrom:              fieldRef:                fieldPath: status.podIP          - name: _namespace_            valueFrom:              fieldRef:                fieldPath: metadata.namespace          - name: _node_name_            valueFrom:              fieldRef:                fieldPath: spec.nodeName          - name: _node_ip_            valueFrom:              fieldRef:                fieldPath: status.hostIP        volumeMounts:        - name: nginx-log          mountPath: /var/log/nginx        - mountPath: /graveyard          name: graveyard      ##### share this volume      volumes:      - name: nginx-log        emptyDir: {}      - name: graveyard        emptyDir:          medium: Memory

        - /bin/mock_log --log-type=nginx --stdout=false --stderr=true --path=/var/log/nginx/access.log --total-count=10 --logs-per-sec=10;          retcode=$?;          touch /graveyard/tombstone;          exit $retcode

        - /etc/init.d/ilogtaild start;          sleep 10;          until [[ -f /graveyard/tombstone ]]; do sleep 3; done;          /etc/init.d/ilogtaild stop;

          - name: ALIYUN_LOG_ENV_TAGS            value: _pod_name_|_pod_ip_|_namespace_|_node_name_|_node_ip_          - name: _pod_name_            valueFrom:              fieldRef:                fieldPath: metadata.name

apiVersion: log.alibabacloud.com/v1alpha1kind: AliyunLogConfigmetadata:  # 设置资源名，在当前Kubernetes集群内唯一。  name: eci-demospec:  # 设置Logstore名称。如果您所指定的Logstore不存在，日志服务会自动创建。  logstore: eci-demo  # 设置Logtail采集配置。  logtailConfig:    # 设置采集的数据源类型。采集标准输出时，需设置为plugin。    inputType: file  # 设置采集的数据源类型。    configName: eci-demo  # 设置Logtail配置名称，与资源名（metadata.name）保持一致。    inputDetail:          # 设置Logtail采集配置的详细信息。      # 指定通过完整正则模式采集容器文本日志。      logType: common_reg_log      # 设置日志文件所在路径。      logPath: /var/log/nginx      # 设置日志文件的名称。支持通配符星号（*）和半角问号（?），例如log_*.log。      filePattern: access.log      # 设置用于匹配日志行首的行首正则表达式。如果为单行模式，设置成'.*'。      logBeginRegex: '.*'      # 设置正则表达式，用于提取日志内容。请根据实际情况设置。      regex: '(\S+)\s(\S+)\s\S+\s\S+\s"(\S+)\s(\S+)\s+([^"]+)"\s+(\S+)\s(\S+)\s(\d+)\s(\d+)\s(\S+)\s"([^"]+)"\s.*'      # 设置提取的字段列表。      key : ["time", "ip", "method", "url", "protocol", "latency", "payload", "status", "response-size", "user-agent"]

spec:  template:    spec:      containers:        - name: logtail          resources:            limits:              cpu: 4000M              memory: 4096Mi            requests:              cpu: 10M              memory: 30Mi          env:            - name: cpu_usage_limit              value: "9"            - name: mem_usage_limit              value: "4096"            - name: max_bytes_per_sec              value: "209715200"            - name: send_request_concurrency              value: "80"            - name: process_thread_count # 会牺牲SLS上下文功能              value: "8"