弹性容器实例：基于 Argo Workflows 和 Serverless Kubernetes 搭建精细化用云工作流

kubectl  apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.5.5/install.yamlcustomresourcedefinition.apiextensions.k8s.io/clusterworkflowtemplates.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/cronworkflows.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/workflowartifactgctasks.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/workfloweventbindings.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/workflows.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/workflowtaskresults.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/workflowtasksets.argoproj.io createdcustomresourcedefinition.apiextensions.k8s.io/workflowtemplates.argoproj.io createdserviceaccount/argo createdserviceaccount/argo-server createdrole.rbac.authorization.k8s.io/argo-role createdclusterrole.rbac.authorization.k8s.io/argo-aggregate-to-admin createdclusterrole.rbac.authorization.k8s.io/argo-aggregate-to-edit createdclusterrole.rbac.authorization.k8s.io/argo-aggregate-to-view createdclusterrole.rbac.authorization.k8s.io/argo-cluster-role createdclusterrole.rbac.authorization.k8s.io/argo-server-cluster-role createdrolebinding.rbac.authorization.k8s.io/argo-binding createdclusterrolebinding.rbac.authorization.k8s.io/argo-binding createdclusterrolebinding.rbac.authorization.k8s.io/argo-server-binding createdconfigmap/workflow-controller-configmap createdservice/argo-server createdpriorityclass.scheduling.k8s.io/workflow-controller createddeployment.apps/argo-server createddeployment.apps/workflow-controller created

apiVersion: v1data:  executor: |    imagePullPolicy: IfNotPresent    image: paas-cn-shanghai.cr.volces.com/argoproj/argoexec:v3.5.5    resources:      requests:        cpu: 0.1        memory: 64Mi      limits:        cpu: 0.5        memory: 512Mikind: ConfigMapmetadata:  name: workflow-controller-configmap  namespace: argo

apiVersion: argoproj.io/v1alpha1kind: Workflowmetadata:  generateName: argo-vci-demo-spec:  entrypoint: hello-hello-hello  templates:  - name: hello-hello-hello
    steps:    - - name: hello1        template: whalesay        arguments:          parameters:          - name: message            value: "hello1"    - - name: hello2a        template: whalesay        arguments:          parameters:          - name: message            value: "hello2a"      - name: hello2b        template: whalesay        arguments:          parameters:          - name: message            value: "hello2b"  - name: whalesay    inputs:      parameters:      - name: message    container:      image: paas-cn-shanghai.cr.volces.com/argoproj/whalesay:latest      command: [cowsay]      args: ["{{inputs.parameters.message}}"]

apiVersion: argoproj.io/v1alpha1kind: Workflowmetadata:  generateName: argo-vci-demo-spec:  entrypoint: hello-hello-hello
  # This spec contains two templates: hello-hello-hello and whalesay  templates:  - name: hello-hello-hello    # Instead of just running a container    # This template has a sequence of steps    steps:    - - name: hello1        template: whalesay        arguments:          parameters:          - name: message            value: "hello1"    - - name: hello2a        template: whalesay        arguments:          parameters:          - name: message            value: "hello2a"      - name: hello2b        template: whalesay        arguments:          parameters:          - name: message            value: "hello2b"
  - name: whalesay    inputs:      parameters:      - name: message    container:      image: paas-cn-shanghai.cr.volces.com/argoproj/whalesay:latest      command: [cowsay]      args: ["{{inputs.parameters.message}}"]#  podGC:#    strategy: OnPodCompletion  # 任务Pod执行完后，completed的Pod会被删除  podMetadata:    annotations:      vke.volcengine.com/burst-to-vci: enforce      vke.volcengine.com/preferred-subnet-ids: subnet-5g1mi8e6aby873inqlbgzmar,subnet-22jvxc4z6vthc7r2qr1q8g9x4,subnet-22jvxceucg3cw7r2qr17sj10n      vci.vke.volcengine.com/preferred-instance-family: vci.u1 #指定 VCI 的规格族

argo submit -n argo argo-vci-demo.yaml --serviceaccount argo

podGC:    strategy: OnPodCompletion  # 任务Pod执行完后，completed的Pod会被删除  podMetadata:    annotations:      vke.volcengine.com/burst-to-vci: enforce      vke.volcengine.com/preferred-subnet-ids: subnet-5g1mi8e6aby873inqlbgzmar      vci.vke.volcengine.com/preferred-instance-family: vci.u1 #指定 VCI 的规格族

apiVersion: scheduling.vke.volcengine.com/v1beta1kind: ResourcePolicymetadata:  name: vke-resourcepolicy   # ResourcePolicy 对象名称。  namespace: argo   # ResourcePolicy 所属命名空间。该命名空间必须与被调度的 Pod 命名空间相同。spec:  selector:   # 被 ResourcePolicy 管理的 Pod 的 Label 选择器。    resource: vci  subsets:   # 资源池配置。  - name: ecs-pool   # 资源池名称。    maxReplicas: 5   # 调度到该资源池的 Pod 数量上限。    maxReplicasRatio: "0.1"  # 调度到该资源池的 Pod，占所有 Pod 的比例阈值，当超过该阈值时，不再向该资源池调度 Pod。    whenNotReachMax: ScheduleAnyWay   # 调度策略，有 DoNotSchdedule 和 ScheduleAnyWay 两个取值。    nodeSelectorTerm:    - key: cluster.vke.volcengine.com/machinepool-name   # 资源池标签键，此处的 machinepool-name 为资源池（节点池）ID 标签键。      operator: In      values:   # 资源池标签值，即实际的资源池（节点池）ID。      - pcodl592d75mk89oame6g  - name: vci-pool   # 资源池名称。    maxReplicas: 100    maxReplicasRatio: "0.1"    nodeSelectorTerm:    - key: type      operator: In      values:      - virtual-kubelet  # 匹配到 VCI 资源对应的虚拟节点。    tolerations: # 如果 Pod 被调度到该资源池，需要给 Pod 打上额外的容忍度。    - effect: NoSchedule      key: vci.vke.volcengine.com/node-type      operator: Equal      value: vci

apiVersion: argoproj.io/v1alpha1kind: Workflowmetadata:  generateName: argo-vci-resourcepolicy-demo-spec:  entrypoint: hello-hello-hello  templates:  - name: hello-hello-hello    steps:    - - name: hello1         template: whalesay        arguments:          parameters:          - name: message            value: "hello1"    - - name: hello2a        template: whalesay-vci        arguments:          parameters:          - name: message            value: "hello2a"      - name: hello2b        template: whalesay-vci        arguments:          parameters:          - name: message            value: "hello2b"  - name: whalesay    inputs:      parameters:      - name: message    container:      image: paas-cn-shanghai.cr.volces.com/argoproj/whalesay:latest      command: [cowsay]      args: ["{{inputs.parameters.message}}"]  - name: whalesay-vci    inputs:      parameters:      - name: message    container:      image: paas-cn-shanghai.cr.volces.com/argoproj/whalesay:latest      command: [cowsay]      args: ["{{inputs.parameters.message}}"]    metadata:      labels:        resource: vci

argo submit -n argo argo-vci-resourcepolicy-demo.yaml  --serviceaccount argo

root@ecs-jumpbox:~/argo-workflow# kubectl create ns argo-jobsnamespace/demo-ns createdroot@ecs-jumpbox:~/argo-workflow# kubectl label namespaces argo-jobs vci=truenamespace/demo-ns labeled

root@ecs-jumpbox:~/argo-workflow# cat argo-role-sa.yaml apiVersion: v1kind: ServiceAccountmetadata:  name: argo  namespace: argo-jobs---apiVersion: rbac.authorization.k8s.io/v1kind: Rolemetadata:  name: argo-role  namespace: argo-jobsrules:- apiGroups:  - coordination.k8s.io  resources:  - leases  verbs:  - create  - get  - update- apiGroups:  - ""  resources:  - secrets  verbs:  - get- apiGroups:  - ""  resources:  - pods  verbs:  - patch- apiGroups:  - argoproj.io  resources:  - workflowtaskresults  verbs:  - patch  - create---apiVersion: rbac.authorization.k8s.io/v1kind: RoleBindingmetadata:  name: argo-binding  namespace: argo-jobsroleRef:  apiGroup: rbac.authorization.k8s.io  kind: Role  name: argo-rolesubjects:- kind: ServiceAccount  name: argo  namespace: argo-jobs

root@ecs-jumpbox:~/argo-workflow# cat vci-profile.yaml apiVersion: v1kind: ConfigMapmetadata:  name: vci-profile         # ConfigMap 名称，必须为 vci-profile。  namespace: kube-system    # vci-profile 文件所属命名空间，必须为 kube-system。data:  preferredSubnetIds: subnet-5g1mi8e6aby873inqlbgzmar  tlsEnable: "true"    # VCI Pod 是否开启并采集日志到火山引擎日志服务。  securityContextPrivilegedConfig: ignore    # VCI Pod 的特权模式兼容性配置。  dnsPolicyClusterFirstWithHostNetConfig: clusterFirst    # VCI Pod 内部的 DNS 解析行为配置。  volumesHostPathConfig: ignore    # VCI Pod 挂载节点文件目录的配置。  hostIpConfig: podIp              # VCI Pod 的 status.hostIP 反显字段配置。  enforceSelectorToVci: "true"     # 是否强制调度 selectors 选中的 Pod 到 VCI。  selectors: |    [      {        "name": "selector-argo-jobs-namespace",        "namespaceSelector": {          "matchLabels": {            "vci": "true"          }        },        "effect": {          "annotations": {            "vci.volcengine.com/tls-enable": "true"          },          "labels": {            "created-by-vci": "true"          }        }      }    ]root@ecs-jumpbox:~/argo-workflow# kubectl create -f vci-profile.yaml configmap/vci-profile createdroot@ecs-jumpbox:~/argo-workflow# kubectl get cm vci-profile -n kube-system -o yamlapiVersion: v1data:  dnsPolicyClusterFirstWithHostNetConfig: clusterFirst  enforceSelectorToVci: "true"  hostIpConfig: podIp  preferredSubnetIds: subnet-5g1mi8e6aby873inqlbgzmar  securityContextPrivilegedConfig: ignore  selectors: |    [      {        "name": "selector-argo-jobs-namespace",        "namespaceSelector": {          "matchLabels": {            "vci": "true"          }        },        "effect": {          "annotations": {            "vci.volcengine.com/tls-enable": "true"          },          "labels": {            "created-by-vci": "true"          }        }      }    ]  tlsEnable: "true"  volumesHostPathConfig: ignorekind: ConfigMapmetadata:  creationTimestamp: "2024-03-31T04:25:07Z"  name: vci-profile  namespace: kube-system  resourceVersion: "1143669"  uid: 471c7e00-61a4-4853-89e0-ee2389a7fe4d

apiVersion: argoproj.io/v1alpha1kind: Workflowmetadata:  generateName: argo-vci-vcipolicy-demo-spec:  entrypoint: hello-hello-hello  templates:  - name: hello-hello-hello    steps:    - - name: hello1        template: whalesay        arguments:          parameters:          - name: message            value: "hello1"    - - name: hello2a        template: whalesay        arguments:          parameters:          - name: message            value: "hello2a"      - name: hello2b        template: whalesay        arguments:          parameters:          - name: message            value: "hello2b"  - name: whalesay    inputs:      parameters:      - name: message    container:      image: paas-cn-shanghai.cr.volces.com/argoproj/whalesay:latest      command: [cowsay]      args: ["{{inputs.parameters.message}}"]

argo submit -n argo-jobs argo-vci-vcipolicy-demo.yaml --serviceaccount argo

        name: workflow-controller        ports:        - containerPort: 9090          name: metrics          protocol: TCP        - containerPort: 6060          protocol: TCP

apiVersion: monitoring.coreos.com/v1kind: PodMonitormetadata:  labels:    volcengine.vmp: "true"  name: argo-workflow-controller-discover  namespace: argospec:  namespaceSelector:    matchNames:    - argo  podMetricsEndpoints:  - interval: 15s    path: /metrics    port: metrics    relabelings:    - action: replace      replacement: argo-workflow-demo      targetLabel: app  selector:    matchLabels:      app: workflow-controller

root@ecs-jumpbox:~# kubectl get pv  shared-data-pv -n argo-jobs NAME             CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                       STORAGECLASS   REASON   AGEshared-data-pv   20Gi       RWX            Retain           Bound    argo-jobs/shared-data-pvc                           3d20hroot@ecs-jumpbox:~# kubectl get pvc -n argo-jobs NAME              STATUS   VOLUME           CAPACITY   ACCESS MODES   STORAGECLASS   AGEshared-data-pvc   Bound    shared-data-pv   20Gi       RWX                           3d20hroot@ecs-jumpbox:~# cat blender_cpu_render.vci.rain_restaurant.yaml apiVersion: argoproj.io/v1alpha1kind: Workflowmetadata:  generateName: render-spec:  entrypoint: main  parallelism: 100   activeDeadlineSeconds: 864000   arguments:    parameters:     - name: filename       value: '/data/restaurant_anim_test/rain_restaurant.blend'    - name: sliceSize       value: 10 #每个渲染任务渲染10帧图片    - name: start       value: 1  # 从动画文件的第一帧开始渲染    - name: stop       value: 300 # 渲染到第300帧    - name: outputLocation       value: "/data/output/rain_restaurant_300/"    - name: samples      value: 300

  volumes:  - name: data-storage    persistentVolumeClaim:      claimName: shared-data-pvc 
  templates: # This defines the steps in our workflow.  - name: main    steps:    - - name: slice         template: gen-slices    - - name: render        template: render-blender        arguments:          parameters:          - name: start            value: "{{item.start}}"          - name: stop            value: "{{item.stop}}"        withParam: "{{steps.slice.outputs.result}}"
  - name: gen-slices     script:      image: cr-demo-cn-beijing.cr.volces.com/xmo/python:3.8-slim-buster      command: [python]      source: |        import json        import sys        frames = range({{workflow.parameters.start}}, {{workflow.parameters.stop}}+1)        n = {{workflow.parameters.sliceSize}}        slices = [frames[i * n:(i + 1) * n] for i in range((len(frames) + n - 1) // n )]        intervals = map(lambda x: {'start': min(x), 'stop': max(x)}, slices)        json.dump(list(intervals), sys.stdout)  - name: render-blender    metadata:      annotations:        vke.volcengine.com/burst-to-vci: enforce # 指定渲染任务Pod调度到VCI    inputs:      parameters:      - name: start      - name: stop      artifacts:      - name: blender_samples         path: /blender_samples.py         raw:          data: |
            import bpy            bpy.data.scenes["Scene"].cycles.samples = {{workflow.parameters.samples}}
    retryStrategy:       limit: 1    container:      image: paas-cn-shanghai.cr.volces.com/argoproj/blender:3.3.1-cpu-ubuntu18.04      command: ["blender"]      workingDir: /      args: [             "-b",            "{{workflow.parameters.filename}}",            "--engine", "CYCLES",            "--factory-startup", "-noaudio",            "--use-extension", "1",            "-o", "{{workflow.parameters.outputLocation}}",            "--python", "blender_samples.py",            "-s", "{{inputs.parameters.start}}",            "-e", "{{inputs.parameters.stop}}",            "-a"      ]      resources:         requests:          memory: 8Gi           cpu: 1         limits:          cpu: 2           memory: 16Gi       volumeMounts:      - name: data-storage         mountPath: /data