tencent cloud

文档反馈

Istio-init crash

最后更新时间:2023-12-26 15:27:21
    在 istio 环境下有 pod 处于 Init:CrashLoopBackOff 状态:
    wk-sys-acl-v1-0-5-7cf7f79d6c-d9qcr 0/2 Init:CrashLoopBackOff 283 64d 172.16.9.229 10.1.128.6 <none> <none>
    查得 istio-init 的日志:
    Environment:
    ------------
    ENVOY_PORT=
    INBOUND_CAPTURE_PORT=
    ISTIO_INBOUND_INTERCEPTION_MODE=
    ISTIO_INBOUND_TPROXY_MARK=
    ISTIO_INBOUND_TPROXY_ROUTE_TABLE=
    ISTIO_INBOUND_PORTS=
    ISTIO_LOCAL_EXCLUDE_PORTS=
    ISTIO_SERVICE_CIDR=
    ISTIO_SERVICE_EXCLUDE_CIDR=
    
    Variables:
    ----------
    PROXY_PORT=15001
    PROXY_INBOUND_CAPTURE_PORT=15006
    PROXY_UID=1337
    PROXY_GID=1337
    INBOUND_INTERCEPTION_MODE=REDIRECT
    INBOUND_TPROXY_MARK=1337
    INBOUND_TPROXY_ROUTE_TABLE=133
    INBOUND_PORTS_INCLUDE=*
    INBOUND_PORTS_EXCLUDE=15090,15021,15020
    OUTBOUND_IP_RANGES_INCLUDE=*
    OUTBOUND_IP_RANGES_EXCLUDE=
    OUTBOUND_PORTS_EXCLUDE=
    KUBEVIRT_INTERFACES=
    ENABLE_INBOUND_IPV6=false
    
    Writing following contents to rules file: /tmp/iptables-rules-1618279687646418248.txt617375845
    * nat
    -N ISTIO_REDIRECT
    -N ISTIO_IN_REDIRECT
    -N ISTIO_INBOUND
    -N ISTIO_OUTPUT
    -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-ports 15001
    -A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-ports 15006
    -A PREROUTING -p tcp -j ISTIO_INBOUND
    -A ISTIO_INBOUND -p tcp --dport 22 -j RETURN
    -A ISTIO_INBOUND -p tcp --dport 15090 -j RETURN
    -A ISTIO_INBOUND -p tcp --dport 15021 -j RETURN
    -A ISTIO_INBOUND -p tcp --dport 15020 -j RETURN
    -A ISTIO_INBOUND -p tcp -j ISTIO_IN_REDIRECT
    -A OUTPUT -p tcp -j ISTIO_OUTPUT
    -A ISTIO_OUTPUT -o lo -s 127.0.0.6/32 -j RETURN
    -A ISTIO_OUTPUT -o lo ! -d 127.0.0.1/32 -m owner --uid-owner 1337 -j ISTIO_IN_REDIRECT
    -A ISTIO_OUTPUT -o lo -m owner ! --uid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -m owner --uid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -o lo ! -d 127.0.0.1/32 -m owner --gid-owner 1337 -j ISTIO_IN_REDIRECT
    -A ISTIO_OUTPUT -o lo -m owner ! --gid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -m owner --gid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -d 127.0.0.1/32 -j RETURN
    -A ISTIO_OUTPUT -j ISTIO_REDIRECT
    COMMIT
    
    iptables-restore --noflush /tmp/iptables-rules-1618279687646418248.txt617375845
    iptables-restore: line 2 failed
    iptables-save
    # Generated by iptables-save v1.6.1 on Tue Apr 13 02:08:07 2021
    *nat
    :PREROUTING ACCEPT [5214353:312861180]
    :INPUT ACCEPT [5214353:312861180]
    :OUTPUT ACCEPT [6203044:504329953]
    :POSTROUTING ACCEPT [6203087:504332485]
    :ISTIO_INBOUND - [0:0]
    :ISTIO_IN_REDIRECT - [0:0]
    :ISTIO_OUTPUT - [0:0]
    :ISTIO_REDIRECT - [0:0]
    -A PREROUTING -p tcp -j ISTIO_INBOUND
    -A OUTPUT -p tcp -j ISTIO_OUTPUT
    -A ISTIO_INBOUND -p tcp -m tcp --dport 22 -j RETURN
    -A ISTIO_INBOUND -p tcp -m tcp --dport 15090 -j RETURN
    -A ISTIO_INBOUND -p tcp -m tcp --dport 15021 -j RETURN
    -A ISTIO_INBOUND -p tcp -m tcp --dport 15020 -j RETURN
    -A ISTIO_INBOUND -p tcp -j ISTIO_IN_REDIRECT
    -A ISTIO_IN_REDIRECT -p tcp -j REDIRECT --to-ports 15006
    -A ISTIO_OUTPUT -s 127.0.0.6/32 -o lo -j RETURN
    -A ISTIO_OUTPUT ! -d 127.0.0.1/32 -o lo -m owner --uid-owner 1337 -j ISTIO_IN_REDIRECT
    -A ISTIO_OUTPUT -o lo -m owner ! --uid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -m owner --uid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT ! -d 127.0.0.1/32 -o lo -m owner --gid-owner 1337 -j ISTIO_IN_REDIRECT
    -A ISTIO_OUTPUT -o lo -m owner ! --gid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -m owner --gid-owner 1337 -j RETURN
    -A ISTIO_OUTPUT -d 127.0.0.1/32 -j RETURN
    -A ISTIO_OUTPUT -j ISTIO_REDIRECT
    -A ISTIO_REDIRECT -p tcp -j REDIRECT --to-ports 15001
    COMMIT
    # Completed on Tue Apr 13 02:08:07 2021
    panic: exit status 1
    
    goroutine 1 [running]:
    istio.io/istio/tools/istio-iptables/pkg/dependencies.(*RealDependencies).RunOrFail(0x3bb0090, 0x22cfd22, 0x10, 0xc0006849c0, 0x2, 0x2)
    istio.io/istio/tools/istio-iptables/pkg/dependencies/implementation.go:44 +0x96
    istio.io/istio/tools/istio-iptables/pkg/cmd.(*IptablesConfigurator).executeIptablesRestoreCommand(0xc0009dfd68, 0x22c5a01, 0x0, 0x0)
    istio.io/istio/tools/istio-iptables/pkg/cmd/run.go:493 +0x387
    istio.io/istio/tools/istio-iptables/pkg/cmd.(*IptablesConfigurator).executeCommands(0xc0009dfd68)
    istio.io/istio/tools/istio-iptables/pkg/cmd/run.go:500 +0x45
    istio.io/istio/tools/istio-iptables/pkg/cmd.(*IptablesConfigurator).run(0xc0009dfd68)
    istio.io/istio/tools/istio-iptables/pkg/cmd/run.go:447 +0x2625
    istio.io/istio/tools/istio-iptables/pkg/cmd.glob..func1(0x3b5d680, 0xc0004cce00, 0x0, 0x10)
    istio.io/istio/tools/istio-iptables/pkg/cmd/root.go:64 +0x148
    github.com/spf13/cobra.(*Command).execute(0x3b5d680, 0xc0004ccd00, 0x10, 0x10, 0x3b5d680, 0xc0004ccd00)
    github.com/spf13/cobra@v1.0.0/command.go:846 +0x29d
    github.com/spf13/cobra.(*Command).ExecuteC(0x3b5d920, 0x0, 0x0, 0x0)
    github.com/spf13/cobra@v1.0.0/command.go:950 +0x349
    github.com/spf13/cobra.(*Command).Execute(...)
    github.com/spf13/cobra@v1.0.0/command.go:887
    main.main()
    istio.io/istio/pilot/cmd/pilot-agent/main.go:505 +0x2d

    原因与解决方案

    详情见 issue

    直接原因

    这种情况应该通常是清理了已退出的 istio-init 容器,导致 k8s 检测到 pod 关联的容器不在了,然后会重新拉起被删除的容器,而 istio-init 的执行不可重入,因为之前已创建了 iptables 规则,导致后拉起的 istio-init 执行 iptables 失败而 crash。

    根因与解决方案

    清理的动作通常是执行了 docker container rmdocker container prunedocker system prune。 一般是 crontab 定时脚本里定时清理了容器导致,需要停止清理。
    联系我们

    联系我们,为您的业务提供专属服务。

    技术支持

    如果你想寻求进一步的帮助,通过工单与我们进行联络。我们提供7x24的工单服务。

    7x24 电话支持