+ CONSUL_BIND= + [ -n ] + CONSUL_CLIENT= + [ -n ] + CONSUL_DATA_DIR=/consul/data + CONSUL_CONFIG_DIR=/consul/config + [ -n ] + [ a = - ] + [ agent = agent ] + shift + set -- consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui + [ consul = consul ] + stat -c %u /consul/data + id -u consul + [ 100 != 100 ] + stat -c %u /consul/config + id -u consul + [ 100 != 100 ] + [ ! -z ] + exec consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui bootstrap_expect > 0: expecting 3 servers ==> Starting Consul agent... ==> Consul agent running! Version: 'v1.0.6' Node ID: '54b018c9-9a7d-17e5-0fad-ecc6749d17b1' Node name: 'dev-consul-server-0' Datacenter: 'dc1' (Segment: '') Server: true (Bootstrap: false) Client Addr: [0.0.0.0] (HTTP: 8500, HTTPS: -1, DNS: 8600) Cluster Addr: 10.242.68.40 (LAN: 8301, WAN: 8302) Encrypt: Gossip: false, TLS-Outgoing: false, TLS-Incoming: false ==> Log data will now stream in as it occurs: 2021/06/02 11:23:53 [INFO] raft: Initial configuration (index=0): [] 2021/06/02 11:23:53 [INFO] raft: Node at 10.242.68.40:8300 [Follower] entering Follower state (Leader: "") 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.68.40 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.68.40 2021/06/02 11:23:53 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/06/02 11:23:53 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.68.40:8300) (DC: dc1) 2021/06/02 11:23:53 [INFO] agent: Started DNS server 0.0.0.0:8600 (tcp) 2021/06/02 11:23:53 [INFO] agent: Started DNS server 0.0.0.0:8600 (udp) 2021/06/02 11:23:53 [INFO] agent: Started HTTP server on [::]:8500 (tcp) 2021/06/02 11:23:53 [INFO] agent: started state syncer 2021/06/02 11:23:53 [INFO] agent: Retry join LAN is supported for: aliyun aws azure digitalocean gce os scaleway softlayer 2021/06/02 11:23:53 [INFO] agent: Joining LAN cluster... 2021/06/02 11:23:53 [INFO] agent: (LAN) joining: [dev-consul-server-0.consul-server dev-consul-server-1.consul-server dev-consul-server-2.consul-server] 2021/06/02 11:23:53 [WARN] memberlist: Failed to resolve dev-consul-server-0.consul-server: lookup dev-consul-server-0.consul-server on 10.96.0.10:53: no such host 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.166.6 2021/06/02 11:23:53 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/06/02 11:23:53 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.166.6:8300) (DC: dc1) 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-fpx4p 10.242.120.133 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:23:53 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:23:53 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.68.40:8302 Theirs: 10.242.68.10:8302 2021/06/02 11:23:53 [ERR] serf: Node name conflicts with another node at 10.242.68.10:8302. Names must be unique! (Resolution enabled: false) 2021/06/02 11:23:53 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.166.6 2021/06/02 11:23:53 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:23:53 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/02 11:23:53 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.68.40:8302 Theirs: 10.242.68.10:8302 2021/06/02 11:23:53 [ERR] serf: Node name conflicts with another node at 10.242.68.10:8302. Names must be unique! (Resolution enabled: false) 2021/06/02 11:23:53 [INFO] consul: Existing Raft peers reported by dev-consul-server-1, disabling bootstrap mode 2021/06/02 11:23:53 [INFO] agent: (LAN) joined: 2 Err: 2021/06/02 11:23:53 [INFO] agent: Join LAN completed. Synced with 2 initial agents 2021/06/02 11:24:00 [ERR] agent: failed to sync remote state: No cluster leader 2021/06/02 11:24:02 [WARN] raft: no known peers, aborting election ==> Failed to check for updates: Get https://checkpoint-api.hashicorp.com/v1/check/consul?arch=amd64&os=linux&signature=6676ba74-663e-4c06-92a2-9d64023a724c&version=1.0.6: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) 2021/06/02 11:24:25 [ERR] agent: Coordinate update error: No cluster leader 2021/06/02 11:24:30 [ERR] http: Request PUT /v1/txn, error: No cluster leader from=10.242.166.8:47872 2021/06/02 11:24:33 [DEBUG] raft-net: 10.242.68.40:8300 accepted connection from: 10.242.0.7:43386 2021/06/02 11:24:34 [WARN] raft: Failed to get previous log: 531 log not found (last: 0) 2021/06/02 11:24:34 [INFO] agent: Synced node info 2021/06/02 11:24:34 [DEBUG] raft-net: 10.242.68.40:8300 accepted connection from: 10.242.0.7:43392 2021/06/02 11:24:57 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/06/02 11:26:09 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:57776->10.242.0.7:8301: i/o timeout 2021/06/02 11:26:09 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:26:11 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (2 peer confirmations) 2021/06/02 11:26:11 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 11:26:11 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:26:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:57784->10.242.0.7:8301: i/o timeout 2021/06/02 11:26:12 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:26:16 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:26:16 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:26:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:57830->10.242.0.7:8301: i/o timeout 2021/06/02 11:26:17 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:26:17 [WARN] raft: Heartbeat timeout from "10.242.0.7:8300" reached, starting election 2021/06/02 11:26:17 [INFO] raft: Node at 10.242.68.40:8300 [Candidate] entering Candidate state in term 8 2021/06/02 11:26:19 [INFO] raft: Duplicate RequestVote for same term: 8 2021/06/02 11:26:19 [DEBUG] raft-net: 10.242.68.40:8300 accepted connection from: 10.242.166.6:56820 2021/06/02 11:26:20 [WARN] memberlist: Was able to connect to dev-consul-server-2 but other probes failed, network may be misconfigured 2021/06/02 11:26:24 [INFO] raft: Node at 10.242.68.40:8300 [Follower] entering Follower state (Leader: "10.242.166.6:8300") 2021/06/02 11:26:24 [INFO] consul: New leader elected: dev-consul-server-1 2021/06/02 11:26:25 [DEBUG] raft-net: 10.242.68.40:8300 accepted connection from: 10.242.166.6:56890 2021/06/02 11:28:16 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58792->10.242.0.7:8301: i/o timeout 2021/06/02 11:28:16 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:28:19 [WARN] memberlist: Was able to connect to dev-consul-server-2 but other probes failed, network may be misconfigured 2021/06/02 11:28:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58864->10.242.0.7:8301: i/o timeout 2021/06/02 11:28:23 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:28:24 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (2 peer confirmations) 2021/06/02 11:28:24 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 11:28:24 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:28:26 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58874->10.242.0.7:8301: i/o timeout 2021/06/02 11:28:26 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:28:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58822->10.242.0.7:8302: i/o timeout 2021/06/02 11:28:33 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:28:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58994->10.242.0.7:8302: i/o timeout 2021/06/02 11:28:53 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:28:53 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:28:56 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 11:28:56 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:28:56 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:29:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:59046->10.242.0.7:8302: i/o timeout 2021/06/02 11:29:03 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:29:18 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 11:29:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:31:03 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:32:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:33:13 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:33:28 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:33:28 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:33:28 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2.dc1) 2021/06/02 11:33:28 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:33:28 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:37:44 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:35372->10.242.0.7:8301: i/o timeout 2021/06/02 11:37:44 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:37:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:35410->10.242.0.7:8301: i/o timeout 2021/06/02 11:37:47 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:37:47 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 11:37:47 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:38:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:35422->10.242.0.7:8302: i/o timeout 2021/06/02 11:38:03 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:38:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:35508->10.242.0.7:8302: i/o timeout 2021/06/02 11:38:13 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:38:23 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:38:27 [ERR] memberlist: Push/Pull with dev-consul-server-2.dc1 failed: read tcp 10.242.68.40:35388->10.242.0.7:8302: i/o timeout 2021/06/02 11:38:28 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:35636->10.242.0.7:8302: i/o timeout 2021/06/02 11:38:28 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:38:31 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 11:38:31 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:38:31 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:38:46 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:38:46 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:38:46 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2.dc1) 2021/06/02 11:38:46 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:38:46 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:41:05 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37156->10.242.0.7:8301: i/o timeout 2021/06/02 11:41:05 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:41:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37184->10.242.0.7:8301: i/o timeout 2021/06/02 11:41:08 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:41:09 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (2 peer confirmations) 2021/06/02 11:41:09 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 11:41:09 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:41:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37212->10.242.0.7:8301: i/o timeout 2021/06/02 11:41:10 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:41:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37120->10.242.0.7:8302: i/o timeout 2021/06/02 11:41:13 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:41:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37206->10.242.0.7:8302: i/o timeout 2021/06/02 11:41:23 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:41:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37252->10.242.0.7:8302: i/o timeout 2021/06/02 11:41:33 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:41:39 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 11:41:43 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 11:41:43 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:41:43 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:41:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:37438->10.242.0.7:8302: i/o timeout 2021/06/02 11:41:48 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:42:03 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:42:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:43:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:46:20 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:46:20 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:46:20 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:46:20 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:46:20 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2) 2021/06/02 11:46:20 [WARN] raft: Rejecting vote request from 10.242.0.7:8300 since we have a leader: 10.242.166.6:8300 2021/06/02 11:47:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:40830->10.242.0.7:8301: i/o timeout 2021/06/02 11:47:53 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:47:55 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 11:47:55 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:48:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:40846->10.242.0.7:8302: i/o timeout 2021/06/02 11:48:08 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:48:13 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:48:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:40888->10.242.0.7:8302: i/o timeout 2021/06/02 11:48:18 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:48:32 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:48:32 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:48:32 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 11:48:32 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.0.7:58946 2021/06/02 11:48:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:41088->10.242.0.7:8302: i/o timeout 2021/06/02 11:48:33 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:48:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:50:53 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:51:23 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:51:33 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:51:57 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:51:57 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:51:57 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:51:57 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:51:59 [WARN] raft: Rejecting vote request from 10.242.0.7:8300 since we have a leader: 10.242.166.6:8300 2021/06/02 11:53:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:43816->10.242.0.7:8301: i/o timeout 2021/06/02 11:53:24 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:53:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:43826->10.242.0.7:8301: i/o timeout 2021/06/02 11:53:27 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 11:53:27 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 11:53:27 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:53:38 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:43838->10.242.0.7:8302: i/o timeout 2021/06/02 11:53:38 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:53:41 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 11:53:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:43882->10.242.0.7:8302: i/o timeout 2021/06/02 11:53:48 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:53:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:44012->10.242.0.7:8302: i/o timeout 2021/06/02 11:53:58 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 11:54:01 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:54:01 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:54:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:55:13 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:55:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:56:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 11:57:23 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:58:02 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 11:58:33 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 11:58:45 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 11:58:45 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 11:58:45 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 11:58:45 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 11:58:45 [WARN] raft: Rejecting vote request from 10.242.0.7:8300 since we have a leader: 10.242.166.6:8300 2021/06/02 12:00:16 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:47536->10.242.0.7:8301: i/o timeout 2021/06/02 12:00:16 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:00:19 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 12:00:19 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:00:20 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:47574->10.242.0.7:8301: i/o timeout 2021/06/02 12:00:20 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:00:25 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 12:00:25 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.0.7:36640 2021/06/02 12:00:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:47570->10.242.0.7:8302: i/o timeout 2021/06/02 12:00:33 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:00:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:47660->10.242.0.7:8302: i/o timeout 2021/06/02 12:00:43 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:00:43 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:00:51 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 12:00:51 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:00:51 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:00:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:47754->10.242.0.7:8302: i/o timeout 2021/06/02 12:00:58 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:02:23 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 12:03:23 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:03:53 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 12:04:33 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:04:40 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 12:04:40 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:04:40 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:04:40 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:04:40 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2) 2021/06/02 12:04:40 [WARN] raft: Rejecting vote request from 10.242.0.7:8300 since we have a leader: 10.242.166.6:8300 2021/06/02 12:06:01 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:50708->10.242.0.7:8301: i/o timeout 2021/06/02 12:06:01 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:06:05 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:50738->10.242.0.7:8301: i/o timeout 2021/06/02 12:06:05 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:06:05 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (2 peer confirmations) 2021/06/02 12:06:05 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 12:06:05 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:06:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:50660->10.242.0.7:8302: i/o timeout 2021/06/02 12:06:08 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:06:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:50746->10.242.0.7:8302: i/o timeout 2021/06/02 12:06:18 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:06:19 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 12:06:19 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.0.7:39366 2021/06/02 12:06:28 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:50834->10.242.0.7:8302: i/o timeout 2021/06/02 12:06:28 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:06:38 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 12:06:38 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:06:38 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:07:03 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 12:09:00 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 12:09:00 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:09:00 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:09:00 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:09:00 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2) 2021/06/02 12:09:01 [WARN] raft: Rejecting vote request from 10.242.0.7:8300 since we have a leader: 10.242.166.6:8300 2021/06/02 12:15:20 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:55658->10.242.0.7:8301: i/o timeout 2021/06/02 12:15:20 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:15:21 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (2 peer confirmations) 2021/06/02 12:15:21 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 12:15:21 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:15:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:55566->10.242.0.7:8302: i/o timeout 2021/06/02 12:15:23 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:15:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:55674->10.242.0.7:8301: i/o timeout 2021/06/02 12:15:23 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:15:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:55658->10.242.0.7:8302: i/o timeout 2021/06/02 12:15:33 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:15:40 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:15:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:55708->10.242.0.7:8302: i/o timeout 2021/06/02 12:15:43 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:15:46 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 12:15:51 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 12:15:51 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:15:51 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:16:23 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 12:17:35 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/02 12:17:50 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:18:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 12:20:08 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:20:08 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:20:08 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.0.7 2021/06/02 12:20:08 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:20:09 [WARN] raft: Rejecting vote request from 10.242.0.7:8300 since we have a leader: 10.242.166.6:8300 2021/06/02 12:21:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58942->10.242.0.7:8301: i/o timeout 2021/06/02 12:21:24 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:21:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58980->10.242.0.7:8301: i/o timeout 2021/06/02 12:21:27 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:21:28 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (2 peer confirmations) 2021/06/02 12:21:28 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.0.7 2021/06/02 12:21:28 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.0.7:8300) (DC: dc1) 2021/06/02 12:21:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:59010->10.242.0.7:8301: i/o timeout 2021/06/02 12:21:30 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/02 12:21:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:58918->10.242.0.7:8302: i/o timeout 2021/06/02 12:21:33 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:21:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:59054->10.242.0.7:8302: i/o timeout 2021/06/02 12:21:48 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:21:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:59144->10.242.0.7:8302: i/o timeout 2021/06/02 12:21:58 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:22:03 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/02 12:22:03 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.0.7 2021/06/02 12:22:03 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/02 12:22:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.68.40:59200->10.242.0.7:8302: i/o timeout 2021/06/02 12:22:08 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/02 12:22:23 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 12:22:30 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:23:10 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.0.7:8301 2021/06/02 12:23:53 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.0.7:8302 2021/06/02 12:25:32 [ERR] yamux: keepalive failed: i/o deadline reached