+ CONSUL_BIND= + [ -n ] + CONSUL_CLIENT= + [ -n ] + CONSUL_DATA_DIR=/consul/data + CONSUL_CONFIG_DIR=/consul/config + [ -n ] + [ a = - ] + [ agent = agent ] + shift + set -- consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui + [ consul = consul ] + stat -c %u /consul/data + id -u consul + [ 100 != 100 ] + stat -c %u /consul/config + id -u consul + [ 100 != 100 ] + [ ! -z ] + exec consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui bootstrap_expect > 0: expecting 3 servers ==> Starting Consul agent... ==> Consul agent running! Version: 'v1.0.6' Node ID: '0e6d1abb-47a7-4121-d0d1-64118dab1223' Node name: 'dev-consul-server-2' Datacenter: 'dc1' (Segment: '') Server: true (Bootstrap: false) Client Addr: [0.0.0.0] (HTTP: 8500, HTTPS: -1, DNS: 8600) Cluster Addr: 10.242.12.17 (LAN: 8301, WAN: 8302) Encrypt: Gossip: false, TLS-Outgoing: false, TLS-Incoming: false ==> Log data will now stream in as it occurs: 2021/04/08 14:16:42 [INFO] raft: Initial configuration (index=0): [] 2021/04/08 14:16:42 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.12.17 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.12.17 2021/04/08 14:16:42 [INFO] agent: Started DNS server 0.0.0.0:8600 (tcp) 2021/04/08 14:16:42 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/04/08 14:16:42 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.12.17:8300) (DC: dc1) 2021/04/08 14:16:42 [INFO] agent: Started DNS server 0.0.0.0:8600 (udp) 2021/04/08 14:16:42 [INFO] agent: Started HTTP server on [::]:8500 (tcp) 2021/04/08 14:16:42 [INFO] agent: started state syncer 2021/04/08 14:16:42 [INFO] agent: Retry join LAN is supported for: aliyun aws azure digitalocean gce os scaleway softlayer 2021/04/08 14:16:42 [INFO] agent: Joining LAN cluster... 2021/04/08 14:16:42 [INFO] agent: (LAN) joining: [dev-consul-server-0.consul-server dev-consul-server-1.consul-server dev-consul-server-2.consul-server] 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.13 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.202 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 14:16:42 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 14:16:42 [INFO] consul: Existing Raft peers reported by dev-consul-server-0, disabling bootstrap mode 2021/04/08 14:16:42 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.254.13 2021/04/08 14:16:42 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.202 2021/04/08 14:16:42 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 14:16:42 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 14:16:42 [WARN] memberlist: Failed to resolve dev-consul-server-2.consul-server: lookup dev-consul-server-2.consul-server on 10.96.0.10:53: no such host 2021/04/08 14:16:42 [INFO] agent: (LAN) joined: 2 Err: 2021/04/08 14:16:42 [INFO] agent: Join LAN completed. Synced with 2 initial agents 2021/04/08 14:16:46 [DEBUG] raft-net: 10.242.12.17:8300 accepted connection from: 10.242.139.202:51408 2021/04/08 14:16:46 [WARN] raft: Failed to get previous log: 1 log not found (last: 0) 2021/04/08 14:16:46 [INFO] consul: New leader elected: dev-consul-server-0 2021/04/08 14:16:47 [INFO] agent: Synced node info 2021/04/08 14:16:47 [DEBUG] raft-net: 10.242.12.17:8300 accepted connection from: 10.242.139.202:51416 ==> Failed to check for updates: Get https://checkpoint-api.hashicorp.com/v1/check/consul?arch=amd64&os=linux&signature=a70fbffc-3215-f173-b8c9-a2b0fecf1890&version=1.0.6: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) 2021/04/08 14:39:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60964->10.242.139.202:8301: i/o timeout 2021/04/08 14:39:12 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 14:39:15 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 14:39:15 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.202 2021/04/08 14:39:15 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:39:22 [WARN] raft: Heartbeat timeout from "10.242.139.202:8300" reached, starting election 2021/04/08 14:39:22 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 3 2021/04/08 14:39:22 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:39:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44538->10.242.139.202:8302: i/o timeout 2021/04/08 14:39:22 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 14:39:23 [DEBUG] raft-net: 10.242.12.17:8300 accepted connection from: 10.242.254.13:39008 2021/04/08 14:39:23 [INFO] raft: Duplicate RequestVote for same term: 3 2021/04/08 14:39:23 [ERR] http: Request GET /v1/catalog/service/config_binding_service, error: No cluster leader from=10.242.17.5:33968 2021/04/08 14:39:29 [WARN] raft: Election timeout reached, restarting election 2021/04/08 14:39:29 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 4 2021/04/08 14:39:29 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:39:29 [INFO] raft: Election won. Tally: 2 2021/04/08 14:39:29 [INFO] raft: Node at 10.242.12.17:8300 [Leader] entering Leader state 2021/04/08 14:39:29 [INFO] raft: Added peer af5f140e-ee05-617b-6b81-fb4efa50aa1e, starting replication 2021/04/08 14:39:29 [INFO] raft: Added peer 4f3a59bd-f126-91ba-02a6-a5276b9eecac, starting replication 2021/04/08 14:39:29 [INFO] consul: cluster leadership acquired 2021/04/08 14:39:29 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:39:29 [INFO] consul: New leader elected: dev-consul-server-2 2021/04/08 14:39:29 [INFO] raft: pipelining replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 14:39:29 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 14:39:29 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:39:31 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.504833872s 2021/04/08 14:39:32 [ERR] raft: Failed to make RequestVote RPC to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:40522->10.242.139.202:8300: i/o timeout 2021/04/08 14:39:32 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.202 2021/04/08 14:39:32 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:39:32 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 14:39:32 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:39:32 [INFO] raft: pipelining replication to peer {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} 2021/04/08 14:39:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:39:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:39:38 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33122->10.242.139.202:8301: i/o timeout 2021/04/08 14:39:38 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 14:39:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:39:39 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500199297s 2021/04/08 14:39:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33148->10.242.139.202:8301: i/o timeout 2021/04/08 14:39:41 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 14:39:42 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:39:42 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:39:42 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.202 2021/04/08 14:39:42 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:39:42 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:39:42 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 14:39:43 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500278254s 2021/04/08 14:39:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:39:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:39:45 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 4.911562149s 2021/04/08 14:39:46 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.202 2021/04/08 14:39:46 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:39:46 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 14:39:47 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 14:39:48 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:39:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:39:50 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:39:50 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:39:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33300->10.242.139.202:8301: i/o timeout 2021/04/08 14:39:50 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 14:39:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:39:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:39:52 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.202 2021/04/08 14:39:52 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:39:52 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:39:52 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.202 2021/04/08 14:39:52 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:39:53 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 14:39:53 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 14:39:54 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:39:55 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 14:39:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:39:58 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:39:58 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:39:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33416->10.242.139.202:8301: i/o timeout 2021/04/08 14:39:58 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 14:39:59 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.50024875s 2021/04/08 14:39:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:01 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 14:40:01 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.202 2021/04/08 14:40:01 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 14:40:01 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 14:40:01 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33470->10.242.139.202:8301: i/o timeout 2021/04/08 14:40:01 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 14:40:01 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 4.979475994s 2021/04/08 14:40:02 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45120->10.242.139.202:8302: i/o timeout 2021/04/08 14:40:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:04 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.503104014s 2021/04/08 14:40:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:05 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:07 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500382129s 2021/04/08 14:40:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:08 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:09 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:09 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:10 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:11 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:11 [INFO] raft: aborting pipeline replication to peer {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} 2021/04/08 14:40:11 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:12 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:40:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45256->10.242.139.202:8302: i/o timeout 2021/04/08 14:40:12 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 14:40:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:14 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500370876s 2021/04/08 14:40:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:16 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 4.923204569s 2021/04/08 14:40:16 [INFO] raft: pipelining replication to peer {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} 2021/04/08 14:40:16 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:19 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500394528s 2021/04/08 14:40:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:21 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 4.989323531s 2021/04/08 14:40:22 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:40634->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45414->10.242.139.202:8302: i/o timeout 2021/04/08 14:40:22 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 14:40:22 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:23 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:40:23 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:40:23 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.202:51376 2021/04/08 14:40:23 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:40:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:40:24 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:40:24 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 7.473170893s 2021/04/08 14:40:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:27 [INFO] raft: aborting pipeline replication to peer {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} 2021/04/08 14:40:27 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:29 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500180692s 2021/04/08 14:40:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:31 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 14:40:31 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.202 2021/04/08 14:40:31 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 14:40:31 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 4.706766786s 2021/04/08 14:40:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45602->10.242.139.202:8302: i/o timeout 2021/04/08 14:40:32 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 14:40:32 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:41404->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:33 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:34 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 7.205634501s 2021/04/08 14:40:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:37 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:41462->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:37 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:43 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:41604->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:44 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:47 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:41648->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:47 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:54 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:41766->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:55 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:57 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:41808->10.242.139.202:8300: i/o timeout 2021/04/08 14:40:57 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:40:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:40:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:00 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:41:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:41:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:03 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:41:03 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:41:05 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:41912->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:06 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:06 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:41:07 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:41944->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:07 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:16 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:42070->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:17 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:17 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:42080->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:17 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:27 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:42240->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:27 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:42248->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:27 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:28 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:37 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:42382->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:37 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:38 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:42394->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:38 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:41:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:41:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:39 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:45 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:41:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:41:47 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:42568->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:41:48 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:49 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:42596->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:51 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:41:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:58 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:42732->10.242.139.202:8300: i/o timeout 2021/04/08 14:41:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:41:59 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:01 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:42778->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:02 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:42:02 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.202:40964 2021/04/08 14:42:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:05 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:09 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:42886->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:12 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:15 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:42952->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:21 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:22 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:43080->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:27 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:42:27 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:42:27 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:30 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:42:31 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:43224->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:37 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:43308->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:42 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:42:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:47 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:42:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:52 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:43560->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:42:57 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:43652->10.242.139.202:8300: i/o timeout 2021/04/08 14:42:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:03 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:43:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:08 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:43:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:09 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:43:09 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:43:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:43:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:13 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:43864->10.242.139.202:8300: i/o timeout 2021/04/08 14:43:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:18 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:43926->10.242.139.202:8300: i/o timeout 2021/04/08 14:43:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:24 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:43:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:28 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:43:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:34 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:44206->10.242.139.202:8300: i/o timeout 2021/04/08 14:43:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:38 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:44536->10.242.139.202:8300: i/o timeout 2021/04/08 14:43:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:45 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:43:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:48 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:43:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:51 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:43:51 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:43:54 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:43:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:55 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:45458->10.242.139.202:8300: i/o timeout 2021/04/08 14:43:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:43:58 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:45548->10.242.139.202:8300: i/o timeout 2021/04/08 14:43:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:06 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:44:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:08 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:44:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:44:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:16 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:45936->10.242.139.202:8300: i/o timeout 2021/04/08 14:44:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:18 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:46016->10.242.139.202:8300: i/o timeout 2021/04/08 14:44:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:27 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:44:28 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:44:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:33 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:44:33 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:44:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:44:36 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:44:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:37 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:46888->10.242.139.202:8300: i/o timeout 2021/04/08 14:44:39 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:46910->10.242.139.202:8300: i/o timeout 2021/04/08 14:44:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:48 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:44:49 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:44:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:44:58 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:47232->10.242.139.202:8300: i/o timeout 2021/04/08 14:44:59 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:47250->10.242.139.202:8300: i/o timeout 2021/04/08 14:44:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:09 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:45:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:09 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:45:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:15 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:45:15 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:45:18 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:45:19 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:47528->10.242.139.202:8300: i/o timeout 2021/04/08 14:45:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:19 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:47532->10.242.139.202:8300: i/o timeout 2021/04/08 14:45:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:29 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:45:30 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:45:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:39 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:47826->10.242.139.202:8300: i/o timeout 2021/04/08 14:45:40 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:47866->10.242.139.202:8300: i/o timeout 2021/04/08 14:45:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:49 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:45:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:51 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:45:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:57 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:45:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:45:57 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:45:59 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:48154->10.242.139.202:8300: i/o timeout 2021/04/08 14:46:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:46:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:01 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:48178->10.242.139.202:8300: i/o timeout 2021/04/08 14:46:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:10 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:46:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:12 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:46:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:20 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:48422->10.242.139.202:8300: i/o timeout 2021/04/08 14:46:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:22 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:48476->10.242.139.202:8300: i/o timeout 2021/04/08 14:46:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:30 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:46:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:33 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:46:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:39 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:46:39 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:46:40 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:48888->10.242.139.202:8300: i/o timeout 2021/04/08 14:46:42 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:46:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:46:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:43 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:48962->10.242.139.202:8300: i/o timeout 2021/04/08 14:46:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:45 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:46:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:50 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:46:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:55 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:46:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:46:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:00 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:47:00 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:49220->10.242.139.202:8300: i/o timeout 2021/04/08 14:47:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:47:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 14:47:05 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:49286->10.242.139.202:8300: i/o timeout 2021/04/08 14:47:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:10 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:47:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:15 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:47:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:20 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:49520->10.242.139.202:8300: i/o timeout 2021/04/08 14:47:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:21 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:47:21 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:47:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:47:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:25 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:49604->10.242.139.202:8300: i/o timeout 2021/04/08 14:47:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:31 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:47:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:36 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:47:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:41 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:49876->10.242.139.202:8300: i/o timeout 2021/04/08 14:47:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:46 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:49936->10.242.139.202:8300: i/o timeout 2021/04/08 14:47:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:51 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:47:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:55 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:47:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:47:57 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:47:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:01 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:50166->10.242.139.202:8300: i/o timeout 2021/04/08 14:48:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:03 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:48:03 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:48:06 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:48:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:07 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:50264->10.242.139.202:8300: i/o timeout 2021/04/08 14:48:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:11 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:48:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:48:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:18 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:48:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:21 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:50474->10.242.139.202:8300: i/o timeout 2021/04/08 14:48:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:28 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:50594->10.242.139.202:8300: i/o timeout 2021/04/08 14:48:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:31 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:48:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:35 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:48:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:39 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:48:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:41 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:50826->10.242.139.202:8300: i/o timeout 2021/04/08 14:48:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:45 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:48:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:48:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:48:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:49 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:50938->10.242.139.202:8300: i/o timeout 2021/04/08 14:48:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:52 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:48:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:48:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:00 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:49:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:02 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:51132->10.242.139.202:8300: i/o timeout 2021/04/08 14:49:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:10 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:51250->10.242.139.202:8300: i/o timeout 2021/04/08 14:49:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:12 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:49:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:49:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:15 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:49:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:21 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:49:22 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:51422->10.242.139.202:8300: i/o timeout 2021/04/08 14:49:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:27 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:49:27 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:49:30 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:49:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:31 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:51596->10.242.139.202:8300: i/o timeout 2021/04/08 14:49:32 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:49:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:42 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:51830->10.242.139.202:8300: i/o timeout 2021/04/08 14:49:42 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:49:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:52 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:49:52 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:52032->10.242.139.202:8300: i/o timeout 2021/04/08 14:49:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:55 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:49:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:49:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:02 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:52184->10.242.139.202:8300: i/o timeout 2021/04/08 14:50:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:04 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:50:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:09 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:50:09 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:50:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:50:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:50:13 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:50:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:14 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:52342->10.242.139.202:8300: i/o timeout 2021/04/08 14:50:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:23 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:52484->10.242.139.202:8300: i/o timeout 2021/04/08 14:50:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:25 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:50:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:33 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:50:35 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:52662->10.242.139.202:8300: i/o timeout 2021/04/08 14:50:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:43 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:52836->10.242.139.202:8300: i/o timeout 2021/04/08 14:50:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:46 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:50:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:51 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:50:51 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:50:53 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:50:54 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:50:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:56 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:53042->10.242.139.202:8300: i/o timeout 2021/04/08 14:50:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:50:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:03 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:53148->10.242.139.202:8300: i/o timeout 2021/04/08 14:51:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:05 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:51:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:07 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:51:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:51:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:13 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:51:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:17 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:53356->10.242.139.202:8300: i/o timeout 2021/04/08 14:51:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:23 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:53458->10.242.139.202:8300: i/o timeout 2021/04/08 14:51:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:28 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:51:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:33 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:51:33 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:51:34 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:51:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:51:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:38 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:53686->10.242.139.202:8300: i/o timeout 2021/04/08 14:51:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:44 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:53802->10.242.139.202:8300: i/o timeout 2021/04/08 14:51:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:49 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:51:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:54 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:51:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:51:59 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:54054->10.242.139.202:8300: i/o timeout 2021/04/08 14:52:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:04 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:54112->10.242.139.202:8300: i/o timeout 2021/04/08 14:52:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:10 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:52:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:52:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:14 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:52:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:15 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:52:15 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:52:15 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:52:18 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:52:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:20 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:54348->10.242.139.202:8300: i/o timeout 2021/04/08 14:52:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:54420->10.242.139.202:8300: i/o timeout 2021/04/08 14:52:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:31 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:52:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:34 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:52:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:41 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:54708->10.242.139.202:8300: i/o timeout 2021/04/08 14:52:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:44 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:54778->10.242.139.202:8300: i/o timeout 2021/04/08 14:52:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:52 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:52:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:55 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:52:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:52:57 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:52:57 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:53:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:53:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:02 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:55050->10.242.139.202:8300: i/o timeout 2021/04/08 14:53:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:05 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:55086->10.242.139.202:8300: i/o timeout 2021/04/08 14:53:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:53:13 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:53:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:15 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:53:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:23 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:55364->10.242.139.202:8300: i/o timeout 2021/04/08 14:53:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:25 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:55398->10.242.139.202:8300: i/o timeout 2021/04/08 14:53:25 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:53:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:34 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:53:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:35 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:53:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:39 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:53:39 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:53:42 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:53:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:44 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:55730->10.242.139.202:8300: i/o timeout 2021/04/08 14:53:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:45 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:55750->10.242.139.202:8300: i/o timeout 2021/04/08 14:53:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:54 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:53:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:55 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:53:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:53:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:04 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:56050->10.242.139.202:8300: i/o timeout 2021/04/08 14:54:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:05 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:56062->10.242.139.202:8300: i/o timeout 2021/04/08 14:54:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:15 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:54:16 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:54:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:21 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:54:21 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:54:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:54:24 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:54:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:25 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:56378->10.242.139.202:8300: i/o timeout 2021/04/08 14:54:26 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:56388->10.242.139.202:8300: i/o timeout 2021/04/08 14:54:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:36 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:54:36 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:54:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:46 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:56712->10.242.139.202:8300: i/o timeout 2021/04/08 14:54:46 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:56716->10.242.139.202:8300: i/o timeout 2021/04/08 14:54:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:56 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:54:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:54:57 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:54:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:03 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:55:03 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:55:06 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:55:06 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:55:06 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:57094->10.242.139.202:8300: i/o timeout 2021/04/08 14:55:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:07 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:57126->10.242.139.202:8300: i/o timeout 2021/04/08 14:55:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:55:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:16 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:55:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:19 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:55:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:26 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:57406->10.242.139.202:8300: i/o timeout 2021/04/08 14:55:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:29 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:57460->10.242.139.202:8300: i/o timeout 2021/04/08 14:55:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:35 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:55:37 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:55:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:40 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:55:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:45 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:55:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:55:47 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:57750->10.242.139.202:8300: i/o timeout 2021/04/08 14:55:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:55:48 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:55:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:50 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:57798->10.242.139.202:8300: i/o timeout 2021/04/08 14:55:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:57 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:55:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:55:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:01 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:56:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:07 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:58068->10.242.139.202:8300: i/o timeout 2021/04/08 14:56:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:11 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:58140->10.242.139.202:8300: i/o timeout 2021/04/08 14:56:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:56:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:15 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:56:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:17 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:56:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:22 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:56:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:27 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:56:27 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:56:27 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:58386->10.242.139.202:8300: i/o timeout 2021/04/08 14:56:30 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:56:30 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:56:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:32 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:58456->10.242.139.202:8300: i/o timeout 2021/04/08 14:56:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:37 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:56:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:43 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:56:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:47 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: read tcp 10.242.12.17:58732->10.242.139.202:8300: i/o timeout 2021/04/08 14:56:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:53 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: read tcp 10.242.12.17:58818->10.242.139.202:8300: i/o timeout 2021/04/08 14:56:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:55 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:56:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:56:57 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:56:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:04 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:57:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:07 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:09 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 14:57:09 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 14:57:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:57:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:57:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:14 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:18 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:57:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:21 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:57:25 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:57:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:28 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:33 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:35 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:57:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:38 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:57:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:46 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:57:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:57:48 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:57:48 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:56 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:57:57 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:57:58 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:58:00 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 14:58:00 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:58:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:07 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:08 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:09 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:58:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:17 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:18 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:21 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:58:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:28 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:28 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:33 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:35 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:58:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:58:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:38 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:39 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:58:49 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:49 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:58:57 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:58:59 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:58:59 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:59:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:09 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:09 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:10 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:59:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:59:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:15 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 14:59:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:19 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:59:20 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:21 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:59:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:29 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:31 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:59:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:33 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:59:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:39 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:59:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:41 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 14:59:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 14:59:48 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 14:59:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:49 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 14:59:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:52 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 14:59:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 14:59:57 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:00 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:00:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:00:00 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:00:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:02 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:09 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:10 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:12 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:00:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:13 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:00:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:20 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:00:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:21 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:23 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:24 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:00:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:30 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:33 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:34 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:00:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:00:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:40 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:00:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 15:00:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:44 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:45 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:48 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:00:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:50 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:54 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:00:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:00:57 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:00:58 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:00:58 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:00:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:00 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:01:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:04 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:07 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:10 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:01:10 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:01:10 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:15 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:01:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:19 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:21 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:01:22 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:01:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:25 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:31 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:31 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:34 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:01:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:36 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:01:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:41 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:01:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 15:01:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:43 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:46 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:01:46 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:51 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:01:55 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:01:58 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:01:58 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:01:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:01 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:02:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:07 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:08 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:10 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:02:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:11 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:19 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:02:19 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:20 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:02:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:21 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:02:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:29 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:29 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:31 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:32 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:02:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:40 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:02:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:41 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:42 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:02:44 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:02:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:50 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:52 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:53 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:02:54 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:02:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:55 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.202:8301 2021/04/08 15:02:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:02:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:01 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:03:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:02 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:03:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:03 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:03:06 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:03:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:11 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:03:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:12 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:03:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:14 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.202 2021/04/08 15:03:14 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:03:14 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.202 2021/04/08 15:03:14 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 15:03:14 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:03:14 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:03:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:03:15 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:03:22 [INFO] raft: pipelining replication to peer {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} 2021/04/08 15:03:24 [INFO] raft: aborting pipeline replication to peer {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: EOF 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:24 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:24 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:25 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: stream closed 2021/04/08 15:03:25 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:25 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:25 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:03:25 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:26 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:26 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:03:26 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 2.500489699s 2021/04/08 15:03:26 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:27 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:27 [ERR] raft: Failed to AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:27 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:28 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:28 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:03:28 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:03:28 [WARN] raft: Failed to contact 4f3a59bd-f126-91ba-02a6-a5276b9eecac in 4.835361427s 2021/04/08 15:03:29 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:03:29 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.202 2021/04/08 15:03:29 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 15:03:29 [INFO] serf: EventMemberUpdate: dev-consul-server-0.dc1 2021/04/08 15:03:29 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.202 2021/04/08 15:03:29 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 15:03:29 [INFO] raft: Updating configuration with RemoveServer (4f3a59bd-f126-91ba-02a6-a5276b9eecac, ) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300}] 2021/04/08 15:03:29 [INFO] raft: Removed peer 4f3a59bd-f126-91ba-02a6-a5276b9eecac, stopping replication after 571 2021/04/08 15:03:29 [INFO] consul: removed server with duplicate address: 10.242.139.202:8300 2021/04/08 15:03:29 [INFO] raft: Updating configuration with AddNonvoter (bbd5f8b3-9364-c88a-2560-bad0880c12e6, 10.242.139.202:8300) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Nonvoter ID:bbd5f8b3-9364-c88a-2560-bad0880c12e6 Address:10.242.139.202:8300}] 2021/04/08 15:03:29 [INFO] raft: Added peer bbd5f8b3-9364-c88a-2560-bad0880c12e6, starting replication 2021/04/08 15:03:29 [WARN] raft: AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300} rejected, sending older logs (next: 1) 2021/04/08 15:03:29 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:03:29 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:03:29 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:03:29 [WARN] Unable to get address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac, using fallback address 10.242.139.202:8300: Could not find address for server id 4f3a59bd-f126-91ba-02a6-a5276b9eecac 2021/04/08 15:03:29 [WARN] raft: AppendEntries to {Voter 4f3a59bd-f126-91ba-02a6-a5276b9eecac 10.242.139.202:8300} rejected, sending older logs (next: 129) 2021/04/08 15:03:31 [INFO] raft: pipelining replication to peer {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300} 2021/04/08 15:03:32 [WARN] memberlist: Was able to connect to dev-consul-server-0.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:03:49 [INFO] autopilot: Promoting Server (ID: "bbd5f8b3-9364-c88a-2560-bad0880c12e6" Address: "10.242.139.202:8300") to voter 2021/04/08 15:03:49 [INFO] raft: Updating configuration with AddStaging (bbd5f8b3-9364-c88a-2560-bad0880c12e6, 10.242.139.202:8300) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:bbd5f8b3-9364-c88a-2560-bad0880c12e6 Address:10.242.139.202:8300}] 2021/04/08 15:03:59 [INFO] raft: aborting pipeline replication to peer {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300} 2021/04/08 15:03:59 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: EOF 2021/04/08 15:03:59 [ERR] raft: Failed to AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:59 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: stream closed 2021/04/08 15:03:59 [ERR] raft: Failed to AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:59 [ERR] raft: Failed to AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:03:59 [ERR] raft: Failed to AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: getsockopt: connection refused 2021/04/08 15:04:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:04:01 [WARN] raft: Failed to contact bbd5f8b3-9364-c88a-2560-bad0880c12e6 in 2.500190908s 2021/04/08 15:04:02 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:04:02 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:04:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:04:03 [WARN] raft: Failed to contact bbd5f8b3-9364-c88a-2560-bad0880c12e6 in 4.364859269s 2021/04/08 15:04:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:04:05 [WARN] raft: Failed to contact bbd5f8b3-9364-c88a-2560-bad0880c12e6 in 6.819356532s 2021/04/08 15:04:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:04:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:04:09 [ERR] raft: Failed to AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:09 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:04:11 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:13 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:04:14 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:04:14 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:04:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:04:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:04:16 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500382314s 2021/04/08 15:04:16 [WARN] raft: Failed to contact quorum of nodes, stepping down 2021/04/08 15:04:16 [INFO] raft: aborting pipeline replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:04:16 [INFO] consul: cluster leadership lost 2021/04/08 15:04:16 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 15:04:19 [ERR] raft: Failed to AppendEntries to {Nonvoter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:20 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:23 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error getting client: failed to get conn: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:23 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/04/08 15:04:23 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 5 2021/04/08 15:04:23 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:04:25 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:58258->10.242.254.13:8300: i/o timeout 2021/04/08 15:04:25 [ERR] memberlist: Conflicting address for dev-consul-server-0. Mine: 10.242.139.202:8301 Theirs: 10.242.139.247:8301 2021/04/08 15:04:25 [WARN] serf: Name conflict for 'dev-consul-server-0' both 10.242.139.202:8301 and 10.242.139.247:8301 are claiming 2021/04/08 15:04:25 [ERR] memberlist: Conflicting address for dev-consul-server-0. Mine: 10.242.139.202:8301 Theirs: 10.242.139.247:8301 2021/04/08 15:04:25 [WARN] serf: Name conflict for 'dev-consul-server-0' both 10.242.139.202:8301 and 10.242.139.247:8301 are claiming 2021/04/08 15:04:25 [ERR] memberlist: Conflicting address for dev-consul-server-0. Mine: 10.242.139.202:8301 Theirs: 10.242.139.247:8301 2021/04/08 15:04:25 [WARN] serf: Name conflict for 'dev-consul-server-0' both 10.242.139.202:8301 and 10.242.139.247:8301 are claiming 2021/04/08 15:04:25 [ERR] memberlist: Conflicting address for dev-consul-server-0. Mine: 10.242.139.202:8301 Theirs: 10.242.139.247:8301 2021/04/08 15:04:25 [WARN] serf: Name conflict for 'dev-consul-server-0' both 10.242.139.202:8301 and 10.242.139.247:8301 are claiming 2021/04/08 15:04:25 [ERR] memberlist: Conflicting address for dev-consul-server-0. Mine: 10.242.139.202:8301 Theirs: 10.242.139.247:8301 2021/04/08 15:04:25 [WARN] serf: Name conflict for 'dev-consul-server-0' both 10.242.139.202:8301 and 10.242.139.247:8301 are claiming 2021/04/08 15:04:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33888->10.242.254.10:8301: i/o timeout 2021/04/08 15:04:27 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:04:29 [ERR] memberlist: Push/Pull with dev-consul-7467bf7b46-dk9sm failed: read tcp 10.242.12.17:33818->10.242.254.10:8301: i/o timeout 2021/04/08 15:04:32 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:04:32 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 6 2021/04/08 15:04:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50818->10.242.254.13:8301: i/o timeout 2021/04/08 15:04:32 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:04:33 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:55724->10.242.254.13:8300: i/o timeout 2021/04/08 15:04:33 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:35 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:04:37 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:04:37 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.202 2021/04/08 15:04:37 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.202:8300) (DC: dc1) 2021/04/08 15:04:38 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:04:38 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 7 2021/04/08 15:04:38 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:04:38 [ERR] http: Request GET /v1/catalog/service/config-binding-service, error: No cluster leader from=10.242.254.29:55452 2021/04/08 15:04:38 [INFO] memberlist: Marking dev-consul-7467bf7b46-dk9sm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:04:38 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:04:39 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:04:39 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.254.13:39146 2021/04/08 15:04:39 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:04:39 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.254.13 2021/04/08 15:04:39 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:04:40 [ERR] agent: Coordinate update error: No cluster leader 2021/04/08 15:04:41 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:04:41 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/04/08 15:04:42 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:55876->10.242.254.13:8300: i/o timeout 2021/04/08 15:04:42 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:43 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:04:44 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:04:44 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.254.10:40814 2021/04/08 15:04:44 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:04:44 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 8 2021/04/08 15:04:44 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:04:44 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:04:48 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:56012->10.242.254.13:8300: i/o timeout 2021/04/08 15:04:48 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:48 [ERR] agent: failed to sync remote state: No cluster leader 2021/04/08 15:04:51 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:04:51 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 9 2021/04/08 15:04:51 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:04:51 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:04:54 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:56120->10.242.254.13:8300: i/o timeout 2021/04/08 15:04:54 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:04:58 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:04:58 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 10 2021/04/08 15:04:58 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:04:58 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:01 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:56250->10.242.254.13:8300: i/o timeout 2021/04/08 15:05:01 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:03 [ERR] agent: Coordinate update error: No cluster leader 2021/04/08 15:05:05 [ERR] memberlist: Push/Pull with dev-consul-server-1.dc1 failed: read tcp 10.242.12.17:41668->10.242.254.13:8302: i/o timeout 2021/04/08 15:05:05 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:05:05 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 11 2021/04/08 15:05:05 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:05:05 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:05 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:05:05 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.202 2021/04/08 15:05:05 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:05:05 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.254.13:8301 2021/04/08 15:05:08 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:56350->10.242.254.13:8300: i/o timeout 2021/04/08 15:05:08 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:08 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:05:12 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 15:05:13 [ERR] agent: failed to sync remote state: No cluster leader 2021/04/08 15:05:13 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:05:13 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:05:14 [WARN] raft: Election timeout reached, restarting election 2021/04/08 15:05:14 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 12 2021/04/08 15:05:14 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:05:14 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:15 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.139.202:8302 Theirs: 10.242.139.247:8302 2021/04/08 15:05:15 [WARN] serf: Name conflict for 'dev-consul-server-0.dc1' both 10.242.139.202:8302 and 10.242.139.247:8302 are claiming 2021/04/08 15:05:15 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:15 [ERR] raft: Failed to make RequestVote RPC to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:56454->10.242.254.13:8300: i/o timeout 2021/04/08 15:05:15 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.139.202:8302 Theirs: 10.242.139.247:8302 2021/04/08 15:05:15 [WARN] serf: Name conflict for 'dev-consul-server-0.dc1' both 10.242.139.202:8302 and 10.242.139.247:8302 are claiming 2021/04/08 15:05:15 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.139.202:8302 Theirs: 10.242.139.247:8302 2021/04/08 15:05:15 [WARN] serf: Name conflict for 'dev-consul-server-0.dc1' both 10.242.139.202:8302 and 10.242.139.247:8302 are claiming 2021/04/08 15:05:16 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.139.202:8302 Theirs: 10.242.139.247:8302 2021/04/08 15:05:16 [WARN] serf: Name conflict for 'dev-consul-server-0.dc1' both 10.242.139.202:8302 and 10.242.139.247:8302 are claiming 2021/04/08 15:05:16 [ERR] memberlist: Conflicting address for dev-consul-server-0.dc1. Mine: 10.242.139.202:8302 Theirs: 10.242.139.247:8302 2021/04/08 15:05:16 [WARN] serf: Name conflict for 'dev-consul-server-0.dc1' both 10.242.139.202:8302 and 10.242.139.247:8302 are claiming 2021/04/08 15:05:16 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:05:17 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.13 2021/04/08 15:05:17 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:05:20 [INFO] raft: Election won. Tally: 2 2021/04/08 15:05:20 [INFO] raft: Node at 10.242.12.17:8300 [Leader] entering Leader state 2021/04/08 15:05:20 [INFO] consul: cluster leadership acquired 2021/04/08 15:05:20 [INFO] raft: Added peer af5f140e-ee05-617b-6b81-fb4efa50aa1e, starting replication 2021/04/08 15:05:20 [INFO] raft: Added peer bbd5f8b3-9364-c88a-2560-bad0880c12e6, starting replication 2021/04/08 15:05:20 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:20 [INFO] consul: New leader elected: dev-consul-server-2 2021/04/08 15:05:20 [INFO] raft: pipelining replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:05:20 [INFO] raft: Updating configuration with AddNonvoter (c52c5261-fd75-2dd5-3bad-46a881682908, 10.242.139.247:8300) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:bbd5f8b3-9364-c88a-2560-bad0880c12e6 Address:10.242.139.202:8300} {Suffrage:Nonvoter ID:c52c5261-fd75-2dd5-3bad-46a881682908 Address:10.242.139.247:8300}] 2021/04/08 15:05:20 [INFO] raft: Added peer c52c5261-fd75-2dd5-3bad-46a881682908, starting replication 2021/04/08 15:05:20 [WARN] raft: AppendEntries to {Nonvoter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} rejected, sending older logs (next: 1) 2021/04/08 15:05:20 [INFO] raft: pipelining replication to peer {Nonvoter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:05:20 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:05:21 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:22 [WARN] raft: Failed to contact bbd5f8b3-9364-c88a-2560-bad0880c12e6 in 2.503547934s 2021/04/08 15:05:24 [ERR] raft: Failed to make RequestVote RPC to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:25 [WARN] raft: Failed to contact bbd5f8b3-9364-c88a-2560-bad0880c12e6 in 4.957402085s 2021/04/08 15:05:27 [WARN] raft: Failed to contact bbd5f8b3-9364-c88a-2560-bad0880c12e6 in 7.40914169s 2021/04/08 15:05:30 [ERR] raft: Failed to AppendEntries to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:30 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:31 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:31 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:40 [ERR] raft: Failed to AppendEntries to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:40 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:40 [INFO] autopilot: Promoting Server (ID: "c52c5261-fd75-2dd5-3bad-46a881682908" Address: "10.242.139.247:8300") to voter 2021/04/08 15:05:40 [INFO] raft: Updating configuration with AddStaging (c52c5261-fd75-2dd5-3bad-46a881682908, 10.242.139.247:8300) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:bbd5f8b3-9364-c88a-2560-bad0880c12e6 Address:10.242.139.202:8300} {Suffrage:Voter ID:c52c5261-fd75-2dd5-3bad-46a881682908 Address:10.242.139.247:8300}] 2021/04/08 15:05:40 [INFO] autopilot: Attempting removal of stale Server (ID: "bbd5f8b3-9364-c88a-2560-bad0880c12e6" Address: "10.242.139.202:8300") 2021/04/08 15:05:40 [INFO] raft: Updating configuration with RemoveServer (bbd5f8b3-9364-c88a-2560-bad0880c12e6, ) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:c52c5261-fd75-2dd5-3bad-46a881682908 Address:10.242.139.247:8300}] 2021/04/08 15:05:40 [INFO] raft: Removed peer bbd5f8b3-9364-c88a-2560-bad0880c12e6, stopping replication after 638 2021/04/08 15:05:41 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:42 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:50 [ERR] raft: Failed to AppendEntries to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:50 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:05:52 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:05:53 [WARN] Unable to get address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6, using fallback address 10.242.139.202:8300: Could not find address for server id bbd5f8b3-9364-c88a-2560-bad0880c12e6 2021/04/08 15:06:00 [ERR] raft: Failed to AppendEntries to {Voter bbd5f8b3-9364-c88a-2560-bad0880c12e6 10.242.139.202:8300}: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:06:03 [ERR] raft: Failed to heartbeat to 10.242.139.202:8300: dial tcp ->10.242.139.202:8300: i/o timeout 2021/04/08 15:06:28 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36086->10.242.254.10:8301: i/o timeout 2021/04/08 15:06:28 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:06:29 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1) 2021/04/08 15:06:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.202:8302 2021/04/08 15:07:02 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:07:02 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:07:15 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:07:39 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37396->10.242.254.10:8301: i/o timeout 2021/04/08 15:07:47 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:07:49 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:07:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:07:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:07:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:07:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:07:59 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:08:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:33 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:11:33 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:11:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41170->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:33 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:11:34 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.501276537s 2021/04/08 15:11:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41252->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:37 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:11:37 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:11:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:38 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500272676s 2021/04/08 15:11:39 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41298->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:39 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:11:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:40 [ERR] memberlist: Push/Pull with dev-consul-7467bf7b46-dk9sm failed: read tcp 10.242.12.17:41122->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:41 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.98132268s 2021/04/08 15:11:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:58168->10.242.254.13:8301: i/o timeout 2021/04/08 15:11:41 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:11:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48880->10.242.254.13:8302: i/o timeout 2021/04/08 15:11:42 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:11:43 [INFO] memberlist: Marking dev-consul-7467bf7b46-dk9sm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:11:43 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:11:43 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' failed, marking health critical 2021/04/08 15:11:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:58206->10.242.254.13:8301: i/o timeout 2021/04/08 15:11:43 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:11:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:45 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:11:45 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' joined, marking health alive 2021/04/08 15:11:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:46 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:58286->10.242.254.13:8301: i/o timeout 2021/04/08 15:11:46 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:11:48 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500244718s 2021/04/08 15:11:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41440->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:49 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:11:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:50 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.945355171s 2021/04/08 15:11:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49060->10.242.254.13:8302: i/o timeout 2021/04/08 15:11:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41494->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:52 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:11:54 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500193438s 2021/04/08 15:11:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:55 [INFO] raft: aborting pipeline replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:11:55 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:11:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:11:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41588->10.242.254.10:8301: i/o timeout 2021/04/08 15:11:56 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:11:58 [INFO] raft: pipelining replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:11:59 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:03 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41708->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:03 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:04 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500284188s 2021/04/08 15:12:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:07 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-dk9sm but other probes failed, network may be misconfigured 2021/04/08 15:12:09 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:09 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:12:09 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-dk9sm but other probes failed, network may be misconfigured 2021/04/08 15:12:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:10 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-dk9sm) 2021/04/08 15:12:13 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:13 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:12:13 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:12:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:14 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41886->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:14 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:14 [INFO] memberlist: Marking dev-consul-7467bf7b46-dk9sm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:12:14 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:12:14 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' failed, marking health critical 2021/04/08 15:12:15 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:12:15 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' joined, marking health alive 2021/04/08 15:12:17 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:17 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:12:17 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500258535s 2021/04/08 15:12:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:21 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42012->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:21 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:34 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42216->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:34 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:36 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59130->10.242.254.13:8301: i/o timeout 2021/04/08 15:12:36 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:12:37 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:37 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.502098675s 2021/04/08 15:12:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:38 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59162->10.242.254.13:8301: i/o timeout 2021/04/08 15:12:38 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:12:40 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.90623551s 2021/04/08 15:12:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:40 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42324->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:40 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49906->10.242.254.13:8302: i/o timeout 2021/04/08 15:12:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42364->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:42 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:43 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1) 2021/04/08 15:12:43 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1.dc1) 2021/04/08 15:12:44 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:12:45 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:12:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:46 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59314->10.242.254.13:8301: i/o timeout 2021/04/08 15:12:46 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:12:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42466->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:49 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42564->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:53 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:53 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500272581s 2021/04/08 15:12:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:56 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.9891251s 2021/04/08 15:12:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42576->10.242.254.10:8301: i/o timeout 2021/04/08 15:12:56 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:12:58 [INFO] raft: aborting pipeline replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:12:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:12:59 [INFO] memberlist: Marking dev-consul-7467bf7b46-dk9sm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:12:59 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:12:59 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' failed, marking health critical 2021/04/08 15:13:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59500->10.242.254.13:8301: i/o timeout 2021/04/08 15:13:00 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:13:00 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500332795s 2021/04/08 15:13:02 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:56756->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:03 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.933414491s 2021/04/08 15:13:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59558->10.242.254.13:8301: i/o timeout 2021/04/08 15:13:03 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:13:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:05 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 7.403875564s 2021/04/08 15:13:06 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:13:06 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.254.10:53480 2021/04/08 15:13:06 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:13:06 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.254.13 2021/04/08 15:13:06 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:13:06 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/04/08 15:13:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59628->10.242.254.13:8301: i/o timeout 2021/04/08 15:13:06 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:13:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50316->10.242.254.13:8302: i/o timeout 2021/04/08 15:13:07 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:13:08 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:36262->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:08 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:11 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:13 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:15 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500507626s 2021/04/08 15:13:15 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.254.13:8301 2021/04/08 15:13:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50404->10.242.254.13:8302: i/o timeout 2021/04/08 15:13:17 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:13:18 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:36420->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:18 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:18 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:19 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.506348329s 2021/04/08 15:13:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:22 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.989394258s 2021/04/08 15:13:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:24 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 7.445138152s 2021/04/08 15:13:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:28 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:36338->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:28 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:28 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:36620->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:28 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:30 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:13:30 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.254.13 2021/04/08 15:13:30 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 15:13:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50774->10.242.254.13:8302: i/o timeout 2021/04/08 15:13:32 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:13:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:35 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.254.13 2021/04/08 15:13:35 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 15:13:35 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.13 2021/04/08 15:13:35 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:13:35 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/04/08 15:13:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:38 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:36782->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:40 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500251641s 2021/04/08 15:13:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60306->10.242.254.13:8301: i/o timeout 2021/04/08 15:13:42 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:13:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:44 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:13:46 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:13:46 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.254.13 2021/04/08 15:13:46 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:13:46 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/04/08 15:13:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:46 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500219796s 2021/04/08 15:13:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51060->10.242.254.13:8302: i/o timeout 2021/04/08 15:13:47 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:13:48 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:36790->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:48 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:49 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.950161537s 2021/04/08 15:13:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:51 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:13:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:52 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:13:52 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' joined, marking health alive 2021/04/08 15:13:53 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500281073s 2021/04/08 15:13:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43642->10.242.254.10:8301: i/o timeout 2021/04/08 15:13:53 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:13:55 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:13:55 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-dk9sm but other probes failed, network may be misconfigured 2021/04/08 15:13:55 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.949976953s 2021/04/08 15:13:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43702->10.242.254.10:8301: i/o timeout 2021/04/08 15:13:57 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:13:57 [WARN] memberlist: Was able to connect to dev-consul-server-1.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:13:58 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:37178->10.242.254.13:8300: i/o timeout 2021/04/08 15:13:58 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 7.411193805s 2021/04/08 15:13:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:13:58 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:14:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43752->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:00 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:01 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:37008->10.242.254.13:8300: i/o timeout 2021/04/08 15:14:01 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.13 2021/04/08 15:14:01 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:14:01 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/04/08 15:14:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43796->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:02 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60680->10.242.254.13:8301: i/o timeout 2021/04/08 15:14:04 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:14:04 [INFO] memberlist: Marking dev-consul-7467bf7b46-dk9sm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:14:04 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:14:04 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' failed, marking health critical 2021/04/08 15:14:06 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1) 2021/04/08 15:14:06 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:14:06 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' joined, marking health alive 2021/04/08 15:14:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:08 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:37344->10.242.254.13:8300: i/o timeout 2021/04/08 15:14:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43922->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:10 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60842->10.242.254.13:8301: i/o timeout 2021/04/08 15:14:13 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:14:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:15 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500234859s 2021/04/08 15:14:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:16 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44050->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:16 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:18 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:37522->10.242.254.13:8300: i/o timeout 2021/04/08 15:14:19 [INFO] raft: pipelining replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:14:21 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:14:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44178->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:24 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:28 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44244->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:28 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:29 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:14:31 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:14:35 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:14:35 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:14:35 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44368->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:35 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:38 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44408->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:38 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:39 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500289806s 2021/04/08 15:14:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:41 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.948001541s 2021/04/08 15:14:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52032->10.242.254.13:8302: i/o timeout 2021/04/08 15:14:42 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:14:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:44 [ERR] raft: Failed to pipeline AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: write tcp 10.242.12.17:37418->10.242.254.13:8300: use of closed network connection 2021/04/08 15:14:44 [INFO] raft: aborting pipeline replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:14:44 [WARN] memberlist: Was able to connect to dev-consul-server-1 but other probes failed, network may be misconfigured 2021/04/08 15:14:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44554->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:45 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:46 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:14:46 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' failed, marking health critical 2021/04/08 15:14:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:47 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500249445s 2021/04/08 15:14:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33222->10.242.254.13:8301: i/o timeout 2021/04/08 15:14:48 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:14:49 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:14:49 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' joined, marking health alive 2021/04/08 15:14:49 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.966229294s 2021/04/08 15:14:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44650->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:50 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:51 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 7.384449264s 2021/04/08 15:14:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52216->10.242.254.13:8302: i/o timeout 2021/04/08 15:14:52 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:14:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44692->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:52 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:54 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33328->10.242.254.13:8301: i/o timeout 2021/04/08 15:14:54 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:14:54 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:37726->10.242.254.13:8300: i/o timeout 2021/04/08 15:14:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:38160->10.242.254.13:8300: i/o timeout 2021/04/08 15:14:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44734->10.242.254.10:8301: i/o timeout 2021/04/08 15:14:56 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:14:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:14:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33416->10.242.254.13:8301: i/o timeout 2021/04/08 15:14:58 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:14:59 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:14:59 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.254.13 2021/04/08 15:14:59 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:14:59 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/04/08 15:15:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44812->10.242.254.10:8301: i/o timeout 2021/04/08 15:15:00 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:15:01 [INFO] memberlist: Marking dev-consul-7467bf7b46-dk9sm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:15:01 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:15:01 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' failed, marking health critical 2021/04/08 15:15:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44862->10.242.254.10:8301: i/o timeout 2021/04/08 15:15:02 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:15:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:04 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:38340->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:04 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:05 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:38354->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:06 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52468->10.242.254.13:8302: i/o timeout 2021/04/08 15:15:07 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:15:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:14 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:38508->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:14 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:16 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:38540->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:17 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52546->10.242.254.13:8302: i/o timeout 2021/04/08 15:15:17 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:15:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:19 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:15:19 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/04/08 15:15:21 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:15:22 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:15:22 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.254.13:43390 2021/04/08 15:15:22 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:15:22 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.254.10:57948 2021/04/08 15:15:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:22 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:15:22 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.254.13 2021/04/08 15:15:22 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 15:15:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:24 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:38684->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:24 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:25 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.254.13:8301 2021/04/08 15:15:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:27 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:38730->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52806->10.242.254.13:8302: i/o timeout 2021/04/08 15:15:27 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:15:27 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:34 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:38854->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:34 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:37 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:38908->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:38 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:42 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.254.13:8302 2021/04/08 15:15:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:44 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:39028->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:45 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:48 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:39106->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:49 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:55 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:39216->10.242.254.13:8300: i/o timeout 2021/04/08 15:15:55 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:15:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:15:59 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:39304->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:00 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:16:00 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/04/08 15:16:00 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:03 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:16:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:05 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:39386->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:05 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-dk9sm 10.242.254.10:8301 2021/04/08 15:16:06 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:10 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:39480->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:12 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:16 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:39578->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:17 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:22 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:39682->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:24 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:27 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:39768->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:29 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:34 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:39884->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:38 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:39 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:39968->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:42 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:16:42 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/04/08 15:16:44 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:45 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:16:45 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:16:45 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-dk9sm 10.242.254.10:8301 2021/04/08 15:16:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:48 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:40138->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:53 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:16:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:40250->10.242.254.13:8300: i/o timeout 2021/04/08 15:16:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:16:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:03 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:40398->10.242.254.13:8300: i/o timeout 2021/04/08 15:17:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:05 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:17:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:14 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:17:15 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:40580->10.242.254.13:8300: i/o timeout 2021/04/08 15:17:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:24 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:17:24 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/04/08 15:17:24 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:40746->10.242.254.13:8300: i/o timeout 2021/04/08 15:17:25 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:17:25 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-dk9sm 10.242.254.10:8301 2021/04/08 15:17:27 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:17:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:35 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:40918->10.242.254.13:8300: i/o timeout 2021/04/08 15:17:35 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:17:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:42 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.254.13:8302 2021/04/08 15:17:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:45 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:17:45 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:41116->10.242.254.13:8300: i/o timeout 2021/04/08 15:17:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:55 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:41292->10.242.254.13:8300: i/o timeout 2021/04/08 15:17:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:17:56 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:17:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:05 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:18:06 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-dk9sm 10.242.254.10:8301 2021/04/08 15:18:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:06 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:18:06 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/04/08 15:18:06 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:41472->10.242.254.13:8300: i/o timeout 2021/04/08 15:18:09 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:18:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:15 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:41626->10.242.254.13:8300: i/o timeout 2021/04/08 15:18:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:18 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:18:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:26 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:18:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:28 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: read tcp 10.242.12.17:41836->10.242.254.13:8300: i/o timeout 2021/04/08 15:18:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:36 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:41962->10.242.254.13:8300: i/o timeout 2021/04/08 15:18:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:39 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:18:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:42 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.254.13:8302 2021/04/08 15:18:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:45 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:18:46 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.254.13:8301 2021/04/08 15:18:46 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:18:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:18:46 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.13 2021/04/08 15:18:46 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:18:46 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/04/08 15:18:46 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1.dc1) 2021/04/08 15:18:46 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.254.13 2021/04/08 15:18:46 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 15:18:47 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:18:47 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' joined, marking health alive 2021/04/08 15:18:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48770->10.242.254.10:8301: i/o timeout 2021/04/08 15:18:48 [INFO] memberlist: Suspect dev-consul-7467bf7b46-dk9sm has failed, no acks received 2021/04/08 15:18:49 [INFO] raft: pipelining replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:18:53 [INFO] serf: EventMemberLeave: dev-consul-7467bf7b46-dk9sm 10.242.254.10 2021/04/08 15:18:53 [INFO] consul: member 'dev-consul-7467bf7b46-dk9sm' left, deregistering 2021/04/08 15:18:53 [INFO] raft: aborting pipeline replication to peer {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300} 2021/04/08 15:18:53 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: EOF 2021/04/08 15:18:53 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: read tcp 10.242.12.17:42484->10.242.254.13:8300: read: connection reset by peer 2021/04/08 15:18:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:54 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:54 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: stream closed 2021/04/08 15:18:54 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:54 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:55 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:55 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:55 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:18:55 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:18:55 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:55 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:55 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 2.500303833s 2021/04/08 15:18:56 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:56 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:57 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:57 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:57 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:18:57 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:18:58 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:58 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 4.989762207s 2021/04/08 15:18:58 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:59 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: getsockopt: connection refused 2021/04/08 15:18:59 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:18:59 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:19:00 [WARN] raft: Failed to contact af5f140e-ee05-617b-6b81-fb4efa50aa1e in 7.440969406s 2021/04/08 15:19:01 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:19:01 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:19:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:07 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:19:07 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.254.13 2021/04/08 15:19:07 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.254.13:8300) (DC: dc1) 2021/04/08 15:19:07 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/04/08 15:19:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:09 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:10 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/04/08 15:19:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:10 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:10 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:12 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:19:13 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:19:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:14 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:19:15 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:19:15 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:19:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:22 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:22 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:24 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:25 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:19:25 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:19:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:34 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:35 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:19:35 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:19:35 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:37 [ERR] memberlist: Push/Pull with dev-consul-server-1.dc1 failed: dial tcp 10.242.254.13:8302: i/o timeout 2021/04/08 15:19:37 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:19:37 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:19:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:41 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:19:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:45 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:46 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:46 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.254.13:8301 2021/04/08 15:19:49 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:19:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:51 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:55 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:19:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/04/08 15:19:58 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:19:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.254.13:8302 Theirs: 10.242.254.59:8302 2021/04/08 15:19:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.254.13:8302 and 10.242.254.59:8302 are claiming 2021/04/08 15:19:59 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.59 2021/04/08 15:19:59 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.59:8300) (DC: dc1) 2021/04/08 15:19:59 [INFO] raft: Updating configuration with AddNonvoter (3454d0fb-1017-7917-b868-ce15c7ffda2e, 10.242.254.59:8300) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:c52c5261-fd75-2dd5-3bad-46a881682908 Address:10.242.139.247:8300} {Suffrage:Nonvoter ID:3454d0fb-1017-7917-b868-ce15c7ffda2e Address:10.242.254.59:8300}] 2021/04/08 15:19:59 [INFO] raft: Added peer 3454d0fb-1017-7917-b868-ce15c7ffda2e, starting replication 2021/04/08 15:19:59 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/04/08 15:19:59 [WARN] raft: AppendEntries to {Nonvoter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} rejected, sending older logs (next: 1) 2021/04/08 15:19:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.254.13:8302 Theirs: 10.242.254.59:8302 2021/04/08 15:19:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.254.13:8302 and 10.242.254.59:8302 are claiming 2021/04/08 15:19:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.254.13:8302 Theirs: 10.242.254.59:8302 2021/04/08 15:19:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.254.13:8302 and 10.242.254.59:8302 are claiming 2021/04/08 15:19:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.254.13:8302 Theirs: 10.242.254.59:8302 2021/04/08 15:19:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.254.13:8302 and 10.242.254.59:8302 are claiming 2021/04/08 15:19:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.254.13:8302 Theirs: 10.242.254.59:8302 2021/04/08 15:19:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.254.13:8302 and 10.242.254.59:8302 are claiming 2021/04/08 15:20:01 [INFO] raft: pipelining replication to peer {Nonvoter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:20:02 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:20:05 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:20:05 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:20:05 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.254.13 2021/04/08 15:20:05 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 15:20:08 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/04/08 15:20:12 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:20:15 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:20:16 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.254.13:8302 2021/04/08 15:20:20 [INFO] autopilot: Promoting Server (ID: "3454d0fb-1017-7917-b868-ce15c7ffda2e" Address: "10.242.254.59:8300") to voter 2021/04/08 15:20:20 [INFO] raft: Updating configuration with AddStaging (3454d0fb-1017-7917-b868-ce15c7ffda2e, 10.242.254.59:8300) to [{Suffrage:Voter ID:af5f140e-ee05-617b-6b81-fb4efa50aa1e Address:10.242.254.13:8300} {Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:c52c5261-fd75-2dd5-3bad-46a881682908 Address:10.242.139.247:8300} {Suffrage:Voter ID:3454d0fb-1017-7917-b868-ce15c7ffda2e Address:10.242.254.59:8300}] 2021/04/08 15:20:20 [INFO] autopilot: Attempting removal of stale Server (ID: "af5f140e-ee05-617b-6b81-fb4efa50aa1e" Address: "10.242.254.13:8300") 2021/04/08 15:20:20 [INFO] raft: Updating configuration with RemoveServer (af5f140e-ee05-617b-6b81-fb4efa50aa1e, ) to [{Suffrage:Voter ID:0e6d1abb-47a7-4121-d0d1-64118dab1223 Address:10.242.12.17:8300} {Suffrage:Voter ID:c52c5261-fd75-2dd5-3bad-46a881682908 Address:10.242.139.247:8300} {Suffrage:Voter ID:3454d0fb-1017-7917-b868-ce15c7ffda2e Address:10.242.254.59:8300}] 2021/04/08 15:20:20 [INFO] raft: Removed peer af5f140e-ee05-617b-6b81-fb4efa50aa1e, stopping replication after 1098 2021/04/08 15:20:23 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:20:25 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:20:33 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:20:35 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:20:44 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:20:45 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:20:54 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:20:56 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:21:05 [WARN] Unable to get address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e, using fallback address 10.242.254.13:8300: Could not find address for server id af5f140e-ee05-617b-6b81-fb4efa50aa1e 2021/04/08 15:21:06 [ERR] raft: Failed to AppendEntries to {Voter af5f140e-ee05-617b-6b81-fb4efa50aa1e 10.242.254.13:8300}: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:21:15 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:21:15 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:21:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51378->10.242.139.195:8301: i/o timeout 2021/04/08 15:21:15 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:21:15 [ERR] raft: Failed to heartbeat to 10.242.254.13:8300: dial tcp ->10.242.254.13:8300: i/o timeout 2021/04/08 15:21:16 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500276433s 2021/04/08 15:21:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:21:16 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.254.13:8302 2021/04/08 15:21:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51426->10.242.139.195:8301: i/o timeout 2021/04/08 15:21:17 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:21:19 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:21:19 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:21:19 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:21:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51468->10.242.139.195:8301: i/o timeout 2021/04/08 15:21:19 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:21:21 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:21:21 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:21:33 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.254.59 2021/04/08 15:21:33 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/04/08 15:22:21 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:22:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:25 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:22:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:26 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:35930->10.242.139.247:8301: i/o timeout 2021/04/08 15:22:26 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:22:27 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500316107s 2021/04/08 15:22:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:28 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:22:30 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.913314056s 2021/04/08 15:22:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:35986->10.242.139.247:8301: i/o timeout 2021/04/08 15:22:30 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:22:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39282->10.242.139.247:8302: i/o timeout 2021/04/08 15:22:32 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:22:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52666->10.242.139.195:8301: i/o timeout 2021/04/08 15:22:32 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:22:32 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.380100715s 2021/04/08 15:22:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:34 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36064->10.242.139.247:8301: i/o timeout 2021/04/08 15:22:34 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:22:35 [INFO] raft: aborting pipeline replication to peer {Nonvoter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:22:35 [ERR] raft: peer {Nonvoter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} has newer term, stopping replication 2021/04/08 15:22:35 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 15:22:35 [INFO] raft: aborting pipeline replication to peer {Nonvoter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:22:35 [INFO] consul: cluster leadership lost 2021/04/08 15:22:36 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2) 2021/04/08 15:22:36 [DEBUG] raft-net: 10.242.12.17:8300 accepted connection from: 10.242.139.247:38888 2021/04/08 15:22:36 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since our last index is greater (1147, 1144) 2021/04/08 15:22:42 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/04/08 15:22:42 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 14 2021/04/08 15:22:42 [INFO] raft: Election won. Tally: 2 2021/04/08 15:22:42 [INFO] raft: Node at 10.242.12.17:8300 [Leader] entering Leader state 2021/04/08 15:22:42 [INFO] consul: cluster leadership acquired 2021/04/08 15:22:42 [INFO] raft: Added peer c52c5261-fd75-2dd5-3bad-46a881682908, starting replication 2021/04/08 15:22:42 [INFO] raft: Added peer 3454d0fb-1017-7917-b868-ce15c7ffda2e, starting replication 2021/04/08 15:22:42 [INFO] consul: New leader elected: dev-consul-server-2 2021/04/08 15:22:42 [INFO] raft: pipelining replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:22:44 [WARN] raft: AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} rejected, sending older logs (next: 1145) 2021/04/08 15:22:45 [INFO] raft: pipelining replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:22:49 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:22:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53082->10.242.139.195:8301: i/o timeout 2021/04/08 15:22:53 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:22:55 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:22:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:22:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53096->10.242.139.195:8301: i/o timeout 2021/04/08 15:22:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36472->10.242.139.247:8301: i/o timeout 2021/04/08 15:22:57 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:22:57 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:22:57 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:22:57 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:22:57 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500224026s 2021/04/08 15:22:57 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:22:57 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:22:59 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:22:59 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 15:23:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53196->10.242.139.195:8301: i/o timeout 2021/04/08 15:23:00 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:23:01 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:23:01 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:23:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53252->10.242.139.195:8301: i/o timeout 2021/04/08 15:23:03 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:23:05 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:23:05 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:23:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53276->10.242.139.195:8301: i/o timeout 2021/04/08 15:23:06 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:23:17 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:23:17 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:23:21 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:23:21 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:23:22 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:23:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53590->10.242.139.195:8301: i/o timeout 2021/04/08 15:23:24 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:23:29 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53674->10.242.139.195:8301: i/o timeout 2021/04/08 15:23:29 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:24:09 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:24:09 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54344->10.242.139.195:8301: i/o timeout 2021/04/08 15:24:09 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:24:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54392->10.242.139.195:8301: i/o timeout 2021/04/08 15:24:12 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:24:15 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:24:15 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:24:15 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:24:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54424->10.242.139.195:8301: i/o timeout 2021/04/08 15:24:15 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:24:17 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:24:18 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500296485s 2021/04/08 15:24:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37830->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:18 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:20 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.997219886s 2021/04/08 15:24:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37884->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:21 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41180->10.242.139.247:8302: i/o timeout 2021/04/08 15:24:22 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:24:23 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.494613285s 2021/04/08 15:24:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37950->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:24 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:25 [INFO] raft: aborting pipeline replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:24:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:26 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37984->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:26 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:26 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:54540->10.242.139.247:8300: i/o timeout 2021/04/08 15:24:27 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500257368s 2021/04/08 15:24:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:28 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37998->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:28 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:29 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:24:29 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:24:29 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:24:29 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:24:30 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.968483225s 2021/04/08 15:24:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:30 [INFO] raft: pipelining replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:24:30 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:24:31 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:24:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:34 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500244763s 2021/04/08 15:24:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:34 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:24:34 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.195:49294 2021/04/08 15:24:35 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:24:35 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:24:35 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:24:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:36 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38156->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:36 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:39 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38238->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:39 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:40 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500249888s 2021/04/08 15:24:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38266->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:41 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:43 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.996296323s 2021/04/08 15:24:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38300->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:43 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:45 [ERR] raft: Failed to pipeline AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: write tcp 10.242.12.17:54718->10.242.139.247:8300: use of closed network connection 2021/04/08 15:24:45 [INFO] raft: aborting pipeline replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:24:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38324->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:45 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:47 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:24:47 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:24:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41622->10.242.139.247:8302: i/o timeout 2021/04/08 15:24:47 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:24:47 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:24:47 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:24:47 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:24:47 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:24:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38378->10.242.139.247:8301: i/o timeout 2021/04/08 15:24:48 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:24:49 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500481477s 2021/04/08 15:24:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:50 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:24:52 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500262083s 2021/04/08 15:24:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:53 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:24:53 [INFO] raft: pipelining replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:24:53 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:24:55 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:24:56 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500321996s 2021/04/08 15:24:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:56 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:24:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:41792->10.242.139.247:8302: i/o timeout 2021/04/08 15:24:57 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:24:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:24:58 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.960243083s 2021/04/08 15:24:58 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:24:59 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:01 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:02 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:02 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:02 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:25:02 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:25:03 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:03 [INFO] raft: aborting pipeline replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:25:03 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:04 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55300->10.242.139.195:8301: i/o timeout 2021/04/08 15:25:04 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:25:06 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500346126s 2021/04/08 15:25:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55334->10.242.139.195:8301: i/o timeout 2021/04/08 15:25:06 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:25:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:08 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.971498885s 2021/04/08 15:25:09 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55382->10.242.139.195:8301: i/o timeout 2021/04/08 15:25:09 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:25:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:10 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:25:10 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.247:59472 2021/04/08 15:25:10 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.439750098s 2021/04/08 15:25:11 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55416->10.242.139.195:8301: i/o timeout 2021/04/08 15:25:11 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:25:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42040->10.242.139.247:8302: i/o timeout 2021/04/08 15:25:12 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:25:13 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:56460->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:13 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:13 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:25:13 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:25:13 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:25:14 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:57102->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:14 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55466->10.242.139.195:8301: i/o timeout 2021/04/08 15:25:14 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:25:14 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:15 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:25:15 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:25:16 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:25:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:16 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55518->10.242.139.195:8301: i/o timeout 2021/04/08 15:25:16 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:25:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:23 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:57256->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:23 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:24 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:57280->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:33 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:57428->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:33 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:35 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:25:35 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:25:35 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:36 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:25:37 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500379661s 2021/04/08 15:25:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:39 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.974074454s 2021/04/08 15:25:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:42 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.454435046s 2021/04/08 15:25:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:43 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:57584->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:43 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:45 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:57458->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:46 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42740->10.242.139.247:8302: i/o timeout 2021/04/08 15:25:52 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:25:53 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:57778->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:53 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:56 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:57826->10.242.139.247:8300: i/o timeout 2021/04/08 15:25:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:25:56 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:25:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:42812->10.242.139.247:8302: i/o timeout 2021/04/08 15:26:02 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:26:03 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:57948->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:04 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:06 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:57996->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:07 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43054->10.242.139.247:8302: i/o timeout 2021/04/08 15:26:12 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:26:14 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:58118->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:14 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:14 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:26:14 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:26:16 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:26:17 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:26:17 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:58166->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:18 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:22 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:26:22 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:26:22 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:26:24 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:58282->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43308->10.242.139.247:8302: i/o timeout 2021/04/08 15:26:27 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:26:27 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:26:28 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:58360->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:29 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:29 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:26:29 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:26:29 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:26:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:40096->10.242.139.247:8301: i/o timeout 2021/04/08 15:26:30 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:26:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:40138->10.242.139.247:8301: i/o timeout 2021/04/08 15:26:32 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:26:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:34 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:26:34 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:40152->10.242.139.247:8301: i/o timeout 2021/04/08 15:26:34 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:26:34 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:26:34 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:26:34 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:26:35 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:58462->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:36 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:39 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:58536->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:40 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:46 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:58658->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:46 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:26:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:48 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:50 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:58744->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:51 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:26:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:26:56 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:26:56 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:26:56 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:26:58 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8302->10.242.139.247:44072: i/o timeout from=10.242.139.247:44072 2021/04/08 15:26:58 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8302->10.242.139.247:44072: i/o timeout from=10.242.139.247:44072 2021/04/08 15:26:58 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:58900->10.242.139.247:8300: i/o timeout 2021/04/08 15:26:59 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:26:59 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:27:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:01 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:58938->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:02 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:04 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:12 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:59124->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:14 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:59144->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:14 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:24 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:24 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:59312->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:28 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:34 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:59470->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:36 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:27:38 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:59532->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:38 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:27:38 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:27:41 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:27:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:43 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:44 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:27:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:46 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:27:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:53 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:59810->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:54 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:59824->10.242.139.247:8300: i/o timeout 2021/04/08 15:27:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:27:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:04 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:28:04 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:28:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:14 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:60156->10.242.139.247:8300: i/o timeout 2021/04/08 15:28:14 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:60158->10.242.139.247:8300: i/o timeout 2021/04/08 15:28:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:16 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:28:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:20 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:28:20 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:28:23 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:28:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:28:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:28:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:35 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:60484->10.242.139.247:8300: i/o timeout 2021/04/08 15:28:35 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:60500->10.242.139.247:8300: i/o timeout 2021/04/08 15:28:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:45 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:28:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:46 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:28:47 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:28:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:55 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:60826->10.242.139.247:8300: i/o timeout 2021/04/08 15:28:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:28:56 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:28:57 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:60872->10.242.139.247:8300: i/o timeout 2021/04/08 15:28:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:02 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:29:02 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:29:05 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:29:05 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:29:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:07 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:29:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:15 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:32940->10.242.139.247:8300: i/o timeout 2021/04/08 15:29:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:17 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:32980->10.242.139.247:8300: i/o timeout 2021/04/08 15:29:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:29:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:28 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:29:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:35 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:29:35 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:29:35 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:29:35 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:29:35 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:29:35 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:33270->10.242.139.247:8300: i/o timeout 2021/04/08 15:29:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43242->10.242.139.247:8301: i/o timeout 2021/04/08 15:29:37 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:29:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:38 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:33316->10.242.139.247:8300: i/o timeout 2021/04/08 15:29:39 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:29:39 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:29:39 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:29:39 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:29:39 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59920->10.242.139.195:8301: i/o timeout 2021/04/08 15:29:39 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:29:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59988->10.242.139.195:8301: i/o timeout 2021/04/08 15:29:41 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:29:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60024->10.242.139.195:8301: i/o timeout 2021/04/08 15:29:43 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:29:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:44 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:29:44 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:29:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60048->10.242.139.195:8301: i/o timeout 2021/04/08 15:29:45 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:29:45 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:29:46 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:29:47 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:29:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60120->10.242.139.195:8301: i/o timeout 2021/04/08 15:29:48 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:29:49 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:29:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60140->10.242.139.195:8301: i/o timeout 2021/04/08 15:29:50 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:29:51 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:29:51 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:29:51 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:29:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:55 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:33652->10.242.139.247:8300: i/o timeout 2021/04/08 15:29:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:29:59 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:33716->10.242.139.247:8300: i/o timeout 2021/04/08 15:30:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:03 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:30:03 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:30:03 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:30:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43730->10.242.139.247:8301: i/o timeout 2021/04/08 15:30:04 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:30:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43774->10.242.139.247:8301: i/o timeout 2021/04/08 15:30:07 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:30:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:09 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43810->10.242.139.247:8301: i/o timeout 2021/04/08 15:30:09 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:30:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:11 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43848->10.242.139.247:8301: i/o timeout 2021/04/08 15:30:11 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:30:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43868->10.242.139.247:8301: i/o timeout 2021/04/08 15:30:13 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:30:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:43912->10.242.139.247:8301: i/o timeout 2021/04/08 15:30:15 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:30:15 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:30:15 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:30:15 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:30:15 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:30:16 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:33980->10.242.139.247:8300: i/o timeout 2021/04/08 15:30:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:20 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:34050->10.242.139.247:8300: i/o timeout 2021/04/08 15:30:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:26 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:30:26 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:30:26 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:30:29 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:30:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:30 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.254.59 2021/04/08 15:30:30 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:30:30 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.254.59:8300) (DC: dc1) 2021/04/08 15:30:30 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/04/08 15:30:30 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:30:30 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.254.59 2021/04/08 15:30:30 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.254.59:8300) (DC: dc1) 2021/04/08 15:30:30 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/04/08 15:30:31 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:30:31 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:30:31 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:30:31 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:30:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:34 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60880->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:34 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:36 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:34324->10.242.139.247:8300: i/o timeout 2021/04/08 15:30:36 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60892->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:36 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47538->10.242.139.247:8302: i/o timeout 2021/04/08 15:30:37 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:30:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:38 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:60972->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:38 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:40 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:32784->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:40 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:41 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:34418->10.242.139.247:8300: i/o timeout 2021/04/08 15:30:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:32824->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:42 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:32868->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:45 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:45 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:30:45 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:30:45 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:30:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:46 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:30:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:32908->10.242.139.195:8301: i/o timeout 2021/04/08 15:30:47 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:30:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:49 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8301->10.242.139.247:59236: i/o timeout from=10.242.139.247:59236 2021/04/08 15:30:49 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8301->10.242.139.247:59236: i/o timeout from=10.242.139.247:59236 2021/04/08 15:30:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47846->10.242.139.247:8302: i/o timeout 2021/04/08 15:30:52 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:30:52 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:30:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:30:56 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:34712->10.242.139.247:8300: i/o timeout 2021/04/08 15:30:57 [ERR] consul.rpc: multiplex conn accept failed: read tcp 10.242.12.17:8300->10.242.139.247:42734: read: connection reset by peer from=10.242.139.247:42734 2021/04/08 15:30:57 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:30:57 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:30:57 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:30:57 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:30:58 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:30:59 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:31:00 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500294854s 2021/04/08 15:31:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44706->10.242.139.247:8301: i/o timeout 2021/04/08 15:31:00 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:31:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:02 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.939720526s 2021/04/08 15:31:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44778->10.242.139.247:8301: i/o timeout 2021/04/08 15:31:04 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:31:05 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.358013322s 2021/04/08 15:31:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44812->10.242.139.247:8301: i/o timeout 2021/04/08 15:31:06 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:31:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48006->10.242.139.247:8302: i/o timeout 2021/04/08 15:31:07 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:31:07 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:31:07 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:31:07 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:31:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44844->10.242.139.247:8301: i/o timeout 2021/04/08 15:31:08 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:31:08 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:34814->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44872->10.242.139.247:8301: i/o timeout 2021/04/08 15:31:10 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:31:11 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:31:11 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:31:11 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:31:11 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:31:11 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:31:11 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:31:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33348->10.242.139.195:8301: i/o timeout 2021/04/08 15:31:13 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:31:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33376->10.242.139.195:8301: i/o timeout 2021/04/08 15:31:15 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:31:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:16 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:35038->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:19 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:35084->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33460->10.242.139.195:8301: i/o timeout 2021/04/08 15:31:19 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:31:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:20 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33468->10.242.139.195:8301: i/o timeout 2021/04/08 15:31:21 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:31:22 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:23 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:23 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:31:24 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:33538->10.242.139.195:8301: i/o timeout 2021/04/08 15:31:24 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:31:24 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:24 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:31:24 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:31:24 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:31:25 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:31:25 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:31:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:27 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:27 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500276911s 2021/04/08 15:31:27 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:31:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:29 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.965013516s 2021/04/08 15:31:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:32 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.428726923s 2021/04/08 15:31:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48508->10.242.139.247:8302: i/o timeout 2021/04/08 15:31:32 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:31:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:35 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:35276->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:36 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:36 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:31:37 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:35388->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:41 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8301->10.242.139.247:59658: i/o timeout from=10.242.139.247:59658 2021/04/08 15:31:41 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8301->10.242.139.247:59658: i/o timeout from=10.242.139.247:59658 2021/04/08 15:31:41 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:31:41 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:31:43 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:31:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:46 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:35528->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:47 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:47 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48786->10.242.139.247:8302: i/o timeout 2021/04/08 15:31:47 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:31:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:57 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:35754->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:57 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:35762->10.242.139.247:8300: i/o timeout 2021/04/08 15:31:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48956->10.242.139.247:8302: i/o timeout 2021/04/08 15:31:57 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:31:57 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:31:58 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:31:58 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:31:58 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:31:58 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:31:58 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:31:58 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:31:58 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:32:00 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 15:32:01 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:34208->10.242.139.195:8301: i/o timeout 2021/04/08 15:32:01 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:32:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:34236->10.242.139.195:8301: i/o timeout 2021/04/08 15:32:04 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:32:07 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:32:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:34308->10.242.139.195:8301: i/o timeout 2021/04/08 15:32:08 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:32:10 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500404112s 2021/04/08 15:32:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:11 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45910->10.242.139.247:8301: i/o timeout 2021/04/08 15:32:11 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:32:12 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:32:12 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:32:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:12 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.950120094s 2021/04/08 15:32:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:14 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45980->10.242.139.247:8301: i/o timeout 2021/04/08 15:32:14 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:32:14 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.395184631s 2021/04/08 15:32:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:16 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:46008->10.242.139.247:8301: i/o timeout 2021/04/08 15:32:16 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:32:16 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:32:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49300->10.242.139.247:8302: i/o timeout 2021/04/08 15:32:17 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:32:17 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:35938->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:18 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:36110->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:46076->10.242.139.247:8301: i/o timeout 2021/04/08 15:32:18 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:32:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:46124->10.242.139.247:8301: i/o timeout 2021/04/08 15:32:21 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:32:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:22 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:32:22 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:32:22 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:32:22 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:32:22 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:32:22 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:32:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:46146->10.242.139.247:8301: i/o timeout 2021/04/08 15:32:23 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:32:25 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:32:25 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:32:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:27 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:32:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:28 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:36308->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:29 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:32:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49558->10.242.139.247:8302: i/o timeout 2021/04/08 15:32:32 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:32:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:37 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:32:37 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:36452->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:39 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:36478->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:40 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:32:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49634->10.242.139.247:8302: i/o timeout 2021/04/08 15:32:42 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:32:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:45 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:32:45 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.195:57640 2021/04/08 15:32:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:47 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:32:47 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:32:47 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:32:48 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:32:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:50 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:36682->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:51 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:32:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:32:56 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:32:58 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:36818->10.242.139.247:8300: i/o timeout 2021/04/08 15:32:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:01 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:36880->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:02 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:02 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:04 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:33:04 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:33:07 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:33:08 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:08 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:10 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:12 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:37050->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:12 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:13 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:14 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:16 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:18 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:37136->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:18 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:23 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:37218->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:24 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:25 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8301->10.242.139.247:60234: i/o timeout from=10.242.139.247:60234 2021/04/08 15:33:25 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8301->10.242.139.247:60234: i/o timeout from=10.242.139.247:60234 2021/04/08 15:33:26 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:28 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:28 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:30 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:32 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:34 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:37400->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:34 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:35 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:36 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:36 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:33:38 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:38 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:37464->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:40 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:41 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:33:41 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:33:41 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:33:41 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:33:41 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2) 2021/04/08 15:33:41 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:33:41 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:33:42 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:43 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500270447s 2021/04/08 15:33:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47520->10.242.139.247:8301: i/o timeout 2021/04/08 15:33:43 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:33:44 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47544->10.242.139.247:8301: i/o timeout 2021/04/08 15:33:45 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:33:46 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.949819997s 2021/04/08 15:33:46 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50838->10.242.139.247:8302: i/o timeout 2021/04/08 15:33:47 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:33:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47590->10.242.139.247:8301: i/o timeout 2021/04/08 15:33:47 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:33:47 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:33:47 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:33:47 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:33:47 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:33:48 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:48 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.415595319s 2021/04/08 15:33:48 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:50 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:51 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:37580->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:52 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:52 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:54 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:56 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:58 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:58 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:33:58 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:37838->10.242.139.247:8300: i/o timeout 2021/04/08 15:33:59 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:33:59 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:34:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:00 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:34:01 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:34:01 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:34:01 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:34:01 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:34:01 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:34:02 [WARN] memberlist: Was able to connect to dev-consul-server-0.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:34:03 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:34:03 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:34:03 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:34:03 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:34:04 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47866->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:04 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:05 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500323035s 2021/04/08 15:34:06 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36294->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:07 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:09 [ERR] raft: peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} has newer term, stopping replication 2021/04/08 15:34:09 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 15:34:09 [INFO] consul: cluster leadership lost 2021/04/08 15:34:09 [INFO] raft: aborting pipeline replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:34:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36362->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:10 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36426->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:13 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48038->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:15 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:15 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/04/08 15:34:15 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 15 2021/04/08 15:34:16 [INFO] raft: Election won. Tally: 2 2021/04/08 15:34:16 [INFO] raft: Node at 10.242.12.17:8300 [Leader] entering Leader state 2021/04/08 15:34:16 [INFO] raft: Added peer c52c5261-fd75-2dd5-3bad-46a881682908, starting replication 2021/04/08 15:34:16 [INFO] raft: Added peer 3454d0fb-1017-7917-b868-ce15c7ffda2e, starting replication 2021/04/08 15:34:16 [INFO] consul: cluster leadership acquired 2021/04/08 15:34:16 [INFO] consul: New leader elected: dev-consul-server-2 2021/04/08 15:34:16 [INFO] raft: pipelining replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:34:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51334->10.242.139.247:8302: i/o timeout 2021/04/08 15:34:17 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:34:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:36508->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:17 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:18 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:34:18 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:34:18 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.502774984s 2021/04/08 15:34:19 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:34:19 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:34:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48112->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:19 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:20 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:20 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.93119267s 2021/04/08 15:34:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48158->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:21 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:22 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:23 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.403263465s 2021/04/08 15:34:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48194->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:23 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:24 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:26 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:38294->10.242.139.247:8300: i/o timeout 2021/04/08 15:34:26 [ERR] raft: peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} has newer term, stopping replication 2021/04/08 15:34:26 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 15:34:26 [INFO] raft: aborting pipeline replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:34:26 [INFO] consul: cluster leadership lost 2021/04/08 15:34:26 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48242->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:26 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:26 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:38302->10.242.139.247:8300: i/o timeout 2021/04/08 15:34:26 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:34:26 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:34:26 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:34:27 [WARN] memberlist: Was able to connect to dev-consul-server-0.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:34:27 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since our last term is greater (15, 14) 2021/04/08 15:34:27 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:34:27 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:34:27 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:34:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48290->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:30 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:32 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/04/08 15:34:32 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 19 2021/04/08 15:34:33 [INFO] raft: Election won. Tally: 2 2021/04/08 15:34:33 [INFO] raft: Node at 10.242.12.17:8300 [Leader] entering Leader state 2021/04/08 15:34:33 [INFO] raft: Added peer c52c5261-fd75-2dd5-3bad-46a881682908, starting replication 2021/04/08 15:34:33 [INFO] raft: Added peer 3454d0fb-1017-7917-b868-ce15c7ffda2e, starting replication 2021/04/08 15:34:33 [INFO] consul: cluster leadership acquired 2021/04/08 15:34:33 [INFO] consul: New leader elected: dev-consul-server-2 2021/04/08 15:34:33 [INFO] raft: pipelining replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:34:33 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48356->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:33 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:33 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:34:35 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48376->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:35 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:35 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.502436263s 2021/04/08 15:34:36 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:34:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:38 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.984109538s 2021/04/08 15:34:38 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48456->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:38 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:40 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.431650844s 2021/04/08 15:34:40 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48478->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:40 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:41 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:34:43 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:38574->10.242.139.247:8300: i/o timeout 2021/04/08 15:34:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48566->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:43 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:43 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:38586->10.242.139.247:8300: i/o timeout 2021/04/08 15:34:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48592->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:45 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:47 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:34:47 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:34:47 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:34:47 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:34:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48636->10.242.139.247:8301: i/o timeout 2021/04/08 15:34:47 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:34:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:50 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:34:50 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:34:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:51 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37124->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:51 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:51 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:34:51 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:34:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51982->10.242.139.247:8302: i/o timeout 2021/04/08 15:34:52 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:34:53 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:37904->10.242.139.247:8300: i/o timeout 2021/04/08 15:34:53 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:34:54 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:34:54 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:38800->10.242.139.247:8300: i/o timeout 2021/04/08 15:34:54 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37178->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:54 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:55 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:34:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37192->10.242.139.195:8301: i/o timeout 2021/04/08 15:34:56 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:34:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:34:57 [WARN] raft: AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} rejected, sending older logs (next: 1318) 2021/04/08 15:34:57 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:34:57 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:34:57 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:34:57 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:34:58 [INFO] raft: pipelining replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:35:08 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:35:08 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500461056s 2021/04/08 15:35:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49014->10.242.139.247:8301: i/o timeout 2021/04/08 15:35:10 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:35:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:11 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.987474188s 2021/04/08 15:35:18 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:35:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37592->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:18 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37622->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49236->10.242.139.247:8301: i/o timeout 2021/04/08 15:35:22 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:35:22 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:35:22 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:35:22 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:35:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37680->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:23 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:23 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:35:23 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:35:26 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:35:26 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:35:26 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37708->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:26 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:30 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500252063s 2021/04/08 15:35:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49372->10.242.139.247:8301: i/o timeout 2021/04/08 15:35:30 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:35:30 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:35:30 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:35:30 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:35:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:37820->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:32 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:32 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.989703676s 2021/04/08 15:35:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:34 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-w5szm) 2021/04/08 15:35:35 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:35:35 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:35:40 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:35:40 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:35:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38008->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:41 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:44 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:35:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:47 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500350145s 2021/04/08 15:35:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38138->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:48 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38208->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:52 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:52 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:35:52 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:35:52 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:35:54 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:35:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:55 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38224->10.242.139.195:8301: i/o timeout 2021/04/08 15:35:55 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:35:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:35:58 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500163939s 2021/04/08 15:35:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49870->10.242.139.247:8301: i/o timeout 2021/04/08 15:35:58 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:35:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:00 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.985152929s 2021/04/08 15:36:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49920->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:00 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:01 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:36:02 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:02 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:36:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49954->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:02 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:05 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50010->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:05 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:06 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:36:06 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:36:06 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1) 2021/04/08 15:36:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53308->10.242.139.247:8302: i/o timeout 2021/04/08 15:36:07 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:36:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50048->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:07 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:08 [ERR] raft: peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} has newer term, stopping replication 2021/04/08 15:36:08 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 15:36:08 [INFO] raft: aborting pipeline replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:36:08 [INFO] raft: aborting pipeline replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:36:08 [INFO] consul: cluster leadership lost 2021/04/08 15:36:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50118->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:12 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:15 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/04/08 15:36:15 [INFO] raft: Node at 10.242.12.17:8300 [Candidate] entering Candidate state in term 20 2021/04/08 15:36:15 [INFO] raft: Election won. Tally: 2 2021/04/08 15:36:15 [INFO] raft: Node at 10.242.12.17:8300 [Leader] entering Leader state 2021/04/08 15:36:15 [INFO] raft: Added peer c52c5261-fd75-2dd5-3bad-46a881682908, starting replication 2021/04/08 15:36:15 [INFO] raft: Added peer 3454d0fb-1017-7917-b868-ce15c7ffda2e, starting replication 2021/04/08 15:36:15 [INFO] consul: cluster leadership acquired 2021/04/08 15:36:15 [INFO] consul: New leader elected: dev-consul-server-2 2021/04/08 15:36:15 [INFO] raft: pipelining replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:36:15 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:36:17 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:36:18 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:19 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500243563s 2021/04/08 15:36:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50282->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:19 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:21 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.94638538s 2021/04/08 15:36:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50294->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:21 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:21 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:36:21 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:36:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53590->10.242.139.247:8302: i/o timeout 2021/04/08 15:36:22 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:36:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:38752->10.242.139.195:8301: i/o timeout 2021/04/08 15:36:23 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:36:23 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.433639389s 2021/04/08 15:36:23 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:36:23 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:36:23 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:36:23 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:36:25 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:38996->10.242.139.247:8300: i/o timeout 2021/04/08 15:36:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:36:26 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:27 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:40452->10.242.139.247:8300: i/o timeout 2021/04/08 15:36:27 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:36:27 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:36:27 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:36:28 [WARN] raft: AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} rejected, sending older logs (next: 1463) 2021/04/08 15:36:30 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 15:36:30 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:30 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:36:30 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:36:31 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50506->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:31 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:32 [WARN] memberlist: Was able to connect to dev-consul-server-0.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:36:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:34 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50548->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:34 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:34 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500275317s 2021/04/08 15:36:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:36 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50582->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:36 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:37 [INFO] raft: pipelining replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:36:38 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:36:38 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:36:38 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-w5szm) 2021/04/08 15:36:40 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:40 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:36:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39140->10.242.139.195:8301: i/o timeout 2021/04/08 15:36:42 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:36:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:45 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.50022894s 2021/04/08 15:36:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39160->10.242.139.195:8301: i/o timeout 2021/04/08 15:36:45 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:36:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:47 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.932732984s 2021/04/08 15:36:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:50826->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:49 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:50 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54178->10.242.139.247:8302: i/o timeout 2021/04/08 15:36:52 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:36:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39320->10.242.139.195:8301: i/o timeout 2021/04/08 15:36:52 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:36:53 [INFO] raft: aborting pipeline replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:36:54 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 15:36:54 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:36:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:55 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.501348686s 2021/04/08 15:36:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39412->10.242.139.195:8301: i/o timeout 2021/04/08 15:36:56 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:36:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:36:58 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 15:36:59 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51060->10.242.139.247:8301: i/o timeout 2021/04/08 15:36:59 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:36:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:00 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:37:00 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:37:00 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:37:00 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:37:01 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39514->10.242.139.195:8301: i/o timeout 2021/04/08 15:37:01 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:37:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:02 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500319241s 2021/04/08 15:37:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54362->10.242.139.247:8302: i/o timeout 2021/04/08 15:37:02 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:37:03 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:41154->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:03 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:03 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8301->10.242.139.195:55302: i/o timeout from=10.242.139.195:55302 2021/04/08 15:37:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:39532->10.242.139.195:8301: i/o timeout 2021/04/08 15:37:03 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8301->10.242.139.195:55302: i/o timeout from=10.242.139.195:55302 2021/04/08 15:37:03 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:37:03 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:03 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:37:03 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:37:03 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:37:04 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.917093994s 2021/04/08 15:37:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:07 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.391428899s 2021/04/08 15:37:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:10 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:40654->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:11 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:13 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:41350->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:13 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:13 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:14 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:14 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:37:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:16 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:17 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:37:17 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.195:57988 2021/04/08 15:37:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:18 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500228161s 2021/04/08 15:37:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:19 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:37:20 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.92190884s 2021/04/08 15:37:20 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:37:21 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:23 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.41082981s 2021/04/08 15:37:23 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:41514->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:23 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:24 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:26 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:27 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54804->10.242.139.247:8302: i/o timeout 2021/04/08 15:37:27 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:37:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:28 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:29 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:30 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:32 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.50025473s 2021/04/08 15:37:33 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:41694->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:33 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:34 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.973857757s 2021/04/08 15:37:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:37 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.404143992s 2021/04/08 15:37:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54878->10.242.139.247:8302: i/o timeout 2021/04/08 15:37:37 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:37:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:38 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:37:38 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:37:40 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:37:40 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:41484->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:41 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:42 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:37:42 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:37:42 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:37:43 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:41858->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51848->10.242.139.247:8301: i/o timeout 2021/04/08 15:37:43 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:37:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51878->10.242.139.247:8301: i/o timeout 2021/04/08 15:37:45 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:37:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:46 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:37:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51924->10.242.139.247:8301: i/o timeout 2021/04/08 15:37:47 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:37:47 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:37:47 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:37:47 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:37:47 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:37:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:51 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:42036->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:52 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:53 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:42068->10.242.139.247:8300: i/o timeout 2021/04/08 15:37:53 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:37:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:37:53 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:37:53 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:37:53 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/04/08 15:37:54 [INFO] raft: pipelining replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:37:56 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:37:56 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/04/08 15:37:56 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/04/08 15:37:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:38:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:02 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:38:02 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' joined, marking health alive 2021/04/08 15:38:03 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:38:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52200->10.242.139.247:8301: i/o timeout 2021/04/08 15:38:04 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:38:04 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:38:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:40632->10.242.139.195:8301: i/o timeout 2021/04/08 15:38:07 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:38:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52268->10.242.139.247:8301: i/o timeout 2021/04/08 15:38:10 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:38:10 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:38:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52332->10.242.139.247:8301: i/o timeout 2021/04/08 15:38:13 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:38:13 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:40780->10.242.139.195:8301: i/o timeout 2021/04/08 15:38:15 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:38:15 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:16 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.50344697s 2021/04/08 15:38:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55672->10.242.139.247:8302: i/o timeout 2021/04/08 15:38:17 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:38:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52428->10.242.139.247:8301: i/o timeout 2021/04/08 15:38:17 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:38:17 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:18 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (1 peer confirmations) 2021/04/08 15:38:18 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:38:18 [INFO] consul: member 'dev-consul-7467bf7b46-w5szm' failed, marking health critical 2021/04/08 15:38:19 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 5.003235273s 2021/04/08 15:38:19 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:38:19 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:38:19 [INFO] consul: member 'dev-consul-server-0' failed, marking health critical 2021/04/08 15:38:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:40864->10.242.139.195:8301: i/o timeout 2021/04/08 15:38:19 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:38:19 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:19 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:38:19 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:38:21 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.496021276s 2021/04/08 15:38:22 [INFO] raft: aborting pipeline replication to peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} 2021/04/08 15:38:22 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:22 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:38:23 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:23 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:38:23 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.247:44174 2021/04/08 15:38:24 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500256443s 2021/04/08 15:38:24 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:42252->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:25 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:25 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:26 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.955627027s 2021/04/08 15:38:27 [WARN] memberlist: Was able to connect to dev-consul-server-0.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:38:27 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:29 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 7.412477698s 2021/04/08 15:38:29 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:30 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:38:31 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:32 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:42726->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:32 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:33 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:35 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:42780->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:35 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:36 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:56006->10.242.139.247:8302: i/o timeout 2021/04/08 15:38:37 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:38:37 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:37 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8301->10.242.139.195:56012: i/o timeout from=10.242.139.195:56012 2021/04/08 15:38:37 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8301->10.242.139.195:56012: i/o timeout from=10.242.139.195:56012 2021/04/08 15:38:39 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:41 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:42 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:42892->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:42 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:43 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:43 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:38:43 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:38:43 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:38:45 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:46 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:42952->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:47 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:38:47 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:47 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:49 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:51 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:52 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:43088->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:52 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:53 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:55 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:57 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8302->10.242.139.247:49570: i/o timeout from=10.242.139.247:49570 2021/04/08 15:38:57 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8302->10.242.139.247:49570: i/o timeout from=10.242.139.247:49570 2021/04/08 15:38:57 [ERR] raft: Failed to heartbeat to 10.242.139.247:8300: read tcp 10.242.12.17:43184->10.242.139.247:8300: i/o timeout 2021/04/08 15:38:57 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:38:58 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:38:59 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:39:01 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:39:01 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:39:01 [WARN] consul: error getting server health from "dev-consul-server-0": rpc error making call: EOF 2021/04/08 15:39:02 [ERR] raft: Failed to AppendEntries to {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300}: read tcp 10.242.12.17:43264->10.242.139.247:8300: i/o timeout 2021/04/08 15:39:02 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:39:04 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/04/08 15:39:05 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:39:06 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:39:06 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:39:07 [WARN] Unable to get address for server id c52c5261-fd75-2dd5-3bad-46a881682908, using fallback address 10.242.139.247:8300: Could not find address for server id c52c5261-fd75-2dd5-3bad-46a881682908 2021/04/08 15:39:07 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:39:09 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 2.500189347s 2021/04/08 15:39:09 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:39:10 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:39:11 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/04/08 15:39:11 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.12.17:8300 2021/04/08 15:39:11 [WARN] raft: Failed to contact c52c5261-fd75-2dd5-3bad-46a881682908 in 4.997360975s 2021/04/08 15:39:11 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:39:11 [ERR] raft: peer {Voter c52c5261-fd75-2dd5-3bad-46a881682908 10.242.139.247:8300} has newer term, stopping replication 2021/04/08 15:39:11 [INFO] raft: Node at 10.242.12.17:8300 [Follower] entering Follower state (Leader: "") 2021/04/08 15:39:11 [INFO] consul: cluster leadership lost 2021/04/08 15:39:11 [INFO] raft: aborting pipeline replication to peer {Voter 3454d0fb-1017-7917-b868-ce15c7ffda2e 10.242.254.59:8300} 2021/04/08 15:39:12 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:39:12 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:39:12 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:39:19 [ERR] http: Request GET /v1/catalog/service/config_binding_service, error: No cluster leader from=10.242.17.5:34858 2021/04/08 15:39:20 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53504->10.242.139.247:8301: i/o timeout 2021/04/08 15:39:20 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:39:20 [DEBUG] raft-net: 10.242.12.17:8300 accepted connection from: 10.242.254.59:48924 2021/04/08 15:39:20 [INFO] consul: New leader elected: dev-consul-server-1 2021/04/08 15:39:21 [DEBUG] raft-net: 10.242.12.17:8300 accepted connection from: 10.242.254.59:48932 2021/04/08 15:39:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53542->10.242.139.247:8301: i/o timeout 2021/04/08 15:39:22 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:39:24 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:39:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53570->10.242.139.247:8301: i/o timeout 2021/04/08 15:39:24 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:39:24 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:39:24 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:39:28 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.254.59:8300 2021/04/08 15:39:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:57034->10.242.139.247:8302: i/o timeout 2021/04/08 15:39:37 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:39:42 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:39:43 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.254.59:8300 2021/04/08 15:39:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:57330->10.242.139.247:8302: i/o timeout 2021/04/08 15:39:52 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:39:59 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:39:59 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:40:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:57486->10.242.139.247:8302: i/o timeout 2021/04/08 15:40:02 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:40:03 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54246->10.242.139.247:8301: i/o timeout 2021/04/08 15:40:03 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:40:05 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54270->10.242.139.247:8301: i/o timeout 2021/04/08 15:40:05 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:40:07 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:40:07 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:40:07 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:40:07 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:40:07 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:40:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:54308->10.242.139.247:8301: i/o timeout 2021/04/08 15:40:08 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:40:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:57572->10.242.139.247:8302: i/o timeout 2021/04/08 15:40:12 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:40:19 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.254.59:8300 2021/04/08 15:40:22 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:40:22 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:40:42 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:40:48 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:40:48 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:40:51 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55062->10.242.139.247:8301: i/o timeout 2021/04/08 15:40:51 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:40:53 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:40:53 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:40:53 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:40:54 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:55106->10.242.139.247:8301: i/o timeout 2021/04/08 15:40:54 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:40:55 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.254.59:8300 2021/04/08 15:40:55 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:40:55 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:40:55 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:40:57 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:40:57 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.247:47426 2021/04/08 15:41:02 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8301->10.242.139.247:36200: i/o timeout from=10.242.139.247:36200 2021/04/08 15:41:02 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8301->10.242.139.247:36200: i/o timeout from=10.242.139.247:36200 2021/04/08 15:41:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:58494->10.242.139.247:8302: i/o timeout 2021/04/08 15:41:07 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:41:17 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:41:17 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.195:33724 2021/04/08 15:41:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:58722->10.242.139.247:8302: i/o timeout 2021/04/08 15:41:17 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:41:23 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:41:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:58970->10.242.139.247:8302: i/o timeout 2021/04/08 15:41:32 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:41:33 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:41:33 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:41:33 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:41:48 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:41:48 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:41:48 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.247 2021/04/08 15:41:48 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:41:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:56034->10.242.139.247:8301: i/o timeout 2021/04/08 15:41:50 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:41:53 [WARN] memberlist: Was able to connect to dev-consul-server-0 but other probes failed, network may be misconfigured 2021/04/08 15:41:53 [INFO] memberlist: Marking dev-consul-server-0 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:41:53 [INFO] serf: EventMemberFailed: dev-consul-server-0 10.242.139.247 2021/04/08 15:41:53 [INFO] consul: Removing LAN server dev-consul-server-0 (Addr: tcp/10.242.139.247:8300) (DC: dc1) 2021/04/08 15:42:03 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:42:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59562->10.242.139.247:8302: i/o timeout 2021/04/08 15:42:07 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:42:10 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:42:11 [WARN] raft: Rejecting vote request from 10.242.139.247:8300 since we have a leader: 10.242.254.59:8300 2021/04/08 15:42:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44816->10.242.139.195:8301: i/o timeout 2021/04/08 15:42:13 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:42:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44844->10.242.139.195:8301: i/o timeout 2021/04/08 15:42:15 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:42:17 [WARN] memberlist: Was able to connect to dev-consul-server-0.dc1 but other probes failed, network may be misconfigured 2021/04/08 15:42:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44894->10.242.139.195:8301: i/o timeout 2021/04/08 15:42:17 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:42:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44914->10.242.139.195:8301: i/o timeout 2021/04/08 15:42:19 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:42:20 [ERR] memberlist: Push/Pull with dev-consul-server-0.dc1 failed: read tcp 10.242.12.17:59324->10.242.139.247:8302: i/o timeout 2021/04/08 15:42:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:44960->10.242.139.195:8301: i/o timeout 2021/04/08 15:42:21 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:42:24 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.195 2021/04/08 15:42:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:45004->10.242.139.195:8301: i/o timeout 2021/04/08 15:42:24 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:42:25 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:42:28 [ERR] yamux: keepalive failed: i/o deadline reached 2021/04/08 15:42:28 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.139.247:48728 2021/04/08 15:42:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:59972->10.242.139.247:8302: i/o timeout 2021/04/08 15:42:32 [INFO] memberlist: Suspect dev-consul-server-0.dc1 has failed, no acks received 2021/04/08 15:42:37 [INFO] memberlist: Marking dev-consul-server-0.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/04/08 15:42:37 [INFO] serf: EventMemberFailed: dev-consul-server-0.dc1 10.242.139.247 2021/04/08 15:42:37 [INFO] consul: Handled member-failed event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:42:39 [ERR] memberlist: failed to receive: read tcp 10.242.12.17:8302->10.242.139.247:51188: i/o timeout from=10.242.139.247:51188 2021/04/08 15:42:39 [ERR] memberlist: Failed to send error: write tcp 10.242.12.17:8302->10.242.139.247:51188: i/o timeout from=10.242.139.247:51188 2021/04/08 15:42:43 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:42:55 [INFO] serf: attempting reconnect to dev-consul-server-0.dc1 10.242.139.247:8302 2021/04/08 15:43:23 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-w5szm 10.242.139.195:8301 2021/04/08 15:44:03 [INFO] serf: attempting reconnect to dev-consul-server-0 10.242.139.247:8301 2021/04/08 15:44:17 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.139.200 2021/04/08 15:44:17 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.139.200:8300) (DC: dc1) 2021/04/08 15:44:17 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.139.200 2021/04/08 15:44:17 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/04/08 15:44:32 [ERR] memberlist: Failed fallback ping: write tcp 10.242.12.17:47432->10.242.139.200:8301: i/o timeout 2021/04/08 15:44:32 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:44:35 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0.dc1) 2021/04/08 15:44:46 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.238 2021/04/08 15:45:36 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:47750->10.242.139.238:8301: i/o timeout 2021/04/08 15:45:36 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:45:36 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-w5szm) 2021/04/08 15:45:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48022->10.242.139.238:8301: i/o timeout 2021/04/08 15:45:50 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:45:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48042->10.242.139.238:8301: i/o timeout 2021/04/08 15:45:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48118->10.242.139.238:8301: i/o timeout 2021/04/08 15:45:56 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:45:57 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-w5szm 10.242.139.238 2021/04/08 15:45:59 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-w5szm 10.242.139.238 2021/04/08 15:46:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48184->10.242.139.238:8301: i/o timeout 2021/04/08 15:46:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48258->10.242.139.238:8301: i/o timeout 2021/04/08 15:46:04 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:46:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49134->10.242.139.200:8301: i/o timeout 2021/04/08 15:46:07 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:46:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48270->10.242.139.238:8301: i/o timeout 2021/04/08 15:46:10 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:48358->10.242.139.238:8301: i/o timeout 2021/04/08 15:46:17 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-w5szm but other probes failed, network may be misconfigured 2021/04/08 15:46:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:49340->10.242.139.200:8301: i/o timeout 2021/04/08 15:46:18 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:49:40 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:51848->10.242.139.238:8301: i/o timeout 2021/04/08 15:49:40 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:50:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53264->10.242.139.200:8301: i/o timeout 2021/04/08 15:50:15 [INFO] memberlist: Suspect dev-consul-server-0 has failed, no acks received 2021/04/08 15:50:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52520->10.242.139.238:8301: i/o timeout 2021/04/08 15:50:19 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:50:22 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-0) 2021/04/08 15:50:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52642->10.242.139.238:8301: i/o timeout 2021/04/08 15:50:27 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:50:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:52690->10.242.139.238:8301: i/o timeout 2021/04/08 15:50:30 [INFO] memberlist: Suspect dev-consul-7467bf7b46-w5szm has failed, no acks received 2021/04/08 15:50:30 [ERR] memberlist: Failed fallback ping: read tcp 10.242.12.17:53496->10.242.139.200:8301: i/o timeout 2021/04/08 15:50:30 [INFO] memberlist: Marking dev-consul-7467bf7b46-w5szm as failed, suspect timeout reached (2 peer confirmations) 2021/04/08 15:50:30 [INFO] serf: EventMemberFa