+ CONSUL_BIND= + [ -n ] + CONSUL_CLIENT= + [ -n ] + CONSUL_DATA_DIR=/consul/data + CONSUL_CONFIG_DIR=/consul/config + [ -n ] + [ a = - ] + [ agent = agent ] + shift + set -- consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui + [ consul = consul ] + stat -c %u /consul/data + id -u consul + [ 100 != 100 ] + stat -c %u /consul/config + id -u consul + [ 100 != 100 ] + [ ! -z ] + exec consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui bootstrap_expect > 0: expecting 3 servers ==> Starting Consul agent... ==> Consul agent running! Version: 'v1.0.6' Node ID: '10c77ddf-c1bf-3392-459d-f53273b4be0f' Node name: 'dev-consul-server-2' Datacenter: 'dc1' (Segment: '') Server: true (Bootstrap: false) Client Addr: [0.0.0.0] (HTTP: 8500, HTTPS: -1, DNS: 8600) Cluster Addr: 10.242.117.136 (LAN: 8301, WAN: 8302) Encrypt: Gossip: false, TLS-Outgoing: false, TLS-Incoming: false ==> Log data will now stream in as it occurs: 2021/06/17 22:40:07 [INFO] raft: Initial configuration (index=0): [] 2021/06/17 22:40:07 [INFO] raft: Node at 10.242.117.136:8300 [Follower] entering Follower state (Leader: "") 2021/06/17 22:40:07 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.117.136 2021/06/17 22:40:07 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.117.136 2021/06/17 22:40:07 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.117.136:8300) (DC: dc1) 2021/06/17 22:40:07 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/17 22:40:08 [INFO] agent: Started DNS server 0.0.0.0:8600 (tcp) 2021/06/17 22:40:08 [INFO] agent: Started DNS server 0.0.0.0:8600 (udp) 2021/06/17 22:40:08 [INFO] agent: Started HTTP server on [::]:8500 (tcp) 2021/06/17 22:40:08 [INFO] agent: started state syncer 2021/06/17 22:40:08 [INFO] agent: Retry join LAN is supported for: aliyun aws azure digitalocean gce os scaleway softlayer 2021/06/17 22:40:08 [INFO] agent: Joining LAN cluster... 2021/06/17 22:40:08 [INFO] agent: (LAN) joining: [dev-consul-server-0.consul-server dev-consul-server-1.consul-server dev-consul-server-2.consul-server] 2021/06/17 22:40:08 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.162.71 2021/06/17 22:40:08 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 22:40:08 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.162.71:8300) (DC: dc1) 2021/06/17 22:40:08 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 22:40:08 [INFO] consul: Found expected number of peers, attempting bootstrap: 10.242.117.136:8300,10.242.162.71:8300,10.242.13.79:8300 2021/06/17 22:40:08 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 22:40:08 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.162.71 2021/06/17 22:40:08 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 22:40:08 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/06/17 22:40:08 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 22:40:08 [WARN] memberlist: Failed to resolve dev-consul-server-2.consul-server: lookup dev-consul-server-2.consul-server on 10.96.0.10:53: no such host 2021/06/17 22:40:08 [INFO] agent: (LAN) joined: 2 Err: 2021/06/17 22:40:08 [INFO] agent: Join LAN completed. Synced with 2 initial agents 2021/06/17 22:40:13 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/06/17 22:40:13 [INFO] raft: Node at 10.242.117.136:8300 [Candidate] entering Candidate state in term 2 2021/06/17 22:40:13 [INFO] raft: Election won. Tally: 2 2021/06/17 22:40:13 [INFO] raft: Node at 10.242.117.136:8300 [Leader] entering Leader state 2021/06/17 22:40:13 [INFO] raft: Added peer 625429b0-2c33-6d50-2432-70aefdff073d, starting replication 2021/06/17 22:40:13 [INFO] raft: Added peer 420ae614-e1ab-47c4-40cd-a971e2ac4c53, starting replication 2021/06/17 22:40:13 [INFO] consul: cluster leadership acquired 2021/06/17 22:40:13 [INFO] consul: New leader elected: dev-consul-server-2 2021/06/17 22:40:13 [WARN] raft: AppendEntries to {Voter 625429b0-2c33-6d50-2432-70aefdff073d 10.242.162.71:8300} rejected, sending older logs (next: 1) 2021/06/17 22:40:13 [WARN] raft: AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} rejected, sending older logs (next: 1) 2021/06/17 22:40:13 [INFO] raft: pipelining replication to peer {Voter 625429b0-2c33-6d50-2432-70aefdff073d 10.242.162.71:8300} 2021/06/17 22:40:13 [INFO] raft: pipelining replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 22:40:13 [INFO] consul: member 'dev-consul-server-2' joined, marking health alive 2021/06/17 22:40:13 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/06/17 22:40:13 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 22:40:13 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 22:40:13 [INFO] agent: Synced node info ==> Failed to check for updates: Get https://checkpoint-api.hashicorp.com/v1/check/consul?arch=amd64&os=linux&signature=0d588d79-944a-6b1f-1c19-6726f2d173eb&version=1.0.6: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) 2021/06/17 22:58:10 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:39796->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:12 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:15 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:35756->10.242.13.79:8301: i/o timeout 2021/06/17 22:58:17 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 22:58:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:20 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:39974->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:20 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:20 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.515393955s 2021/06/17 22:58:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:24 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:25 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-rx74l but other probes failed, network may be misconfigured 2021/06/17 22:58:26 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:26 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/17 22:58:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40190->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:27 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:30 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-rx74l but other probes failed, network may be misconfigured 2021/06/17 22:58:31 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500186078s 2021/06/17 22:58:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:34 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:35 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40386->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:35 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:35 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 22:58:35 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 22:58:35 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-rx74l) 2021/06/17 22:58:35 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 22:58:35 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 22:58:40 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:36360->10.242.13.79:8301: i/o timeout 2021/06/17 22:58:40 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 22:58:40 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:41 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500183437s 2021/06/17 22:58:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:36412->10.242.13.79:8301: i/o timeout 2021/06/17 22:58:43 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 22:58:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:44 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.976654223s 2021/06/17 22:58:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:46 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40610->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:46 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:46 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.44424453s 2021/06/17 22:58:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42768->10.242.13.79:8302: i/o timeout 2021/06/17 22:58:47 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 22:58:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40704->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:49 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:49 [INFO] raft: aborting pipeline replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 22:58:49 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 22:58:49 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 22:58:50 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:58:51 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:36644->10.242.13.79:8301: i/o timeout 2021/06/17 22:58:51 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 22:58:51 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 22:58:51 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 22:58:51 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 22:58:51 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 22:58:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:51 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500157105s 2021/06/17 22:58:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:36666->10.242.13.79:8301: i/o timeout 2021/06/17 22:58:53 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 22:58:53 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 22:58:53 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 22:58:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:54 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.99501569s 2021/06/17 22:58:55 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40856->10.242.13.74:8301: i/o timeout 2021/06/17 22:58:55 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 22:58:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:56 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.481382646s 2021/06/17 22:58:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:58:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:43048->10.242.13.79:8302: i/o timeout 2021/06/17 22:58:58 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 22:58:59 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38636->10.242.13.79:8300: i/o timeout 2021/06/17 22:58:59 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:60060->10.242.13.79:8300: i/o timeout 2021/06/17 22:58:59 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:58:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:00 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:43186->10.242.13.79:8302: i/o timeout 2021/06/17 22:59:08 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 22:59:08 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 22:59:09 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:60354->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:09 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:10 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60366->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:10 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:15 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 22:59:15 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 22:59:15 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 22:59:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:43588->10.242.13.79:8302: i/o timeout 2021/06/17 22:59:18 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 22:59:19 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:60628->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:20 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60650->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:21 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:23 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 22:59:23 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:59452 2021/06/17 22:59:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:29 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:60914->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:29 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:29 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 22:59:29 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 22:59:31 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60944->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:32 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:32 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 22:59:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:39 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:32920->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:39 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:42 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:32982->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:43 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:48 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 22:59:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:49 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:33218->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:49 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:53 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:33306->10.242.13.79:8300: i/o timeout 2021/06/17 22:59:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:54 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 22:59:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 22:59:59 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:33448->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:00 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:04 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:33558->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:05 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:08 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:00:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:10 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:33706->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:10 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:11 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:00:11 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:00:14 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:00:15 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:33806->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:16 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:20 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:33962->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:22 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:26 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:34086->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:28 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:00:28 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:32 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:34548->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:34 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:00:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:38 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:34920->10.242.13.79:8300: i/o timeout 2021/06/17 23:00:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:40 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:00:40 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:00:40 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:00:40 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:00:40 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:00:40 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:00:40 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:00:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:41 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:00:41 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:00:42 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:00:42 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:00:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:00:43 [WARN] raft: Rejecting vote request from 10.242.13.79:8300 since we have a leader: 10.242.117.136:8300 2021/06/17 23:00:43 [DEBUG] raft-net: 10.242.117.136:8300 accepted connection from: 10.242.13.79:51518 2021/06/17 23:00:44 [INFO] raft: pipelining replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:01:00 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/17 23:01:00 [WARN] consul: error getting server health from "dev-consul-server-0": last request still outstanding 2021/06/17 23:02:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42174->10.242.13.79:8301: i/o timeout 2021/06/17 23:02:06 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:02:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42224->10.242.13.79:8301: i/o timeout 2021/06/17 23:02:08 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:02:08 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500171785s 2021/06/17 23:02:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:11 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:46404->10.242.13.74:8301: i/o timeout 2021/06/17 23:02:11 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:02:11 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.991068961s 2021/06/17 23:02:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:13 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.449751687s 2021/06/17 23:02:14 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:46466->10.242.13.74:8301: i/o timeout 2021/06/17 23:02:14 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:02:14 [INFO] raft: aborting pipeline replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:02:15 [ERR] memberlist: Push/Pull with dev-consul-7467bf7b46-rx74l failed: read tcp 10.242.117.136:46318->10.242.13.74:8301: i/o timeout 2021/06/17 23:02:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42392->10.242.13.79:8301: i/o timeout 2021/06/17 23:02:17 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:02:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:48682->10.242.13.79:8302: i/o timeout 2021/06/17 23:02:17 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:02:18 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500201712s 2021/06/17 23:02:19 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:02:19 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:02:19 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:02:19 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:02:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:20 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:46652->10.242.13.74:8301: i/o timeout 2021/06/17 23:02:20 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:02:20 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:02:20 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:02:20 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.94070977s 2021/06/17 23:02:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:22 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:02:22 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.74:51180 2021/06/17 23:02:23 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.418405311s 2021/06/17 23:02:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:23 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:02:23 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:02:24 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:37600->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:24 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:26 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:35356->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:26 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:02:26 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:02:26 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:48804->10.242.13.79:8302: i/o timeout 2021/06/17 23:02:27 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:02:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:34 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:37844->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:34 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:36 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:37906->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:37 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:49160->10.242.13.79:8302: i/o timeout 2021/06/17 23:02:37 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:02:38 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:02:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:44 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38080->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:44 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:47 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38168->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:47 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:02:47 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:02:47 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:02:48 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:51 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:02:51 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:51282 2021/06/17 23:02:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:49514->10.242.13.79:8302: i/o timeout 2021/06/17 23:02:52 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:02:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:54 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38322->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:54 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:02:58 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38422->10.242.13.79:8300: i/o timeout 2021/06/17 23:02:59 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:02:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:04 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38566->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:04 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:05 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:03:05 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:03:08 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:03:08 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:03:09 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38682->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:09 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:14 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38800->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:14 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:18 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:03:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:19 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38930->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:20 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:24 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39040->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:25 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:30 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:39196->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:32 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:35 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39294->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:35 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:42 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:39466->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:43 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:45 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39550->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:46 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:47 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:03:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:47 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:03:50 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:03:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:53 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:39734->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:57 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39824->10.242.13.79:8300: i/o timeout 2021/06/17 23:03:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:03:58 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:03:59 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:03:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:05 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:40024->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:08 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:04:09 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:40134->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:14 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:04:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:18 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:40350->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:24 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:40476->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:24 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:04:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:29 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:04:29 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:04:32 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:04:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:34 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:40722->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:34 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:04:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:38 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:04:38 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:04:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:44 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:40948->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:45 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:04:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:04:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:55 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:41224->10.242.13.79:8300: i/o timeout 2021/06/17 23:04:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:04:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:05 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:41442->10.242.13.79:8300: i/o timeout 2021/06/17 23:05:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:06 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:05:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:11 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:05:11 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:05:14 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:05:15 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:05:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:16 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:41716->10.242.13.79:8300: i/o timeout 2021/06/17 23:05:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:18 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:05:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:25 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:41928->10.242.13.79:8300: i/o timeout 2021/06/17 23:05:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:27 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:05:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:35 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:05:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:37 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:42232->10.242.13.79:8300: i/o timeout 2021/06/17 23:05:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:45 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:42414->10.242.13.79:8300: i/o timeout 2021/06/17 23:05:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:48 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:05:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:53 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:05:53 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:05:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:05:56 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:05:56 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:05:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:05:58 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:05:58 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:42732->10.242.13.79:8300: i/o timeout 2021/06/17 23:05:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:05 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:42890->10.242.13.79:8300: i/o timeout 2021/06/17 23:06:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:09 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:06:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:16 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:06:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:19 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:43218->10.242.13.79:8300: i/o timeout 2021/06/17 23:06:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:26 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:43368->10.242.13.79:8300: i/o timeout 2021/06/17 23:06:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:30 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:06:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:35 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:06:35 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:06:36 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:06:38 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:06:38 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:06:38 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:06:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:40 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:43818->10.242.13.79:8300: i/o timeout 2021/06/17 23:06:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:46 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:43952->10.242.13.79:8300: i/o timeout 2021/06/17 23:06:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:51 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:06:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:56 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:06:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:06:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:01 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:44316->10.242.13.79:8300: i/o timeout 2021/06/17 23:07:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:06 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:44434->10.242.13.79:8300: i/o timeout 2021/06/17 23:07:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:12 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:07:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:16 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:07:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:17 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:07:17 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:07:18 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:07:20 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:07:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:22 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:44816->10.242.13.79:8300: i/o timeout 2021/06/17 23:07:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:26 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:44916->10.242.13.79:8300: i/o timeout 2021/06/17 23:07:27 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:07:27 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2) 2021/06/17 23:07:27 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:07:27 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:07:27 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:07:27 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:07:27 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:07:27 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:07:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:07:33 [WARN] raft: Rejecting vote request from 10.242.13.79:8300 since we have a leader: 10.242.117.136:8300 2021/06/17 23:07:37 [ERR] raft: peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} has newer term, stopping replication 2021/06/17 23:07:37 [INFO] raft: Node at 10.242.117.136:8300 [Follower] entering Follower state (Leader: "") 2021/06/17 23:07:37 [INFO] raft: aborting pipeline replication to peer {Voter 625429b0-2c33-6d50-2432-70aefdff073d 10.242.162.71:8300} 2021/06/17 23:07:37 [INFO] consul: cluster leadership lost 2021/06/17 23:07:40 [WARN] raft: Rejecting vote request from 10.242.13.79:8300 since our last index is greater (461, 401) 2021/06/17 23:07:44 [ERR] agent: Coordinate update error: No cluster leader 2021/06/17 23:07:46 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/06/17 23:07:46 [INFO] raft: Node at 10.242.117.136:8300 [Candidate] entering Candidate state in term 5 2021/06/17 23:07:46 [INFO] raft: Election won. Tally: 2 2021/06/17 23:07:46 [INFO] raft: Node at 10.242.117.136:8300 [Leader] entering Leader state 2021/06/17 23:07:46 [INFO] raft: Added peer 625429b0-2c33-6d50-2432-70aefdff073d, starting replication 2021/06/17 23:07:46 [INFO] raft: Added peer 420ae614-e1ab-47c4-40cd-a971e2ac4c53, starting replication 2021/06/17 23:07:46 [INFO] consul: cluster leadership acquired 2021/06/17 23:07:46 [INFO] consul: New leader elected: dev-consul-server-2 2021/06/17 23:07:46 [INFO] raft: pipelining replication to peer {Voter 625429b0-2c33-6d50-2432-70aefdff073d 10.242.162.71:8300} 2021/06/17 23:07:46 [WARN] raft: AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} rejected, sending older logs (next: 402) 2021/06/17 23:07:46 [INFO] raft: pipelining replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:08:46 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:55958->10.242.13.74:8301: i/o timeout 2021/06/17 23:08:46 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:08:47 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:08:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:55976->10.242.13.74:8301: i/o timeout 2021/06/17 23:08:49 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:08:49 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:08:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:08:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:08:52 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500238709s 2021/06/17 23:08:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:58192->10.242.13.79:8302: i/o timeout 2021/06/17 23:08:52 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:08:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:51932->10.242.13.79:8301: i/o timeout 2021/06/17 23:08:53 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:08:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:08:54 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.925906281s 2021/06/17 23:08:55 [ERR] memberlist: Push/Pull with dev-consul-7467bf7b46-rx74l failed: read tcp 10.242.117.136:55954->10.242.13.74:8301: i/o timeout 2021/06/17 23:08:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52024->10.242.13.79:8301: i/o timeout 2021/06/17 23:08:56 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:08:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:08:57 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.370693165s 2021/06/17 23:08:57 [INFO] memberlist: Marking dev-consul-7467bf7b46-rx74l as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:08:57 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:08:57 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:08:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:56224->10.242.13.74:8301: i/o timeout 2021/06/17 23:08:58 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:08:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:08:59 [INFO] raft: aborting pipeline replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:08:59 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:08:59 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:09:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52140->10.242.13.79:8301: i/o timeout 2021/06/17 23:09:00 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:09:00 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:45304->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:01 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:09:01 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500227431s 2021/06/17 23:09:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52208->10.242.13.79:8301: i/o timeout 2021/06/17 23:09:02 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:09:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:58438->10.242.13.79:8302: i/o timeout 2021/06/17 23:09:02 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:09:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:04 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.94513846s 2021/06/17 23:09:04 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:09:04 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:09:04 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:09:05 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52274->10.242.13.79:8301: i/o timeout 2021/06/17 23:09:05 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:09:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:06 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.402910846s 2021/06/17 23:09:07 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:09:07 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.74:54162 2021/06/17 23:09:07 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:09:07 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:54388 2021/06/17 23:09:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:09 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:47370->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:09 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:11 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:47422->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:12 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:58678->10.242.13.79:8302: i/o timeout 2021/06/17 23:09:12 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:09:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:19 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:47614->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:22 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:47680->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:22 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:22 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:09:22 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:09:22 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:09:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:27 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:09:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:59040->10.242.13.79:8302: i/o timeout 2021/06/17 23:09:27 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:09:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:29 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:47862->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:29 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:32 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:47934->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:33 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:39 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48206->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:39 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:40 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:09:40 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:09:43 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:09:43 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48292->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:44 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:49 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48452->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:49 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:54 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48576->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:09:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:09:59 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48694->10.242.13.79:8300: i/o timeout 2021/06/17 23:09:59 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:05 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48836->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:06 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:07 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:10:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:09 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48942->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:10 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:16 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49094->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:17 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:20 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:49194->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:20 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:22 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:10:22 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:10:25 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:10:25 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:10:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:27 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49380->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:29 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:30 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:49444->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:32 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:39 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49660->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:42 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:49710->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:44 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:47 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:10:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49966->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:54 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:50016->10.242.13.79:8300: i/o timeout 2021/06/17 23:10:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:10:58 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:10:59 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:11:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:04 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:11:04 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:11:07 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:11:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:08 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:11:08 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:50366->10.242.13.79:8300: i/o timeout 2021/06/17 23:11:09 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:50396->10.242.13.79:8300: i/o timeout 2021/06/17 23:11:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:11:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:20 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:11:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:27 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:11:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:29 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:50858->10.242.13.79:8300: i/o timeout 2021/06/17 23:11:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:30 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:50870->10.242.13.79:8300: i/o timeout 2021/06/17 23:11:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:40 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:11:40 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:11:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:46 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:11:46 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:11:49 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:11:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:50 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:51354->10.242.13.79:8300: i/o timeout 2021/06/17 23:11:50 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:51362->10.242.13.79:8300: i/o timeout 2021/06/17 23:11:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:11:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:00 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:12:01 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:12:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:07 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:12:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:10 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:51864->10.242.13.79:8300: i/o timeout 2021/06/17 23:12:11 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:51884->10.242.13.79:8300: i/o timeout 2021/06/17 23:12:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:20 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:12:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:22 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:12:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:26 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:12:26 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:12:29 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:12:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:30 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:52336->10.242.13.79:8300: i/o timeout 2021/06/17 23:12:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:32 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:52384->10.242.13.79:8300: i/o timeout 2021/06/17 23:12:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:41 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:12:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:43 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:12:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:47 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:12:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:51 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:52926->10.242.13.79:8300: i/o timeout 2021/06/17 23:12:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:53 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:52982->10.242.13.79:8300: i/o timeout 2021/06/17 23:12:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:12:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:01 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:13:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:04 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:13:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:08 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:13:08 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:13:08 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:13:11 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:13:11 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:53406->10.242.13.79:8300: i/o timeout 2021/06/17 23:13:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:14 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:53484->10.242.13.79:8300: i/o timeout 2021/06/17 23:13:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:21 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:13:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:25 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:13:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:27 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:13:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:31 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:53884->10.242.13.79:8300: i/o timeout 2021/06/17 23:13:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:35 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:53978->10.242.13.79:8300: i/o timeout 2021/06/17 23:13:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:41 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:13:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:46 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:13:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:50 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:13:50 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:13:51 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:54364->10.242.13.79:8300: i/o timeout 2021/06/17 23:13:53 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:13:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:13:56 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:54466->10.242.13.79:8300: i/o timeout 2021/06/17 23:13:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:02 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:14:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:07 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:14:07 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:14:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:08 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:14:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:12 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:54850->10.242.13.79:8300: i/o timeout 2021/06/17 23:14:13 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:14:13 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:14:13 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:14:13 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:14:13 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1.dc1) 2021/06/17 23:14:13 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:14:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:14:14 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:14:14 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:14:16 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:35872->10.242.13.74:8301: i/o timeout 2021/06/17 23:14:16 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:14:22 [INFO] raft: pipelining replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:15:23 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:15:25 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:37598->10.242.13.74:8301: i/o timeout 2021/06/17 23:15:26 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:15:26 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500172364s 2021/06/17 23:15:27 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:15:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:29 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:33482->10.242.13.79:8301: i/o timeout 2021/06/17 23:15:29 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:15:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:37734->10.242.13.74:8301: i/o timeout 2021/06/17 23:15:32 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:15:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:34 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500335488s 2021/06/17 23:15:35 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:55348->10.242.13.79:8300: i/o timeout 2021/06/17 23:15:35 [ERR] memberlist: Push/Pull with dev-consul-server-1 failed: read tcp 10.242.117.136:33450->10.242.13.79:8301: i/o timeout 2021/06/17 23:15:36 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:33758->10.242.13.79:8301: i/o timeout 2021/06/17 23:15:36 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:15:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:36 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.985541978s 2021/06/17 23:15:37 [INFO] memberlist: Marking dev-consul-7467bf7b46-rx74l as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:15:37 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:15:37 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:15:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40076->10.242.13.79:8302: i/o timeout 2021/06/17 23:15:37 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:15:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:39 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:33872->10.242.13.79:8301: i/o timeout 2021/06/17 23:15:39 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:15:39 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.413869395s 2021/06/17 23:15:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:40 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:15:40 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:15:40 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:15:40 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:15:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:33918->10.242.13.79:8301: i/o timeout 2021/06/17 23:15:41 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:15:41 [INFO] raft: aborting pipeline replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:15:41 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:15:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:43 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:15:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:44 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.502684297s 2021/06/17 23:15:46 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:57282->10.242.13.79:8300: i/o timeout 2021/06/17 23:15:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:46 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.980706425s 2021/06/17 23:15:46 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:15:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40204->10.242.13.79:8302: i/o timeout 2021/06/17 23:15:47 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:15:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:49 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.429342991s 2021/06/17 23:15:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:51 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:57428->10.242.13.79:8300: i/o timeout 2021/06/17 23:15:51 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:15:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:52 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:15:52 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:57458 2021/06/17 23:15:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:15:56 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:57544->10.242.13.79:8300: i/o timeout 2021/06/17 23:15:57 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:15:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:40568->10.242.13.79:8302: i/o timeout 2021/06/17 23:15:57 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:15:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:01 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:16:01 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:16:01 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:16:01 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:57666->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:01 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:02 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:16:02 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:16:05 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:16:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:07 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:57814->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:08 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:11 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:57912->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:11 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:16:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:18 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58086->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:21 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:58154->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:22 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:23 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:16:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:29 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58346->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:30 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:32 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:58398->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:32 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:40 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58608->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:41 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:42 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:58640->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:44 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:16:44 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:16:47 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:16:47 [ERR] memberlist: failed to receive: read tcp 10.242.117.136:8302->10.242.13.79:52948: i/o timeout from=10.242.13.79:52948 2021/06/17 23:16:47 [ERR] memberlist: Failed to send error: write tcp 10.242.117.136:8302->10.242.13.79:52948: i/o timeout from=10.242.13.79:52948 2021/06/17 23:16:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:51 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58880->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:52 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:58896->10.242.13.79:8300: i/o timeout 2021/06/17 23:16:52 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:53 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:16:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:16:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:02 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:59138->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:03 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:03 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:59152->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:03 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:17:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:05 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:13 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:59394->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:17:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:14 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:15 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:59440->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:18 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:24 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:59670->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:26 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:17:26 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:17:27 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:28 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:59784->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:29 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:17:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:34 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:37 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:59970->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:43 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:17:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:44 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60148->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:52 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:60342->10.242.13.79:8300: i/o timeout 2021/06/17 23:17:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:17:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:17:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:02 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:18:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:05 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60656->10.242.13.79:8300: i/o timeout 2021/06/17 23:18:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:08 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:18:08 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:18:11 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:18:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:12 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:60820->10.242.13.79:8300: i/o timeout 2021/06/17 23:18:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:18:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:16 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:18:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:22 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:18:23 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:18:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:26 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:32932->10.242.13.79:8300: i/o timeout 2021/06/17 23:18:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:32 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:33072->10.242.13.79:8300: i/o timeout 2021/06/17 23:18:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:37 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:18:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:18:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:47 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:33540->10.242.13.79:8300: i/o timeout 2021/06/17 23:18:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:50 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:18:50 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:18:52 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:33660->10.242.13.79:8300: i/o timeout 2021/06/17 23:18:53 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:18:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:18:58 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:19:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:03 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:19:03 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:19:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:08 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:34056->10.242.13.79:8300: i/o timeout 2021/06/17 23:19:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:13 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:34146->10.242.13.79:8300: i/o timeout 2021/06/17 23:19:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:19:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:19:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:23 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:19:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:29 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:34550->10.242.13.79:8300: i/o timeout 2021/06/17 23:19:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:32 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:19:32 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:19:33 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:34630->10.242.13.79:8300: i/o timeout 2021/06/17 23:19:35 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:19:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:40 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:19:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:43 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:19:43 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:19:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:50 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:35044->10.242.13.79:8300: i/o timeout 2021/06/17 23:19:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:53 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:35114->10.242.13.79:8300: i/o timeout 2021/06/17 23:19:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:19:54 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:19:54 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:19:55 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1.dc1) 2021/06/17 23:19:55 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:19:55 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:19:55 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:19:57 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-rx74l) 2021/06/17 23:19:57 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:19:57 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:20:03 [INFO] raft: pipelining replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:21:05 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:21:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:41890->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:06 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:06 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500179235s 2021/06/17 23:21:08 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:46052->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:08 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:11 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:46158->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:11 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:11 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.502843698s 2021/06/17 23:21:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:13 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42034->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:13 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:13 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.989579524s 2021/06/17 23:21:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:15 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:46226->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:15 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:16 [INFO] memberlist: Marking dev-consul-7467bf7b46-rx74l as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:21:16 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:21:16 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:21:16 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.447819074s 2021/06/17 23:21:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42136->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:17 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:17 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:21:17 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:21:17 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:21:17 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:21:17 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:21:17 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:21:17 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:21:17 [ERR] raft: Failed to pipeline AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: write tcp 10.242.117.136:35568->10.242.13.79:8300: use of closed network connection 2021/06/17 23:21:17 [INFO] raft: aborting pipeline replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:21:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:19 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42222->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:19 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:19 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:35626->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:20 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500169067s 2021/06/17 23:21:21 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42270->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:21 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:22 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.988174752s 2021/06/17 23:21:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:48530->10.242.13.79:8302: i/o timeout 2021/06/17 23:21:22 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:21:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42318->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:23 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:23 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:21:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:25 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.475731581s 2021/06/17 23:21:26 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42388->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:26 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:27 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:37416->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:29 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42474->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:29 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:30 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:21:30 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:21:30 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:21:30 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:21:30 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:37486->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:31 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:42618->10.242.13.79:8301: i/o timeout 2021/06/17 23:21:31 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:21:31 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:21:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:48882->10.242.13.79:8302: i/o timeout 2021/06/17 23:21:32 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:21:33 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:21:33 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:60816 2021/06/17 23:21:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:37 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:37672->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:37 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:21:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:41 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:37860->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:21:42 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:21:42 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:21:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:49006->10.242.13.79:8302: i/o timeout 2021/06/17 23:21:42 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:21:44 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:47082->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:44 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:46 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:47126->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:46 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:47 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38022->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:47 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:21:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:47176->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:48 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:48 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:21:48 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:21:51 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:47256->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:51 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:51 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:21:51 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:21:51 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:21:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38122->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:52 [ERR] memberlist: failed to receive: read tcp 10.242.117.136:8301->10.242.13.74:34950: i/o timeout from=10.242.13.74:34950 2021/06/17 23:21:52 [ERR] memberlist: Failed to send error: write tcp 10.242.117.136:8301->10.242.13.74:34950: i/o timeout from=10.242.13.74:34950 2021/06/17 23:21:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:47276->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:53 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:53 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:21:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:55 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:47348->10.242.13.74:8301: i/o timeout 2021/06/17 23:21:55 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:21:55 [INFO] memberlist: Marking dev-consul-7467bf7b46-rx74l as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:21:55 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:21:55 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:21:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:21:57 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38264->10.242.13.79:8300: i/o timeout 2021/06/17 23:21:57 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:21:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:03 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38380->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:03 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:22:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:04 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:07 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38518->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:08 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:22:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:14 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38642->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:14 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:18 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:38746->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:18 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:24 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:38906->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:25 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:28 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39002->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:28 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:30 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:22:30 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:22:33 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:22:33 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:22:33 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/17 23:22:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:35 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:39178->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:37 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:37 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:22:37 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:22:37 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:22:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:38 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39254->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:41 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:44314->10.242.13.79:8301: i/o timeout 2021/06/17 23:22:41 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:22:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:42 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:22:42 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:22:42 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:22:42 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:22:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:44358->10.242.13.79:8301: i/o timeout 2021/06/17 23:22:43 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:22:43 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:22:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:47 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:39452->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:49 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39504->10.242.13.79:8300: i/o timeout 2021/06/17 23:22:49 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:50 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:22:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:22:59 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:39754->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:00 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:39780->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:02 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:23:03 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:23:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:08 [ERR] memberlist: failed to receive: read tcp 10.242.117.136:8302->10.242.13.79:56124: i/o timeout from=10.242.13.79:56124 2021/06/17 23:23:08 [ERR] memberlist: Failed to send error: write tcp 10.242.117.136:8302->10.242.13.79:56124: i/o timeout from=10.242.13.79:56124 2021/06/17 23:23:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:12 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:23:12 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:23:12 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:40068->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:13 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:40072->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:15 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:23:15 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:23:15 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/17 23:23:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:18 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:23:18 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:23:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:23 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:23:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:28 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:40446->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:28 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:40466->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:38 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:23:39 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:23:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:48 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:40976->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:49 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:41002->10.242.13.79:8300: i/o timeout 2021/06/17 23:23:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:54 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:23:54 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:23:57 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:23:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:23:58 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:24:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:00 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:24:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:03 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:24:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:08 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:41458->10.242.13.79:8300: i/o timeout 2021/06/17 23:24:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:10 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:41502->10.242.13.79:8300: i/o timeout 2021/06/17 23:24:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:24:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:24:20 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:21 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:24:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:29 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:41950->10.242.13.79:8300: i/o timeout 2021/06/17 23:24:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:31 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:42004->10.242.13.79:8300: i/o timeout 2021/06/17 23:24:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:36 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:24:36 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:24:39 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:24:39 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:24:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:24:43 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:24:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:49 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:42536->10.242.13.79:8300: i/o timeout 2021/06/17 23:24:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:42614->10.242.13.79:8300: i/o timeout 2021/06/17 23:24:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:58 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:24:59 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:25:00 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:02 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:04 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:04 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:25:06 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:08 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:09 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:43030->10.242.13.79:8300: i/o timeout 2021/06/17 23:25:10 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:12 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:14 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:14 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:43136->10.242.13.79:8300: i/o timeout 2021/06/17 23:25:16 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:18 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:18 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:25:18 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:25:19 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:25:21 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:25:22 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:23 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:25:24 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:25 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:25:26 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:28 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:29 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:43514->10.242.13.79:8300: i/o timeout 2021/06/17 23:25:30 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:32 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:34 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:35 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:43652->10.242.13.79:8300: i/o timeout 2021/06/17 23:25:36 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:38 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:39 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:25:40 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:42 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:44 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:46 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:46 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:25:48 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:49 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:44010->10.242.13.79:8300: i/o timeout 2021/06/17 23:25:50 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:52 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:54 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:55 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-rx74l) 2021/06/17 23:25:55 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:25:55 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:25:56 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:25:56 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:25:56 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:25:56 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:25:56 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:25:56 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:25:56 [WARN] raft: Rejecting vote request from 10.242.13.79:8300 since we have a leader: 10.242.117.136:8300 2021/06/17 23:26:00 [ERR] raft: peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} has newer term, stopping replication 2021/06/17 23:26:00 [INFO] raft: Node at 10.242.117.136:8300 [Follower] entering Follower state (Leader: "") 2021/06/17 23:26:00 [INFO] raft: aborting pipeline replication to peer {Voter 625429b0-2c33-6d50-2432-70aefdff073d 10.242.162.71:8300} 2021/06/17 23:26:00 [INFO] consul: cluster leadership lost 2021/06/17 23:26:01 [WARN] consul.coordinate: Batch update failed: node is not the leader 2021/06/17 23:26:04 [WARN] raft: Rejecting vote request from 10.242.13.79:8300 since our last index is greater (794, 707) 2021/06/17 23:26:07 [DEBUG] raft-net: 10.242.117.136:8300 accepted connection from: 10.242.162.71:51722 2021/06/17 23:26:07 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/06/17 23:26:07 [INFO] raft: Node at 10.242.117.136:8300 [Candidate] entering Candidate state in term 8 2021/06/17 23:26:07 [INFO] raft: Election won. Tally: 2 2021/06/17 23:26:07 [INFO] raft: Node at 10.242.117.136:8300 [Leader] entering Leader state 2021/06/17 23:26:07 [INFO] raft: Added peer 625429b0-2c33-6d50-2432-70aefdff073d, starting replication 2021/06/17 23:26:07 [INFO] raft: Added peer 420ae614-e1ab-47c4-40cd-a971e2ac4c53, starting replication 2021/06/17 23:26:07 [INFO] consul: cluster leadership acquired 2021/06/17 23:26:07 [INFO] consul: New leader elected: dev-consul-server-2 2021/06/17 23:26:07 [INFO] raft: pipelining replication to peer {Voter 625429b0-2c33-6d50-2432-70aefdff073d 10.242.162.71:8300} 2021/06/17 23:26:07 [WARN] raft: AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} rejected, sending older logs (next: 708) 2021/06/17 23:26:07 [INFO] raft: pipelining replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:27:43 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:51792->10.242.13.79:8301: i/o timeout 2021/06/17 23:27:43 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:27:44 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500244596s 2021/06/17 23:27:44 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:27:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:27:46 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:55964->10.242.13.74:8301: i/o timeout 2021/06/17 23:27:46 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:27:48 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:27:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:51910->10.242.13.79:8301: i/o timeout 2021/06/17 23:27:49 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:27:49 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500208509s 2021/06/17 23:27:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:27:51 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:56110->10.242.13.74:8301: i/o timeout 2021/06/17 23:27:51 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:27:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:27:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:56152->10.242.13.74:8301: i/o timeout 2021/06/17 23:27:53 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:27:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:27:53 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500200485s 2021/06/17 23:27:55 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52052->10.242.13.79:8301: i/o timeout 2021/06/17 23:27:55 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:27:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:27:56 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500249894s 2021/06/17 23:27:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52134->10.242.13.79:8301: i/o timeout 2021/06/17 23:27:57 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:27:57 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:27:57 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:27:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:27:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:58368->10.242.13.79:8302: i/o timeout 2021/06/17 23:27:57 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:27:59 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:52196->10.242.13.79:8301: i/o timeout 2021/06/17 23:27:59 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:28:00 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:28:00 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:28:00 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:28:00 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:28:01 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:02 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:28:02 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:03 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:28:03 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:05 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:28:05 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.74:35724 2021/06/17 23:28:05 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500166418s 2021/06/17 23:28:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:07 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.984313448s 2021/06/17 23:28:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:58604->10.242.13.79:8302: i/o timeout 2021/06/17 23:28:07 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:28:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:10 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.446341842s 2021/06/17 23:28:10 [ERR] raft: Failed to pipeline AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: write tcp 10.242.117.136:44660->10.242.13.79:8300: use of closed network connection 2021/06/17 23:28:10 [INFO] raft: aborting pipeline replication to peer {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300} 2021/06/17 23:28:10 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:12 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500231801s 2021/06/17 23:28:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:13 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:44154->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:14 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:15 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.970607955s 2021/06/17 23:28:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:17 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 7.396268498s 2021/06/17 23:28:17 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:58850->10.242.13.79:8302: i/o timeout 2021/06/17 23:28:17 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:28:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:20 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:47688->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:20 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:24 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:47774->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:25 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:25 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:28:25 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:28:25 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:28:26 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:28:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:59094->10.242.13.79:8302: i/o timeout 2021/06/17 23:28:27 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:28:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:30 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:47932->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:30 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:35 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48046->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:35 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:38 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:28:38 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:28:38 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:28:38 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:36092 2021/06/17 23:28:40 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:28:40 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:28:40 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48178->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:40 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:43 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:28:43 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:28:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:45 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48306->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:46 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:50 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48420->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:50 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:56 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48584->10.242.13.79:8300: i/o timeout 2021/06/17 23:28:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:28:57 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:28:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:00 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48676->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:00 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:07 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:48858->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:08 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:10 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:48934->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:10 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:16 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:29:16 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:29:16 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:29:16 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:29:16 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:29:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:18 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:54134->10.242.13.79:8301: i/o timeout 2021/06/17 23:29:18 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:29:18 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49120->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:19 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:29:19 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:29:20 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:54194->10.242.13.79:8301: i/o timeout 2021/06/17 23:29:20 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:29:20 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:49184->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:22 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:29:22 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:29:22 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:29:22 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:29:22 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:29:23 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:54266->10.242.13.79:8301: i/o timeout 2021/06/17 23:29:23 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:29:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:23 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:29:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:60602->10.242.13.79:8302: i/o timeout 2021/06/17 23:29:27 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:29:29 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49402->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:30 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:31 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:49446->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:31 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:40 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49688->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:41 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:49718->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:42 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:42 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:60978->10.242.13.79:8302: i/o timeout 2021/06/17 23:29:42 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:29:43 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:49992->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:52 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:32874->10.242.13.79:8302: i/o timeout 2021/06/17 23:29:52 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:29:53 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:50002->10.242.13.79:8300: i/o timeout 2021/06/17 23:29:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:55 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:29:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:29:57 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:29:57 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:29:57 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:29:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:01 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:30:01 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:30:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:33248->10.242.13.79:8302: i/o timeout 2021/06/17 23:30:02 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:30:03 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:30:04 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:30:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:05 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:50318->10.242.13.79:8300: i/o timeout 2021/06/17 23:30:05 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:50322->10.242.13.79:8300: i/o timeout 2021/06/17 23:30:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:10 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:30:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:11 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:30:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:20 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:50690->10.242.13.79:8300: i/o timeout 2021/06/17 23:30:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:21 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:50706->10.242.13.79:8300: i/o timeout 2021/06/17 23:30:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:31 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:30:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:32 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:30:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:41 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:51280->10.242.13.79:8300: i/o timeout 2021/06/17 23:30:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:42 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:51314->10.242.13.79:8300: i/o timeout 2021/06/17 23:30:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:43 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:30:43 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:30:43 [ERR] memberlist: failed to receive: read tcp 10.242.117.136:8302->10.242.13.79:60460: i/o timeout from=10.242.13.79:60460 2021/06/17 23:30:43 [ERR] memberlist: Failed to send error: write tcp 10.242.117.136:8302->10.242.13.79:60460: i/o timeout from=10.242.13.79:60460 2021/06/17 23:30:43 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:30:46 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:30:46 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:30:46 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:39576 2021/06/17 23:30:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:51 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:30:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:53 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:30:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:30:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:01 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:51770->10.242.13.79:8300: i/o timeout 2021/06/17 23:31:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:03 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:51816->10.242.13.79:8300: i/o timeout 2021/06/17 23:31:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:11 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:31:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:14 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:31:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:21 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:52248->10.242.13.79:8300: i/o timeout 2021/06/17 23:31:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:23 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:31:24 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:52310->10.242.13.79:8300: i/o timeout 2021/06/17 23:31:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:25 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:31:25 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:31:28 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:31:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:31 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:31:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:35 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:31:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:41 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:52738->10.242.13.79:8300: i/o timeout 2021/06/17 23:31:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:45 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:52812->10.242.13.79:8300: i/o timeout 2021/06/17 23:31:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:52 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:31:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:56 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:31:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:31:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:02 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:53226->10.242.13.79:8300: i/o timeout 2021/06/17 23:32:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:03 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:32:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:06 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:53332->10.242.13.79:8300: i/o timeout 2021/06/17 23:32:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:07 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:32:07 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:32:10 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:32:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:12 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:32:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:17 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:32:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:22 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:53716->10.242.13.79:8300: i/o timeout 2021/06/17 23:32:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:27 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:53840->10.242.13.79:8300: i/o timeout 2021/06/17 23:32:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:32 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:32:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:38 [WARN] Unable to get address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53, using fallback address 10.242.13.79:8300: Could not find address for server id 420ae614-e1ab-47c4-40cd-a971e2ac4c53 2021/06/17 23:32:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:42 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: read tcp 10.242.117.136:54196->10.242.13.79:8300: i/o timeout 2021/06/17 23:32:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:43 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:32:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:32:45 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:32:45 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:32:45 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-rx74l) 2021/06/17 23:32:45 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:32:45 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:32:45 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:32:46 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:32:46 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:32:46 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:32:46 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:32:46 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:32:46 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:32:47 [WARN] raft: Rejecting vote request from 10.242.13.79:8300 since we have a leader: 10.242.117.136:8300 2021/06/17 23:32:47 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1.dc1) 2021/06/17 23:32:47 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:32:47 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:32:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: EOF 2021/06/17 23:32:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:52 [ERR] raft: Failed to AppendEntries to {Voter 420ae614-e1ab-47c4-40cd-a971e2ac4c53 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:52 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:32:53 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: stream closed 2021/06/17 23:32:53 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:53 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 2.500223537s 2021/06/17 23:32:54 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:54 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:32:55 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:55 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:56 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:32:56 [WARN] raft: Failed to contact 420ae614-e1ab-47c4-40cd-a971e2ac4c53 in 4.985974743s 2021/06/17 23:32:56 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:32:56 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:32:56 [INFO] serf: EventMemberUpdate: dev-consul-server-1 2021/06/17 23:32:56 [INFO] raft: Updating configuration with RemoveServer (420ae614-e1ab-47c4-40cd-a971e2ac4c53, ) to [{Suffrage:Voter ID:10c77ddf-c1bf-3392-459d-f53273b4be0f Address:10.242.117.136:8300} {Suffrage:Voter ID:625429b0-2c33-6d50-2432-70aefdff073d Address:10.242.162.71:8300}] 2021/06/17 23:32:56 [INFO] raft: Removed peer 420ae614-e1ab-47c4-40cd-a971e2ac4c53, stopping replication after 929 2021/06/17 23:32:56 [INFO] consul: removed server with duplicate address: 10.242.13.79:8300 2021/06/17 23:32:56 [INFO] raft: Updating configuration with AddNonvoter (ebca623a-87c4-901f-68a8-6494fc8a0cda, 10.242.13.79:8300) to [{Suffrage:Voter ID:10c77ddf-c1bf-3392-459d-f53273b4be0f Address:10.242.117.136:8300} {Suffrage:Voter ID:625429b0-2c33-6d50-2432-70aefdff073d Address:10.242.162.71:8300} {Suffrage:Nonvoter ID:ebca623a-87c4-901f-68a8-6494fc8a0cda Address:10.242.13.79:8300}] 2021/06/17 23:32:56 [INFO] raft: Added peer ebca623a-87c4-901f-68a8-6494fc8a0cda, starting replication 2021/06/17 23:32:56 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:56 [WARN] raft: AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300} rejected, sending older logs (next: 1) 2021/06/17 23:32:56 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:56 [INFO] serf: EventMemberUpdate: dev-consul-server-1.dc1 2021/06/17 23:32:56 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:56 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:56 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [INFO] raft: pipelining replication to peer {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300} 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:58 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:32:59 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:00 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:00 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:01 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:01 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:02 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:03 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:04 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:05 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:06 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:06 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:07 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:08 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:08 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:09 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:10 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:11 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:11 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:12 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:13 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:14 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:14 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:15 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:16 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:16 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:17 [INFO] autopilot: Promoting Server (ID: "ebca623a-87c4-901f-68a8-6494fc8a0cda" Address: "10.242.13.79:8300") to voter 2021/06/17 23:33:17 [INFO] raft: Updating configuration with AddStaging (ebca623a-87c4-901f-68a8-6494fc8a0cda, 10.242.13.79:8300) to [{Suffrage:Voter ID:10c77ddf-c1bf-3392-459d-f53273b4be0f Address:10.242.117.136:8300} {Suffrage:Voter ID:625429b0-2c33-6d50-2432-70aefdff073d Address:10.242.162.71:8300} {Suffrage:Voter ID:ebca623a-87c4-901f-68a8-6494fc8a0cda Address:10.242.13.79:8300}] 2021/06/17 23:33:17 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:18 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:18 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:19 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:20 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:21 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:21 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:22 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:23 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:23 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:24 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:25 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:26 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:27 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:27 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:28 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:29 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:30 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:31 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:32 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:32 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:33 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:34 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:35 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:36 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:36 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:37 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:38 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:39 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:40 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:40 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:41 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:42 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:43 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:43 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:44 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:45 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:46 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:46 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:33:47 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:60884->10.242.13.79:8301: i/o timeout 2021/06/17 23:33:47 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:33:47 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:48 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:60948->10.242.13.79:8301: i/o timeout 2021/06/17 23:33:50 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:33:50 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:33:50 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 2.500598829s 2021/06/17 23:33:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:33:53 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:36902->10.242.13.74:8301: i/o timeout 2021/06/17 23:33:53 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:33:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:33:54 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 2.500223398s 2021/06/17 23:33:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:33:56 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:32850->10.242.13.79:8301: i/o timeout 2021/06/17 23:33:56 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:33:57 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 4.93677434s 2021/06/17 23:33:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:33:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:39140->10.242.13.79:8302: i/o timeout 2021/06/17 23:33:57 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:33:58 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:37050->10.242.13.74:8301: i/o timeout 2021/06/17 23:33:58 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:33:58 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:54808->10.242.13.79:8300: i/o timeout 2021/06/17 23:33:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:33:59 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:33:59 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 7.377665648s 2021/06/17 23:34:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:32982->10.242.13.79:8301: i/o timeout 2021/06/17 23:34:00 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:34:01 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:34:01 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:34:01 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:34:01 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:34:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:37172->10.242.13.74:8301: i/o timeout 2021/06/17 23:34:02 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:34:02 [INFO] raft: aborting pipeline replication to peer {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300} 2021/06/17 23:34:02 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:37218->10.242.13.74:8301: i/o timeout 2021/06/17 23:34:04 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:34:04 [INFO] memberlist: Marking dev-consul-7467bf7b46-rx74l as failed, suspect timeout reached (1 peer confirmations) 2021/06/17 23:34:04 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:34:04 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:34:04 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 2.500194441s 2021/06/17 23:34:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:37248->10.242.13.74:8301: i/o timeout 2021/06/17 23:34:06 [INFO] memberlist: Suspect dev-consul-7467bf7b46-rx74l has failed, no acks received 2021/06/17 23:34:06 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:34:06 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.13.79:41106 2021/06/17 23:34:07 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 4.962435116s 2021/06/17 23:34:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:07 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:34:07 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:34:07 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:39386->10.242.13.79:8302: i/o timeout 2021/06/17 23:34:07 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:34:09 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:54906->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:09 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 7.427073841s 2021/06/17 23:34:10 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:10 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:34:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:12 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:56498->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:12 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:15 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:34:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:20 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:56696->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:20 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:22 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:56746->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:22 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:22 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:39758->10.242.13.79:8302: i/o timeout 2021/06/17 23:34:22 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:34:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:26 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:34:26 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:34:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:30 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:56952->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:31 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:32 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:56992->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:32 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:32 [ERR] memberlist: Failed fallback ping: read tcp 10.242.117.136:39876->10.242.13.79:8302: i/o timeout 2021/06/17 23:34:32 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:34:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:41 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:57206->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:42 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:42 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:57236->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:42 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:49 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:34:49 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:34:52 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:34:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:57490->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:52 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:57496->10.242.13.79:8300: i/o timeout 2021/06/17 23:34:52 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:53 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:34:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:55 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:34:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:34:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:02 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:57734->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:02 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:03 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:57756->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:04 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:12 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:57984->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:13 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:35:14 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58024->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:16 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:23 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:58232->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:23 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:26 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58312->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:28 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:31 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:35:31 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:35:33 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:58486->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:34 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:35:35 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:35 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:35:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:38 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58624->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:41 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:45 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:58792->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:47 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:51 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:58958->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:35:57 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:35:57 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:59108->10.242.13.79:8300: i/o timeout 2021/06/17 23:35:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:02 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:36:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:07 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:59352->10.242.13.79:8300: i/o timeout 2021/06/17 23:36:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:12 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:59472->10.242.13.79:8300: i/o timeout 2021/06/17 23:36:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:13 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:36:13 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:36:15 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:36:16 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:36:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:18 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:36:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:23 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:36:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:28 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:59856->10.242.13.79:8300: i/o timeout 2021/06/17 23:36:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:33 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:59960->10.242.13.79:8300: i/o timeout 2021/06/17 23:36:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:39 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:36:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:43 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:36:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:49 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60458->10.242.13.79:8300: i/o timeout 2021/06/17 23:36:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:53 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:60550->10.242.13.79:8300: i/o timeout 2021/06/17 23:36:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:55 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:36:55 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:36:55 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-rx74l 10.242.13.74:8301 2021/06/17 23:36:58 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:36:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:36:59 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:37:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:03 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:37:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:09 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:60964->10.242.13.79:8300: i/o timeout 2021/06/17 23:37:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:13 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:32826->10.242.13.79:8300: i/o timeout 2021/06/17 23:37:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:37:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:20 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:37:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:23 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:37:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:30 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:33260->10.242.13.79:8300: i/o timeout 2021/06/17 23:37:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:33 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:33314->10.242.13.79:8300: i/o timeout 2021/06/17 23:37:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:35 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:37:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:37 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:37:37 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:37:40 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:37:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:42 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:37:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:43 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:37:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:52 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:33776->10.242.13.79:8300: i/o timeout 2021/06/17 23:37:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:53 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:33820->10.242.13.79:8300: i/o timeout 2021/06/17 23:37:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:37:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:02 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:38:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:04 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:38:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:12 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:34276->10.242.13.79:8300: i/o timeout 2021/06/17 23:38:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:14 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:34304->10.242.13.79:8300: i/o timeout 2021/06/17 23:38:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:16 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:38:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:19 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:38:19 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:38:22 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:38:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:23 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:38:24 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:38:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:33 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:34774->10.242.13.79:8300: i/o timeout 2021/06/17 23:38:34 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:34794->10.242.13.79:8300: i/o timeout 2021/06/17 23:38:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:43 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:38:44 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:38:44 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:38:45 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:54 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:35288->10.242.13.79:8300: i/o timeout 2021/06/17 23:38:54 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:35290->10.242.13.79:8300: i/o timeout 2021/06/17 23:38:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:56 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:38:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:38:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:01 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:39:01 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:39:04 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:39:04 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:39:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:05 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:39:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:13 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:14 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:35764->10.242.13.79:8300: i/o timeout 2021/06/17 23:39:15 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:15 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:35792->10.242.13.79:8300: i/o timeout 2021/06/17 23:39:17 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:19 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:21 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:23 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:25 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:39:25 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:26 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:39:27 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:29 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:31 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:33 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:35 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:36254->10.242.13.79:8300: i/o timeout 2021/06/17 23:39:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:36 [INFO] serf: attempting reconnect to dev-consul-server-1 10.242.13.79:8301 2021/06/17 23:39:36 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:36298->10.242.13.79:8300: i/o timeout 2021/06/17 23:39:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:43 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/17 23:39:43 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: EOF 2021/06/17 23:39:45 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:39:46 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:39:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:47 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:39:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:55 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: read tcp 10.242.117.136:36850->10.242.13.79:8300: i/o timeout 2021/06/17 23:39:57 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:39:57 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:36912->10.242.13.79:8300: i/o timeout 2021/06/17 23:39:59 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:01 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:03 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:05 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:05 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:07 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:09 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:09 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:09 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-xqj4q 10.242.166.75 2021/06/17 23:40:09 [INFO] consul: member 'dev-consul-7467bf7b46-xqj4q' joined, marking health alive 2021/06/17 23:40:11 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:12 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:40:12 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.79 2021/06/17 23:40:12 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:40:12 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:40:12 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:40:12 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:40:12 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:40:12 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-1) 2021/06/17 23:40:12 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:40:12 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' failed, marking health critical 2021/06/17 23:40:13 [INFO] raft: pipelining replication to peer {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300} 2021/06/17 23:40:13 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:40:13 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' joined, marking health alive 2021/06/17 23:40:20 [INFO] raft: aborting pipeline replication to peer {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300} 2021/06/17 23:40:20 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: read tcp 10.242.117.136:37508->10.242.13.79:8300: read: connection reset by peer 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:20 [INFO] serf: EventMemberLeave: dev-consul-7467bf7b46-rx74l 10.242.13.74 2021/06/17 23:40:20 [INFO] consul: member 'dev-consul-7467bf7b46-rx74l' left, deregistering 2021/06/17 23:40:20 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:21 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:21 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error making call: stream closed 2021/06/17 23:40:21 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:21 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:22 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:22 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:22 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 2.513094618s 2021/06/17 23:40:22 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:23 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:23 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:23 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:24 [INFO] memberlist: Suspect dev-consul-server-1 has failed, no acks received 2021/06/17 23:40:24 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:24 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:25 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 4.918574628s 2021/06/17 23:40:25 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:25 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:25 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:26 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:26 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:27 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:27 [INFO] memberlist: Marking dev-consul-server-1 as failed, suspect timeout reached (2 peer confirmations) 2021/06/17 23:40:27 [INFO] serf: EventMemberFailed: dev-consul-server-1 10.242.13.79 2021/06/17 23:40:27 [INFO] consul: Removing LAN server dev-consul-server-1 (Addr: tcp/10.242.13.79:8300) (DC: dc1) 2021/06/17 23:40:27 [INFO] consul: member 'dev-consul-server-1' failed, marking health critical 2021/06/17 23:40:27 [WARN] raft: Failed to contact ebca623a-87c4-901f-68a8-6494fc8a0cda in 7.363169682s 2021/06/17 23:40:27 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:27 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:27 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:40:28 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:28 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/17 23:40:29 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:29 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:29 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:30 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:31 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:31 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:31 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: getsockopt: connection refused 2021/06/17 23:40:32 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:32 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:34 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:35 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:37 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:37 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:40:39 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:41 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:41 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:42 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:40:43 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:43 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:40:46 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:47 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:48 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:40:49 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:51 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:40:51 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:53 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:55 [WARN] consul: error getting server health from "dev-consul-server-1": last request still outstanding 2021/06/17 23:40:55 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:40:57 [INFO] memberlist: Marking dev-consul-server-1.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/17 23:40:57 [INFO] serf: EventMemberFailed: dev-consul-server-1.dc1 10.242.13.79 2021/06/17 23:40:57 [INFO] consul: Handled member-failed event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/17 23:40:58 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/17 23:40:58 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.13.79:8302 Theirs: 10.242.13.116:8302 2021/06/17 23:40:58 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.13.79:8302 and 10.242.13.116:8302 are claiming 2021/06/17 23:40:58 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.13.116 2021/06/17 23:40:58 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.13.116:8300) (DC: dc1) 2021/06/17 23:40:58 [INFO] raft: Updating configuration with AddNonvoter (739634f4-5730-3ecf-590a-ea4f2bf04eaf, 10.242.13.116:8300) to [{Suffrage:Voter ID:10c77ddf-c1bf-3392-459d-f53273b4be0f Address:10.242.117.136:8300} {Suffrage:Voter ID:625429b0-2c33-6d50-2432-70aefdff073d Address:10.242.162.71:8300} {Suffrage:Voter ID:ebca623a-87c4-901f-68a8-6494fc8a0cda Address:10.242.13.79:8300} {Suffrage:Nonvoter ID:739634f4-5730-3ecf-590a-ea4f2bf04eaf Address:10.242.13.116:8300}] 2021/06/17 23:40:58 [INFO] raft: Added peer 739634f4-5730-3ecf-590a-ea4f2bf04eaf, starting replication 2021/06/17 23:40:58 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/17 23:40:58 [WARN] raft: AppendEntries to {Nonvoter 739634f4-5730-3ecf-590a-ea4f2bf04eaf 10.242.13.116:8300} rejected, sending older logs (next: 1) 2021/06/17 23:40:58 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:40:58 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.13.79:8302 Theirs: 10.242.13.116:8302 2021/06/17 23:40:58 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.13.79:8302 and 10.242.13.116:8302 are claiming 2021/06/17 23:40:58 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.13.79:8302 Theirs: 10.242.13.116:8302 2021/06/17 23:40:58 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.13.79:8302 and 10.242.13.116:8302 are claiming 2021/06/17 23:40:58 [INFO] raft: pipelining replication to peer {Nonvoter 739634f4-5730-3ecf-590a-ea4f2bf04eaf 10.242.13.116:8300} 2021/06/17 23:40:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.13.79:8302 Theirs: 10.242.13.116:8302 2021/06/17 23:40:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.13.79:8302 and 10.242.13.116:8302 are claiming 2021/06/17 23:40:59 [ERR] memberlist: Conflicting address for dev-consul-server-1.dc1. Mine: 10.242.13.79:8302 Theirs: 10.242.13.116:8302 2021/06/17 23:40:59 [WARN] serf: Name conflict for 'dev-consul-server-1.dc1' both 10.242.13.79:8302 and 10.242.13.116:8302 are claiming 2021/06/17 23:41:01 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:41:07 [WARN] consul: error getting server health from "dev-consul-server-1": rpc error getting client: failed to get conn: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:41:09 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:41:11 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:41:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:41:17 [INFO] autopilot: Promoting Server (ID: "739634f4-5730-3ecf-590a-ea4f2bf04eaf" Address: "10.242.13.116:8300") to voter 2021/06/17 23:41:17 [INFO] raft: Updating configuration with AddStaging (739634f4-5730-3ecf-590a-ea4f2bf04eaf, 10.242.13.116:8300) to [{Suffrage:Voter ID:10c77ddf-c1bf-3392-459d-f53273b4be0f Address:10.242.117.136:8300} {Suffrage:Voter ID:625429b0-2c33-6d50-2432-70aefdff073d Address:10.242.162.71:8300} {Suffrage:Voter ID:ebca623a-87c4-901f-68a8-6494fc8a0cda Address:10.242.13.79:8300} {Suffrage:Voter ID:739634f4-5730-3ecf-590a-ea4f2bf04eaf Address:10.242.13.116:8300}] 2021/06/17 23:41:17 [INFO] autopilot: Attempting removal of stale Server (ID: "ebca623a-87c4-901f-68a8-6494fc8a0cda" Address: "10.242.13.79:8300") 2021/06/17 23:41:17 [INFO] raft: Updating configuration with RemoveServer (ebca623a-87c4-901f-68a8-6494fc8a0cda, ) to [{Suffrage:Voter ID:10c77ddf-c1bf-3392-459d-f53273b4be0f Address:10.242.117.136:8300} {Suffrage:Voter ID:625429b0-2c33-6d50-2432-70aefdff073d Address:10.242.162.71:8300} {Suffrage:Voter ID:739634f4-5730-3ecf-590a-ea4f2bf04eaf Address:10.242.13.116:8300}] 2021/06/17 23:41:17 [INFO] raft: Removed peer ebca623a-87c4-901f-68a8-6494fc8a0cda, stopping replication after 1124 2021/06/17 23:41:19 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:41:21 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:41:25 [INFO] memberlist: Suspect dev-consul-server-1.dc1 has failed, no acks received 2021/06/17 23:41:30 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:41:31 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:41:40 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:41:42 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:41:51 [WARN] Unable to get address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda, using fallback address 10.242.13.79:8300: Could not find address for server id ebca623a-87c4-901f-68a8-6494fc8a0cda 2021/06/17 23:41:52 [ERR] raft: Failed to AppendEntries to {Nonvoter ebca623a-87c4-901f-68a8-6494fc8a0cda 10.242.13.79:8300}: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:42:01 [ERR] raft: Failed to heartbeat to 10.242.13.79:8300: dial tcp ->10.242.13.79:8300: i/o timeout 2021/06/17 23:42:13 [INFO] serf: attempting reconnect to dev-consul-server-1.dc1 10.242.13.79:8302 2021/06/17 23:42:16 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.13.116 2021/06/17 23:42:16 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan"