+ CONSUL_BIND= + [ -n ] + CONSUL_CLIENT= + [ -n ] + CONSUL_DATA_DIR=/consul/data + CONSUL_CONFIG_DIR=/consul/config + [ -n ] + [ a = - ] + [ agent = agent ] + shift + set -- consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui + [ consul = consul ] + stat -c %u /consul/data + id -u consul + [ 100 != 100 ] + stat -c %u /consul/config + id -u consul + [ 100 != 100 ] + [ ! -z ] + exec consul agent -data-dir=/consul/data -config-dir=/consul/config -bootstrap-expect=3 -enable-script-checks -retry-join=dev-consul-server-0.consul-server -retry-join=dev-consul-server-1.consul-server -retry-join=dev-consul-server-2.consul-server -client=0.0.0.0 -server -ui bootstrap_expect > 0: expecting 3 servers ==> Starting Consul agent... ==> Consul agent running! Version: 'v1.0.6' Node ID: '33937557-46d2-960a-31ca-f58895c96c3d' Node name: 'dev-consul-server-0' Datacenter: 'dc1' (Segment: '') Server: true (Bootstrap: false) Client Addr: [0.0.0.0] (HTTP: 8500, HTTPS: -1, DNS: 8600) Cluster Addr: 10.242.251.73 (LAN: 8301, WAN: 8302) Encrypt: Gossip: false, TLS-Outgoing: false, TLS-Incoming: false ==> Log data will now stream in as it occurs: 2021/06/06 10:39:52 [INFO] raft: Initial configuration (index=0): [] 2021/06/06 10:39:52 [INFO] raft: Node at 10.242.251.73:8300 [Follower] entering Follower state (Leader: "") 2021/06/06 10:39:52 [INFO] serf: EventMemberJoin: dev-consul-server-0.dc1 10.242.251.73 2021/06/06 10:39:52 [INFO] serf: EventMemberJoin: dev-consul-server-0 10.242.251.73 2021/06/06 10:39:52 [INFO] consul: Handled member-join event for server "dev-consul-server-0.dc1" in area "wan" 2021/06/06 10:39:52 [INFO] consul: Adding LAN server dev-consul-server-0 (Addr: tcp/10.242.251.73:8300) (DC: dc1) 2021/06/06 10:39:52 [INFO] agent: Started DNS server 0.0.0.0:8600 (tcp) 2021/06/06 10:39:52 [INFO] agent: Started DNS server 0.0.0.0:8600 (udp) 2021/06/06 10:39:52 [INFO] agent: Started HTTP server on [::]:8500 (tcp) 2021/06/06 10:39:52 [INFO] agent: started state syncer 2021/06/06 10:39:52 [INFO] agent: Retry join LAN is supported for: aliyun aws azure digitalocean gce os scaleway softlayer 2021/06/06 10:39:52 [INFO] agent: Joining LAN cluster... 2021/06/06 10:39:52 [INFO] agent: (LAN) joining: [dev-consul-server-0.consul-server dev-consul-server-1.consul-server dev-consul-server-2.consul-server] 2021/06/06 10:39:52 [WARN] memberlist: Failed to resolve dev-consul-server-0.consul-server: lookup dev-consul-server-0.consul-server on 10.96.0.10:53: no such host 2021/06/06 10:39:53 [WARN] memberlist: Failed to resolve dev-consul-server-1.consul-server: lookup dev-consul-server-1.consul-server on 10.96.0.10:53: no such host 2021/06/06 10:39:53 [WARN] memberlist: Failed to resolve dev-consul-server-2.consul-server: lookup dev-consul-server-2.consul-server on 10.96.0.10:53: no such host 2021/06/06 10:39:53 [INFO] agent: (LAN) joined: 0 Err: 3 error(s) occurred: * Failed to resolve dev-consul-server-0.consul-server: lookup dev-consul-server-0.consul-server on 10.96.0.10:53: no such host * Failed to resolve dev-consul-server-1.consul-server: lookup dev-consul-server-1.consul-server on 10.96.0.10:53: no such host * Failed to resolve dev-consul-server-2.consul-server: lookup dev-consul-server-2.consul-server on 10.96.0.10:53: no such host 2021/06/06 10:39:53 [WARN] agent: Join LAN failed: , retrying in 30s 2021/06/06 10:39:59 [ERR] agent: failed to sync remote state: No cluster leader 2021/06/06 10:40:01 [WARN] raft: no known peers, aborting election ==> Failed to check for updates: Get https://checkpoint-api.hashicorp.com/v1/check/consul?arch=amd64&os=linux&signature=7ab18714-a955-17bb-551e-3621a6d463f1&version=1.0.6: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers) 2021/06/06 10:40:18 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 10:40:19 [ERR] agent: Coordinate update error: No cluster leader 2021/06/06 10:40:23 [INFO] agent: (LAN) joining: [dev-consul-server-0.consul-server dev-consul-server-1.consul-server dev-consul-server-2.consul-server] 2021/06/06 10:40:23 [WARN] memberlist: Failed to resolve dev-consul-server-1.consul-server: lookup dev-consul-server-1.consul-server on 10.96.0.10:53: no such host 2021/06/06 10:40:23 [WARN] memberlist: Failed to resolve dev-consul-server-2.consul-server: lookup dev-consul-server-2.consul-server on 10.96.0.10:53: no such host 2021/06/06 10:40:23 [INFO] agent: (LAN) joined: 1 Err: 2021/06/06 10:40:23 [INFO] agent: Join LAN completed. Synced with 1 initial agents 2021/06/06 10:40:27 [ERR] agent: failed to sync remote state: No cluster leader 2021/06/06 10:40:35 [INFO] serf: EventMemberJoin: dev-consul-server-1 10.242.28.70 2021/06/06 10:40:35 [INFO] consul: Adding LAN server dev-consul-server-1 (Addr: tcp/10.242.28.70:8300) (DC: dc1) 2021/06/06 10:40:35 [INFO] serf: EventMemberJoin: dev-consul-server-1.dc1 10.242.28.70 2021/06/06 10:40:35 [INFO] consul: Handled member-join event for server "dev-consul-server-1.dc1" in area "wan" 2021/06/06 10:40:45 [ERR] agent: Coordinate update error: No cluster leader 2021/06/06 10:40:49 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.250.213 2021/06/06 10:40:49 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.250.213:8300) (DC: dc1) 2021/06/06 10:40:49 [INFO] consul: Found expected number of peers, attempting bootstrap: 10.242.28.70:8300,10.242.250.213:8300,10.242.251.73:8300 2021/06/06 10:40:49 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.250.213 2021/06/06 10:40:49 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/06 10:40:49 [ERR] agent: failed to sync remote state: No cluster leader 2021/06/06 10:40:53 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/06/06 10:40:53 [INFO] raft: Node at 10.242.251.73:8300 [Candidate] entering Candidate state in term 2 2021/06/06 10:40:54 [INFO] raft: Election won. Tally: 2 2021/06/06 10:40:54 [INFO] raft: Node at 10.242.251.73:8300 [Leader] entering Leader state 2021/06/06 10:40:54 [INFO] raft: Added peer a36601df-85d4-1f63-93f7-36d64b9ca9f6, starting replication 2021/06/06 10:40:54 [INFO] raft: Added peer 26770bf2-b099-b744-a1d7-56c3d305bf3f, starting replication 2021/06/06 10:40:54 [INFO] consul: cluster leadership acquired 2021/06/06 10:40:54 [INFO] consul: New leader elected: dev-consul-server-0 2021/06/06 10:40:54 [INFO] raft: pipelining replication to peer {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300} 2021/06/06 10:40:54 [INFO] consul: member 'dev-consul-server-2' joined, marking health alive 2021/06/06 10:40:54 [INFO] consul: member 'dev-consul-server-0' joined, marking health alive 2021/06/06 10:40:54 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' joined, marking health alive 2021/06/06 10:40:54 [INFO] consul: member 'dev-consul-server-1' joined, marking health alive 2021/06/06 10:40:54 [WARN] raft: AppendEntries to {Voter a36601df-85d4-1f63-93f7-36d64b9ca9f6 10.242.28.70:8300} rejected, sending older logs (next: 1) 2021/06/06 10:40:54 [INFO] raft: pipelining replication to peer {Voter a36601df-85d4-1f63-93f7-36d64b9ca9f6 10.242.28.70:8300} 2021/06/06 10:40:56 [INFO] agent: Synced node info 2021/06/06 11:02:17 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/06 11:02:17 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:02:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:02:37 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-8d2xr) 2021/06/06 11:02:48 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:34730->10.242.250.213:8301: i/o timeout 2021/06/06 11:02:48 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:02:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:46352->10.242.250.201:8301: i/o timeout 2021/06/06 11:02:50 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:02:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:46408->10.242.250.201:8301: i/o timeout 2021/06/06 11:02:57 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:02:58 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-8d2xr) 2021/06/06 11:03:10 [WARN] memberlist: Refuting a suspect message (from: dev-consul-7467bf7b46-8d2xr) 2021/06/06 11:03:50 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:35212->10.242.250.213:8301: i/o timeout 2021/06/06 11:03:50 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:03:54 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:46854->10.242.250.201:8301: i/o timeout 2021/06/06 11:03:54 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:03:55 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:03:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:03:57 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.500221961s 2021/06/06 11:03:57 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:35256->10.242.250.213:8301: i/o timeout 2021/06/06 11:03:57 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:03:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:03:59 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 4.950917619s 2021/06/06 11:04:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:00 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:46882->10.242.250.201:8301: i/o timeout 2021/06/06 11:04:00 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:04:02 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 7.391181976s 2021/06/06 11:04:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:47856->10.242.250.213:8302: i/o timeout 2021/06/06 11:04:02 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:04:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:46904->10.242.250.201:8301: i/o timeout 2021/06/06 11:04:02 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:04:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:04 [INFO] raft: aborting pipeline replication to peer {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300} 2021/06/06 11:04:04 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:35318->10.242.250.213:8301: i/o timeout 2021/06/06 11:04:04 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:04:05 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:54318->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:05 [INFO] memberlist: Marking dev-consul-server-2 as failed, suspect timeout reached (1 peer confirmations) 2021/06/06 11:04:05 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.250.213 2021/06/06 11:04:05 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.250.213:8300) (DC: dc1) 2021/06/06 11:04:05 [INFO] consul: member 'dev-consul-server-2' failed, marking health critical 2021/06/06 11:04:06 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:06 [INFO] memberlist: Marking dev-consul-7467bf7b46-8d2xr as failed, suspect timeout reached (1 peer confirmations) 2021/06/06 11:04:06 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:04:06 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' failed, marking health critical 2021/06/06 11:04:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:06 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:46946->10.242.250.201:8301: i/o timeout 2021/06/06 11:04:06 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:04:07 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.500235486s 2021/06/06 11:04:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:09 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 4.990592781s 2021/06/06 11:04:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:12 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 7.450731753s 2021/06/06 11:04:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:47910->10.242.250.213:8302: i/o timeout 2021/06/06 11:04:12 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:04:12 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:04:12 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.250.201:55804 2021/06/06 11:04:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:14 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34448->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:14 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:16 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34470->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:17 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:22 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:04:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:24 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34522->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:24 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:27 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34536->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:27 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:48050->10.242.250.213:8302: i/o timeout 2021/06/06 11:04:27 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:04:28 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:29 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:04:29 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:04:29 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.250.213:40834 2021/06/06 11:04:29 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:04:31 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:04:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:32 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/06 11:04:32 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.250.213 2021/06/06 11:04:32 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/06 11:04:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:34 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34596->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:34 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:37 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:48120->10.242.250.213:8302: i/o timeout 2021/06/06 11:04:37 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:04:38 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34618->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:39 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:44 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34670->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:44 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:49 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34688->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:50 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:54 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:54 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34736->10.242.250.213:8300: i/o timeout 2021/06/06 11:04:54 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:04:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:04:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:00 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34764->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:01 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:02 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:05:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:04 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34808->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:05 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:10 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:05:10 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:05:11 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34846->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:12 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:13 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:05:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:15 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34896->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:15 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:22 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:34936->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:22 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.250.213:8302 2021/06/06 11:05:23 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:25 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:34966->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:26 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:33 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35016->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:35 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:36 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35040->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:37 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:42 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-8d2xr 10.242.250.201:8301 2021/06/06 11:05:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:45 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35102->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:47 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35110->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:48 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:49 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:05:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:52 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:05:52 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:05:55 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:05:55 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/06 11:05:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:05:58 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35184->10.242.250.213:8300: i/o timeout 2021/06/06 11:05:59 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35192->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:04 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:06:05 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:06:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:14 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35298->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:15 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35310->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:22 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:06:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:25 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:06:25 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:06:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:34 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:06:34 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:06:35 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35468->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:35 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35470->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:37 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:06:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:45 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:06:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:46 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:06:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:52 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.250.213:8302 2021/06/06 11:06:54 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:55 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35610->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:06:56 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35616->10.242.250.213:8300: i/o timeout 2021/06/06 11:06:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:02 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-8d2xr 10.242.250.201:8301 2021/06/06 11:07:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:05 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:07:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:07 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:07:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:15 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35766->10.242.250.213:8300: i/o timeout 2021/06/06 11:07:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:16 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:07:16 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:07:17 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35774->10.242.250.213:8300: i/o timeout 2021/06/06 11:07:19 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:07:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:26 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:07:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:28 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:07:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:36 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:35910->10.242.250.213:8300: i/o timeout 2021/06/06 11:07:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:38 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:35926->10.242.250.213:8300: i/o timeout 2021/06/06 11:07:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:42 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-8d2xr 10.242.250.201:8301 2021/06/06 11:07:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:46 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:07:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:49 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:07:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:52 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.250.213:8302 2021/06/06 11:07:54 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:56 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36054->10.242.250.213:8300: i/o timeout 2021/06/06 11:07:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:58 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:07:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:07:58 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:07:59 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:36068->10.242.250.213:8300: i/o timeout 2021/06/06 11:08:01 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:08:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:06 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:08:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:10 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:08:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:16 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36200->10.242.250.213:8300: i/o timeout 2021/06/06 11:08:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:20 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:36224->10.242.250.213:8300: i/o timeout 2021/06/06 11:08:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:22 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-8d2xr 10.242.250.201:8301 2021/06/06 11:08:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:26 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:08:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:31 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:08:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:36 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36342->10.242.250.213:8300: i/o timeout 2021/06/06 11:08:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:40 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:08:40 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:08:41 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:36374->10.242.250.213:8300: i/o timeout 2021/06/06 11:08:43 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:08:43 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/06 11:08:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:46 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:08:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:51 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:08:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:54 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:08:56 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36480->10.242.250.213:8300: i/o timeout 2021/06/06 11:08:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:01 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:36514->10.242.250.213:8300: i/o timeout 2021/06/06 11:09:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:02 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:09:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:07 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:09:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:12 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:09:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:17 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36630->10.242.250.213:8300: i/o timeout 2021/06/06 11:09:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:22 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:09:22 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:09:22 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:36670->10.242.250.213:8300: i/o timeout 2021/06/06 11:09:25 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:09:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:27 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:09:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:33 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:09:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:37 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36784->10.242.250.213:8300: i/o timeout 2021/06/06 11:09:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:42 [INFO] serf: attempting reconnect to dev-consul-7467bf7b46-8d2xr 10.242.250.201:8301 2021/06/06 11:09:43 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:36834->10.242.250.213:8300: i/o timeout 2021/06/06 11:09:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:47 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-5qlm6 10.242.225.217 2021/06/06 11:09:47 [INFO] consul: member 'dev-consul-7467bf7b46-5qlm6' joined, marking health alive 2021/06/06 11:09:47 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:09:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:54 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:54 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:09:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:09:57 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:36990->10.242.250.213:8300: i/o timeout 2021/06/06 11:09:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:04 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:10:04 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:10:04 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:37074->10.242.250.213:8300: i/o timeout 2021/06/06 11:10:07 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:10:07 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:10:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:15 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:10:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:17 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:37166->10.242.250.213:8300: i/o timeout 2021/06/06 11:10:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:22 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.250.213:8302 2021/06/06 11:10:22 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:10:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:25 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:37228->10.242.250.213:8300: i/o timeout 2021/06/06 11:10:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:28 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:10:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:36 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:10:38 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:37336->10.242.250.213:8300: i/o timeout 2021/06/06 11:10:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:46 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:10:46 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:10:46 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:37398->10.242.250.213:8300: i/o timeout 2021/06/06 11:10:48 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:10:49 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:10:49 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/06 11:10:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:52 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:54 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:56 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:58 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:10:58 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:10:58 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:37518->10.242.250.213:8300: i/o timeout 2021/06/06 11:11:00 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:02 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:11:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:08 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:37618->10.242.250.213:8300: i/o timeout 2021/06/06 11:11:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:08 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:10 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:18 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:37704->10.242.250.213:8300: i/o timeout 2021/06/06 11:11:19 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:24 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:25 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.500297633s 2021/06/06 11:11:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:28 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:11:28 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:11:28 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 4.916753173s 2021/06/06 11:11:28 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:28 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:11:28 [ERR] consul.rpc: multiplex conn accept failed: keepalive timeout from=10.242.250.201:58476 2021/06/06 11:11:30 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 7.414135882s 2021/06/06 11:11:31 [ERR] memberlist: failed to receive: read tcp 10.242.251.73:8301->10.242.250.213:39588: i/o timeout from=10.242.250.213:39588 2021/06/06 11:11:31 [ERR] memberlist: Failed to send error: write tcp 10.242.251.73:8301->10.242.250.213:39588: i/o timeout from=10.242.250.213:39588 2021/06/06 11:11:31 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:11:32 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:34 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:37814->10.242.250.213:8300: i/o timeout 2021/06/06 11:11:34 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:35 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:36 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:38 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:38 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:37898->10.242.250.213:8300: i/o timeout 2021/06/06 11:11:40 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:40 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.250.213 2021/06/06 11:11:40 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/06 11:11:41 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:42 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:11:42 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' joined, marking health alive 2021/06/06 11:11:42 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:42 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.500181429s 2021/06/06 11:11:44 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:45 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:45 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:50526->10.242.250.201:8301: i/o timeout 2021/06/06 11:11:45 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:11:46 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:46 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:47 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:48 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.250.213 2021/06/06 11:11:48 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.250.213:8300) (DC: dc1) 2021/06/06 11:11:48 [INFO] consul: member 'dev-consul-server-2' joined, marking health alive 2021/06/06 11:11:48 [WARN] memberlist: Refuting a suspect message (from: dev-consul-server-2.dc1) 2021/06/06 11:11:48 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:48 [DEBUG] raft-net: 10.242.251.73:8300 accepted connection from: 10.242.250.213:38274 2021/06/06 11:11:48 [WARN] raft: Rejecting vote request from 10.242.250.213:8300 since we have a leader: 10.242.251.73:8300 2021/06/06 11:11:48 [WARN] memberlist: Was able to connect to dev-consul-7467bf7b46-8d2xr but other probes failed, network may be misconfigured 2021/06/06 11:11:49 [INFO] memberlist: Marking dev-consul-7467bf7b46-8d2xr as failed, suspect timeout reached (2 peer confirmations) 2021/06/06 11:11:49 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:11:49 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' failed, marking health critical 2021/06/06 11:11:49 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:38946->10.242.250.213:8301: i/o timeout 2021/06/06 11:11:49 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:11:50 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:11:50 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.500268956s 2021/06/06 11:11:51 [ERR] raft: peer {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300} has newer term, stopping replication 2021/06/06 11:11:51 [INFO] raft: Node at 10.242.251.73:8300 [Follower] entering Follower state (Leader: "") 2021/06/06 11:11:51 [INFO] raft: aborting pipeline replication to peer {Voter a36601df-85d4-1f63-93f7-36d64b9ca9f6 10.242.28.70:8300} 2021/06/06 11:11:51 [INFO] consul: cluster leadership lost 2021/06/06 11:11:52 [WARN] consul.coordinate: Batch update failed: node is not the leader 2021/06/06 11:11:54 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.250.213 2021/06/06 11:11:54 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.250.213:8300) (DC: dc1) 2021/06/06 11:11:54 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:39000->10.242.250.213:8301: i/o timeout 2021/06/06 11:11:54 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:11:58 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/06/06 11:11:58 [INFO] raft: Node at 10.242.251.73:8300 [Candidate] entering Candidate state in term 3 2021/06/06 11:11:58 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:58 [INFO] raft: Election won. Tally: 2 2021/06/06 11:11:58 [INFO] raft: Node at 10.242.251.73:8300 [Leader] entering Leader state 2021/06/06 11:11:58 [INFO] raft: Added peer a36601df-85d4-1f63-93f7-36d64b9ca9f6, starting replication 2021/06/06 11:11:58 [INFO] raft: Added peer 26770bf2-b099-b744-a1d7-56c3d305bf3f, starting replication 2021/06/06 11:11:58 [INFO] consul: cluster leadership acquired 2021/06/06 11:11:58 [INFO] consul: New leader elected: dev-consul-server-0 2021/06/06 11:11:58 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:11:58 [INFO] raft: pipelining replication to peer {Voter a36601df-85d4-1f63-93f7-36d64b9ca9f6 10.242.28.70:8300} 2021/06/06 11:11:58 [INFO] consul: member 'dev-consul-server-2' failed, marking health critical 2021/06/06 11:11:59 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:01 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.501298282s 2021/06/06 11:12:01 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:12:01 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/06 11:12:02 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:02 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:51644->10.242.250.213:8302: i/o timeout 2021/06/06 11:12:02 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:12:03 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 4.954465709s 2021/06/06 11:12:04 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:06 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 7.414860085s 2021/06/06 11:12:06 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:08 [ERR] raft: Failed to make RequestVote RPC to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:38062->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:08 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:37954->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:08 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:08 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:08 [ERR] memberlist: failed to receive: read tcp 10.242.251.73:8301->10.242.250.201:55120: i/o timeout from=10.242.250.201:55120 2021/06/06 11:12:08 [ERR] memberlist: Failed to send error: write tcp 10.242.251.73:8301->10.242.250.201:55120: i/o timeout from=10.242.250.201:55120 2021/06/06 11:12:09 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:38178->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:10 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:10 [ERR] yamux: keepalive failed: i/o deadline reached 2021/06/06 11:12:10 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: EOF 2021/06/06 11:12:11 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:12:12 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:12 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:51722->10.242.250.213:8302: i/o timeout 2021/06/06 11:12:12 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:12:12 [INFO] serf: attempting reconnect to dev-consul-server-2 10.242.250.213:8301 2021/06/06 11:12:14 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:15 [WARN] raft: Rejecting vote request from 10.242.250.213:8300 since we have a leader: 10.242.251.73:8300 2021/06/06 11:12:16 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:18 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:18 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:38252->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:18 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:20 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:38266->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:20 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:20 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:22 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:23 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:12:23 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' joined, marking health alive 2021/06/06 11:12:24 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:24 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:50892->10.242.250.201:8301: i/o timeout 2021/06/06 11:12:24 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:12:26 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:28 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:28 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: read tcp 10.242.251.73:38350->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:28 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:28 [ERR] memberlist: Failed fallback ping: read tcp 10.242.251.73:50944->10.242.250.201:8301: i/o timeout 2021/06/06 11:12:28 [INFO] memberlist: Suspect dev-consul-7467bf7b46-8d2xr has failed, no acks received 2021/06/06 11:12:28 [INFO] memberlist: Marking dev-consul-7467bf7b46-8d2xr as failed, suspect timeout reached (2 peer confirmations) 2021/06/06 11:12:28 [INFO] serf: EventMemberFailed: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:12:28 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' failed, marking health critical 2021/06/06 11:12:30 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:30 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: read tcp 10.242.251.73:38370->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:31 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.250.213 2021/06/06 11:12:31 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.250.213:8300) (DC: dc1) 2021/06/06 11:12:31 [INFO] consul: member 'dev-consul-server-2' joined, marking health alive 2021/06/06 11:12:32 [WARN] raft: Rejecting vote request from 10.242.250.213:8300 since we have a leader: 10.242.251.73:8300 2021/06/06 11:12:32 [ERR] raft: peer {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300} has newer term, stopping replication 2021/06/06 11:12:32 [INFO] raft: Node at 10.242.251.73:8300 [Follower] entering Follower state (Leader: "") 2021/06/06 11:12:32 [INFO] raft: aborting pipeline replication to peer {Voter a36601df-85d4-1f63-93f7-36d64b9ca9f6 10.242.28.70:8300} 2021/06/06 11:12:32 [INFO] consul: cluster leadership lost 2021/06/06 11:12:32 [INFO] serf: EventMemberJoin: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:12:32 [WARN] consul.coordinate: Batch update failed: node is not the leader 2021/06/06 11:12:39 [WARN] raft: Rejecting vote request from 10.242.250.213:8300 since our last term is greater (3, 2) 2021/06/06 11:12:39 [INFO] serf: EventMemberLeave: dev-consul-7467bf7b46-8d2xr 10.242.250.201 2021/06/06 11:12:41 [WARN] raft: Heartbeat timeout from "" reached, starting election 2021/06/06 11:12:41 [INFO] raft: Node at 10.242.251.73:8300 [Candidate] entering Candidate state in term 7 2021/06/06 11:12:41 [ERR] raft: Failed to make RequestVote RPC to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: EOF 2021/06/06 11:12:41 [INFO] raft: Election won. Tally: 2 2021/06/06 11:12:41 [INFO] raft: Node at 10.242.251.73:8300 [Leader] entering Leader state 2021/06/06 11:12:41 [INFO] raft: Added peer a36601df-85d4-1f63-93f7-36d64b9ca9f6, starting replication 2021/06/06 11:12:41 [INFO] raft: Added peer 26770bf2-b099-b744-a1d7-56c3d305bf3f, starting replication 2021/06/06 11:12:41 [INFO] consul: cluster leadership acquired 2021/06/06 11:12:41 [INFO] consul: New leader elected: dev-consul-server-0 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: EOF 2021/06/06 11:12:41 [INFO] raft: pipelining replication to peer {Voter a36601df-85d4-1f63-93f7-36d64b9ca9f6 10.242.28.70:8300} 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:41 [INFO] consul: member 'dev-consul-7467bf7b46-8d2xr' left, deregistering 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:41 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:41 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:42 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:42 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:42 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:42 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:12:43 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error making call: stream closed 2021/06/06 11:12:43 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:43 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 2.503279965s 2021/06/06 11:12:44 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:44 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:12:44 [WARN] consul: error getting server health from "dev-consul-server-1": context deadline exceeded 2021/06/06 11:12:44 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: getsockopt: connection refused 2021/06/06 11:12:45 [INFO] serf: EventMemberFailed: dev-consul-server-2 10.242.250.213 2021/06/06 11:12:45 [INFO] consul: Removing LAN server dev-consul-server-2 (Addr: tcp/10.242.250.213:8300) (DC: dc1) 2021/06/06 11:12:45 [INFO] consul: member 'dev-consul-server-2' failed, marking health critical 2021/06/06 11:12:45 [INFO] memberlist: Suspect dev-consul-server-2 has failed, no acks received 2021/06/06 11:12:46 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 4.996529825s 2021/06/06 11:12:46 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:12:46 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:47 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:48 [WARN] raft: Failed to contact 26770bf2-b099-b744-a1d7-56c3d305bf3f in 7.462843s 2021/06/06 11:12:49 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:51 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:53 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:55 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:12:55 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error getting client: failed to get conn: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:55 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:55 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:12:56 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:12:58 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:12:58 [WARN] consul: error getting server health from "dev-consul-server-0": context deadline exceeded 2021/06/06 11:12:59 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:01 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:01 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:03 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:05 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:05 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:06 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:07 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:07 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error getting client: failed to get conn: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:10 [WARN] consul: error getting server health from "dev-consul-server-2": context deadline exceeded 2021/06/06 11:13:11 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:11 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:13 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:15 [WARN] consul: error getting server health from "dev-consul-server-2": last request still outstanding 2021/06/06 11:13:15 [INFO] memberlist: Marking dev-consul-server-2.dc1 as failed, suspect timeout reached (0 peer confirmations) 2021/06/06 11:13:15 [INFO] serf: EventMemberFailed: dev-consul-server-2.dc1 10.242.250.213 2021/06/06 11:13:15 [INFO] consul: Handled member-failed event for server "dev-consul-server-2.dc1" in area "wan" 2021/06/06 11:13:15 [INFO] memberlist: Suspect dev-consul-server-2.dc1 has failed, no acks received 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2. Mine: 10.242.250.213:8301 Theirs: 10.242.250.243:8301 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2' both 10.242.250.213:8301 and 10.242.250.243:8301 are claiming 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2.dc1. Mine: 10.242.250.213:8302 Theirs: 10.242.250.243:8302 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2.dc1' both 10.242.250.213:8302 and 10.242.250.243:8302 are claiming 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2. Mine: 10.242.250.213:8301 Theirs: 10.242.250.243:8301 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2' both 10.242.250.213:8301 and 10.242.250.243:8301 are claiming 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2. Mine: 10.242.250.213:8301 Theirs: 10.242.250.243:8301 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2' both 10.242.250.213:8301 and 10.242.250.243:8301 are claiming 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2. Mine: 10.242.250.213:8301 Theirs: 10.242.250.243:8301 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2' both 10.242.250.213:8301 and 10.242.250.243:8301 are claiming 2021/06/06 11:13:16 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:16 [INFO] serf: EventMemberJoin: dev-consul-server-2 10.242.250.243 2021/06/06 11:13:16 [INFO] consul: Adding LAN server dev-consul-server-2 (Addr: tcp/10.242.250.243:8300) (DC: dc1) 2021/06/06 11:13:16 [INFO] raft: Updating configuration with AddNonvoter (2d48a069-8891-9dbb-ccdc-d3b965c27227, 10.242.250.243:8300) to [{Suffrage:Voter ID:a36601df-85d4-1f63-93f7-36d64b9ca9f6 Address:10.242.28.70:8300} {Suffrage:Voter ID:26770bf2-b099-b744-a1d7-56c3d305bf3f Address:10.242.250.213:8300} {Suffrage:Voter ID:33937557-46d2-960a-31ca-f58895c96c3d Address:10.242.251.73:8300} {Suffrage:Nonvoter ID:2d48a069-8891-9dbb-ccdc-d3b965c27227 Address:10.242.250.243:8300}] 2021/06/06 11:13:16 [INFO] raft: Added peer 2d48a069-8891-9dbb-ccdc-d3b965c27227, starting replication 2021/06/06 11:13:16 [INFO] consul: member 'dev-consul-server-2' joined, marking health alive 2021/06/06 11:13:16 [WARN] raft: AppendEntries to {Nonvoter 2d48a069-8891-9dbb-ccdc-d3b965c27227 10.242.250.243:8300} rejected, sending older logs (next: 1) 2021/06/06 11:13:16 [INFO] raft: pipelining replication to peer {Nonvoter 2d48a069-8891-9dbb-ccdc-d3b965c27227 10.242.250.243:8300} 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2.dc1. Mine: 10.242.250.213:8302 Theirs: 10.242.250.243:8302 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2.dc1' both 10.242.250.213:8302 and 10.242.250.243:8302 are claiming 2021/06/06 11:13:16 [ERR] memberlist: Conflicting address for dev-consul-server-2.dc1. Mine: 10.242.250.213:8302 Theirs: 10.242.250.243:8302 2021/06/06 11:13:16 [WARN] serf: Name conflict for 'dev-consul-server-2.dc1' both 10.242.250.213:8302 and 10.242.250.243:8302 are claiming 2021/06/06 11:13:17 [ERR] memberlist: Conflicting address for dev-consul-server-2.dc1. Mine: 10.242.250.213:8302 Theirs: 10.242.250.243:8302 2021/06/06 11:13:17 [WARN] serf: Name conflict for 'dev-consul-server-2.dc1' both 10.242.250.213:8302 and 10.242.250.243:8302 are claiming 2021/06/06 11:13:17 [ERR] memberlist: Conflicting address for dev-consul-server-2.dc1. Mine: 10.242.250.213:8302 Theirs: 10.242.250.243:8302 2021/06/06 11:13:17 [WARN] serf: Name conflict for 'dev-consul-server-2.dc1' both 10.242.250.213:8302 and 10.242.250.243:8302 are claiming 2021/06/06 11:13:17 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:19 [WARN] consul: error getting server health from "dev-consul-server-2": rpc error getting client: failed to get conn: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:21 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:22 [INFO] serf: attempting reconnect to dev-consul-server-2.dc1 10.242.250.213:8302 2021/06/06 11:13:27 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:29 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:31 [INFO] autopilot: Promoting Server (ID: "2d48a069-8891-9dbb-ccdc-d3b965c27227" Address: "10.242.250.243:8300") to voter 2021/06/06 11:13:31 [INFO] raft: Updating configuration with AddStaging (2d48a069-8891-9dbb-ccdc-d3b965c27227, 10.242.250.243:8300) to [{Suffrage:Voter ID:a36601df-85d4-1f63-93f7-36d64b9ca9f6 Address:10.242.28.70:8300} {Suffrage:Voter ID:26770bf2-b099-b744-a1d7-56c3d305bf3f Address:10.242.250.213:8300} {Suffrage:Voter ID:33937557-46d2-960a-31ca-f58895c96c3d Address:10.242.251.73:8300} {Suffrage:Voter ID:2d48a069-8891-9dbb-ccdc-d3b965c27227 Address:10.242.250.243:8300}] 2021/06/06 11:13:31 [INFO] autopilot: Attempting removal of stale Server (ID: "26770bf2-b099-b744-a1d7-56c3d305bf3f" Address: "10.242.250.213:8300") 2021/06/06 11:13:31 [INFO] raft: Updating configuration with RemoveServer (26770bf2-b099-b744-a1d7-56c3d305bf3f, ) to [{Suffrage:Voter ID:a36601df-85d4-1f63-93f7-36d64b9ca9f6 Address:10.242.28.70:8300} {Suffrage:Voter ID:33937557-46d2-960a-31ca-f58895c96c3d Address:10.242.251.73:8300} {Suffrage:Voter ID:2d48a069-8891-9dbb-ccdc-d3b965c27227 Address:10.242.250.243:8300}] 2021/06/06 11:13:31 [INFO] raft: Removed peer 26770bf2-b099-b744-a1d7-56c3d305bf3f, stopping replication after 575 2021/06/06 11:13:31 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:39 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:41 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:42 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:13:51 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:52 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:13:54 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:14:02 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:14:04 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:14:10 [WARN] Unable to get address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f, using fallback address 10.242.250.213:8300: Could not find address for server id 26770bf2-b099-b744-a1d7-56c3d305bf3f 2021/06/06 11:14:12 [ERR] raft: Failed to AppendEntries to {Voter 26770bf2-b099-b744-a1d7-56c3d305bf3f 10.242.250.213:8300}: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:14:20 [ERR] raft: Failed to heartbeat to 10.242.250.213:8300: dial tcp ->10.242.250.213:8300: i/o timeout 2021/06/06 11:14:58 [INFO] serf: EventMemberJoin: dev-consul-server-2.dc1 10.242.250.243 2021/06/06 11:14:58 [INFO] consul: Handled member-join event for server "dev-consul-server-2.dc1" in area "wan"