at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:36:54.825+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 486 message to send. [2023-03-11T19:36:54.826+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23120, above threshold 10. [2023-03-11T19:36:55.861+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 486 msgs to /events/POLICY-PDP-PAP. Oldest: 58282216 ms [2023-03-11T19:36:55.862+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:36:55.862+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:36:55.882+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:55.882+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:55.889+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:55.889+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:55.893+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:55.893+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:55.896+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:36:55.897+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:36:55.897+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 486 message to send. [2023-03-11T19:36:55.897+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23121, above threshold 10. [2023-03-11T19:36:56.911+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 486 msgs to /events/POLICY-PDP-PAP. Oldest: 58283266 ms [2023-03-11T19:36:56.911+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:36:56.912+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:36:56.917+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:36:56.918+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:56.934+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:36:56.935+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:56.943+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:56.944+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:56.959+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:36:56.959+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:36:56.959+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 486 message to send. [2023-03-11T19:36:56.960+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23122, above threshold 10. [2023-03-11T19:36:57.961+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 486 msgs to /events/POLICY-PDP-PAP. Oldest: 58284316 ms [2023-03-11T19:36:57.962+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:36:57.965+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:36:57.976+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:57.976+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:57.984+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:57.984+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:58.017+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:58.018+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:58.028+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:36:58.029+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:36:58.029+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 486 message to send. [2023-03-11T19:36:58.029+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23123, above threshold 10. [2023-03-11T19:36:59.061+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 486 msgs to /events/POLICY-PDP-PAP. Oldest: 58285416 ms [2023-03-11T19:36:59.064+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:36:59.064+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:36:59.076+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:59.076+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:59.084+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:59.084+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:59.108+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:36:59.108+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:59.117+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:36:59.117+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:36:59.117+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 486 message to send. [2023-03-11T19:36:59.117+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23124, above threshold 10. [2023-03-11T19:36:59.888+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch java.net.SocketException: Connection reset at java.base/java.net.SocketInputStream.read(SocketInputStream.java:186) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:36:59.888+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:36:59.888+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:36:59.889+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:36:59.892+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:36:59.892+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:59.895+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:36:59.895+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:59.898+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:36:59.898+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:36:59.906+00:00|WARN|HttpClient|DMAAP-source-POLICY-PDP-PAP] Error executing HTTP request. message-router:3904 failed to respond; blacklisting for 2 minutes [2023-03-11T19:36:59.906+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of message-router:3904 failed to respond [2023-03-11T19:36:59.906+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-03-11T19:37:00.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 486 msgs to /events/POLICY-PDP-PAP. Oldest: 58286516 ms [2023-03-11T19:37:00.162+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:37:00.162+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:37:14.907+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch org.apache.http.NoHttpResponseException: message-router:3904 failed to respond at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:37:14.909+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:37:14.909+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:37:14.909+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:37:14.925+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (org.apache.http.NoHttpResponseException) caught when processing request to {}->http://message-router:3904: The target server failed to respond [2023-03-11T19:37:14.925+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:14.934+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (org.apache.http.NoHttpResponseException) caught when processing request to {}->http://message-router:3904: The target server failed to respond [2023-03-11T19:37:14.934+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:14.946+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:37:14.946+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:14.949+00:00|WARN|HttpClient|DMAAP-source-POLICY-PDP-PAP] Error executing HTTP request. Connection reset; blacklisting for 2 minutes [2023-03-11T19:37:14.949+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of Connection reset [2023-03-11T19:37:14.949+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-03-11T19:37:29.950+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch java.net.SocketException: Connection reset at java.base/java.net.SocketInputStream.read(SocketInputStream.java:186) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:37:29.951+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:37:29.951+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:37:29.951+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:37:29.964+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:37:29.964+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:29.982+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:37:29.983+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:30.006+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:37:30.006+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:30.020+00:00|WARN|HttpClient|DMAAP-source-POLICY-PDP-PAP] Error executing HTTP request. Connection reset; blacklisting for 2 minutes [2023-03-11T19:37:30.020+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of Connection reset [2023-03-11T19:37:30.020+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-03-11T19:37:32.888+00:00|WARN|PdpMessageHandler|Timer-0] not registered: object:engine/apex/handler [2023-03-11T19:37:32.890+00:00|INFO|network|Timer-0] [OUT|DMAAP|POLICY-PDP-PAP] {"pdpType":"apex","state":"PASSIVE","healthy":"HEALTHY","description":"Pdp Heartbeat","statistics":{"pdpInstanceId":"apex-7fd94bb6-8411-46e5-8859-f3d0db7bc3be","timeStamp":"2023-03-11T19:37:32.888628Z","pdpGroupName":"defaultGroup","policyExecutedCount":0,"policyExecutedSuccessCount":0,"policyExecutedFailCount":0,"policyDeployCount":0,"policyDeploySuccessCount":0,"policyDeployFailCount":0,"policyUndeployCount":0,"policyUndeploySuccessCount":0,"policyUndeployFailCount":0},"messageName":"PDP_STATUS","requestId":"897fe716-6d04-4ac4-b5d9-e57fac73c7a5","timestampMs":1678563452888,"name":"apex-7fd94bb6-8411-46e5-8859-f3d0db7bc3be","pdpGroup":"defaultGroup"} [2023-03-11T19:37:45.021+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch java.net.SocketException: Connection reset at java.base/java.net.SocketInputStream.read(SocketInputStream.java:186) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:37:45.022+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:37:45.022+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:37:45.022+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:37:45.036+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:37:45.036+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:45.039+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset [2023-03-11T19:37:45.039+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:45.045+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] I/O exception (org.apache.http.NoHttpResponseException) caught when processing request to {}->http://message-router:3904: The target server failed to respond [2023-03-11T19:37:45.046+00:00|INFO|RetryExec|DMAAP-source-POLICY-PDP-PAP] Retrying request to {}->http://message-router:3904 [2023-03-11T19:37:45.058+00:00|WARN|HttpClient|DMAAP-source-POLICY-PDP-PAP] Error executing HTTP request. Connection reset; blacklisting for 2 minutes [2023-03-11T19:37:45.058+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of Connection reset [2023-03-11T19:37:45.058+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-03-11T19:38:00.058+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch java.net.SocketException: Connection reset at java.base/java.net.SocketInputStream.read(SocketInputStream.java:186) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:00.059+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:38:00.059+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:00.059+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:38:29.988+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:29.988+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:30.011+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:30.011+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:30.027+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:30.027+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:30.038+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:30.039+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:30.039+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 486 message to send. [2023-03-11T19:38:30.039+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23125, above threshold 10. [2023-03-11T19:38:31.065+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58377420 ms [2023-03-11T19:38:31.068+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:31.069+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:31.106+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:31.106+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:31.121+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:31.124+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:31.138+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:31.138+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:31.152+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:31.153+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:31.153+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:31.153+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23126, above threshold 10. [2023-03-11T19:38:32.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58378516 ms [2023-03-11T19:38:32.168+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:32.168+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:32.175+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:32.176+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:32.187+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:32.188+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:32.193+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:32.193+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:32.197+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:32.198+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:32.198+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:32.198+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23127, above threshold 10. [2023-03-11T19:38:33.211+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58379566 ms [2023-03-11T19:38:33.216+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:33.216+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:33.223+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:33.224+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:33.257+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:33.257+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:33.271+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:33.271+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:33.278+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:33.279+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:33.279+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:33.279+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23128, above threshold 10. [2023-03-11T19:38:34.311+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58380666 ms [2023-03-11T19:38:34.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:34.314+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:34.320+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:34.320+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:34.332+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:34.332+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:34.338+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:34.338+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:34.346+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:34.347+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:34.347+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:34.348+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23129, above threshold 10. [2023-03-11T19:38:35.361+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58381716 ms [2023-03-11T19:38:35.364+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:35.364+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:35.379+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:35.381+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:35.390+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:35.390+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:35.404+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:35.404+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:35.424+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:35.425+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:35.425+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:35.425+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23130, above threshold 10. [2023-03-11T19:38:36.461+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58382816 ms [2023-03-11T19:38:36.462+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:36.462+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:36.467+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:36.467+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:36.470+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:36.470+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:36.474+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:36.474+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:36.478+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:36.479+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:36.479+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:36.479+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23131, above threshold 10. [2023-03-11T19:38:37.511+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58383866 ms [2023-03-11T19:38:37.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:37.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:37.530+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:37.530+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:37.537+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:37.537+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:37.542+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:37.542+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:37.546+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:37.546+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:37.546+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:37.546+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23132, above threshold 10. [2023-03-11T19:38:38.561+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58384916 ms [2023-03-11T19:38:38.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:38.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:38.571+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:38.571+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:38.582+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:38.582+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:38.586+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:38.587+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:38.590+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:38.591+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:38.591+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:38.591+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23133, above threshold 10. [2023-03-11T19:38:39.611+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58385966 ms [2023-03-11T19:38:39.614+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:39.615+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:39.619+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:39.620+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:39.627+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:39.627+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:39.643+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:39.644+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:39.654+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:39.654+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:39.655+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:39.655+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23134, above threshold 10. [2023-03-11T19:38:40.661+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58387016 ms [2023-03-11T19:38:40.666+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:40.666+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:40.702+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:40.702+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:40.734+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:40.735+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:40.742+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:40.742+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:40.769+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:40.771+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:40.771+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:40.772+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23135, above threshold 10. [2023-03-11T19:38:41.812+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58388166 ms [2023-03-11T19:38:41.823+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:41.824+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:41.827+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:41.828+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:41.845+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:41.845+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:41.853+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:41.853+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:41.857+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:41.858+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:41.858+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:41.858+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23136, above threshold 10. [2023-03-11T19:38:42.867+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58389222 ms [2023-03-11T19:38:42.871+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:42.871+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:42.895+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:42.895+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:42.924+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:42.924+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:42.950+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:42.950+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:42.959+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:42.960+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:42.961+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:42.961+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23137, above threshold 10. [2023-03-11T19:38:43.961+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58390316 ms [2023-03-11T19:38:43.964+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:43.964+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:43.970+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:43.971+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:43.974+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:43.974+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:43.978+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:43.978+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:43.983+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:43.983+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:43.983+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:43.983+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23138, above threshold 10. [2023-03-11T19:38:45.012+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58391367 ms [2023-03-11T19:38:45.015+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:45.015+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:45.025+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:45.025+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:45.032+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:45.032+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:45.040+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:45.041+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:45.047+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:45.047+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:45.048+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:45.048+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23139, above threshold 10. [2023-03-11T19:38:45.099+00:00|WARN|HttpClient|DMAAP-source-POLICY-PDP-PAP] Error executing HTTP request. Read timed out; blacklisting for 2 minutes [2023-03-11T19:38:45.099+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of Read timed out [2023-03-11T19:38:45.099+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-03-11T19:38:46.061+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58392416 ms [2023-03-11T19:38:46.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:46.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:46.085+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:46.085+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:46.090+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:46.091+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:46.095+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:46.095+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:46.102+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:46.103+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:46.103+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:46.103+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23140, above threshold 10. [2023-03-11T19:38:47.111+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58393466 ms [2023-03-11T19:38:47.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:47.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:47.125+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:47.125+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:47.128+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:47.128+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:47.131+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:47.131+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:47.138+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:47.138+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:47.138+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:47.138+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23141, above threshold 10. [2023-03-11T19:38:48.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58394516 ms [2023-03-11T19:38:48.164+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:48.164+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:48.174+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:48.174+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:48.178+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:48.178+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:48.187+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:48.187+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:48.198+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:48.199+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:48.199+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:48.199+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23142, above threshold 10. [2023-03-11T19:38:49.211+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58395566 ms [2023-03-11T19:38:49.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:49.214+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:49.219+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:49.220+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:49.223+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:49.223+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:49.228+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:49.229+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:49.234+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:49.235+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:49.235+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:49.235+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23143, above threshold 10. [2023-03-11T19:38:50.265+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58396620 ms [2023-03-11T19:38:50.268+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:50.268+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:50.275+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:50.275+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:50.281+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:50.281+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:50.284+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:50.284+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:50.288+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:50.288+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:50.288+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:50.288+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23144, above threshold 10. [2023-03-11T19:38:51.311+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58397666 ms [2023-03-11T19:38:51.314+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:51.315+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:51.326+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:51.326+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:51.340+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:51.341+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:51.348+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:51.348+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:51.362+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:51.362+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:51.362+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:51.362+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23145, above threshold 10. [2023-03-11T19:38:52.411+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58398766 ms [2023-03-11T19:38:52.414+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:52.414+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:52.419+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:52.419+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:52.429+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:52.429+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:52.435+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:52.435+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:52.446+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:52.447+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:52.447+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:52.447+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23146, above threshold 10. [2023-03-11T19:38:53.461+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58399816 ms [2023-03-11T19:38:53.469+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:53.469+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:53.482+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:53.482+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:53.518+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:53.519+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:53.559+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:53.559+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:53.582+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:53.582+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:53.582+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:53.582+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23147, above threshold 10. [2023-03-11T19:38:54.612+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58400967 ms [2023-03-11T19:38:54.615+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:54.615+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:54.647+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:54.647+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:54.655+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:54.655+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:54.665+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:54.665+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:54.686+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:54.686+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:54.686+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:54.686+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23148, above threshold 10. [2023-03-11T19:38:55.711+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58402066 ms [2023-03-11T19:38:55.715+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:55.716+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:55.729+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:55.729+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:55.735+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:38:55.735+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:55.746+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:55.746+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:55.753+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:55.753+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:55.753+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:55.753+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23149, above threshold 10. [2023-03-11T19:38:56.761+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58403116 ms [2023-03-11T19:38:56.769+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:56.769+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:56.793+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:56.793+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:56.827+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:56.828+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:56.845+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:56.845+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:56.849+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:56.849+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:56.849+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:56.849+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23150, above threshold 10. [2023-03-11T19:38:57.861+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58404216 ms [2023-03-11T19:38:57.871+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:57.871+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:57.882+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:57.882+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:57.898+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:57.898+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:57.904+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:57.904+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:57.910+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:57.910+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:57.910+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:57.910+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23151, above threshold 10. [2023-03-11T19:38:58.911+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58405266 ms [2023-03-11T19:38:58.914+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:58.914+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:58.944+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:58.944+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:58.948+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:58.948+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:58.951+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:58.951+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:58.956+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:58.957+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:58.957+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:58.957+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23152, above threshold 10. [2023-03-11T19:38:59.961+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58406316 ms [2023-03-11T19:38:59.964+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:38:59.964+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:38:59.970+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:59.970+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:59.975+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:59.975+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:59.980+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:38:59.980+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:38:59.989+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:38:59.990+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:38:59.991+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:38:59.991+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23153, above threshold 10. [2023-03-11T19:39:00.102+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch java.net.SocketTimeoutException: Read timed out at java.base/java.net.SocketInputStream.socketRead0(Native Method) at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:00.102+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:39:00.102+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:00.102+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:39:01.012+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58407367 ms [2023-03-11T19:39:01.017+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:01.017+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:01.030+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:01.030+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:01.066+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:01.066+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:01.105+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:01.105+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:01.128+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:01.128+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:01.128+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:01.128+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23154, above threshold 10. [2023-03-11T19:39:02.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58408516 ms [2023-03-11T19:39:02.164+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:02.165+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:02.175+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:02.175+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:02.183+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:02.184+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:02.190+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:02.192+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:02.198+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:02.199+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:02.199+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:02.199+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23155, above threshold 10. [2023-03-11T19:39:03.211+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58409566 ms [2023-03-11T19:39:03.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:03.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:03.222+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:03.222+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:03.234+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:03.234+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:03.251+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:03.252+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:03.263+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:03.263+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:03.264+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:03.264+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23156, above threshold 10. [2023-03-11T19:39:04.311+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58410666 ms [2023-03-11T19:39:04.315+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:04.315+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:04.323+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:04.324+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:04.337+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:04.337+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:04.355+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:04.355+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:04.362+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:04.362+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:04.362+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:04.362+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23157, above threshold 10. [2023-03-11T19:39:05.411+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58411766 ms [2023-03-11T19:39:05.414+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:05.414+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:05.421+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:05.421+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:05.431+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:05.431+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:05.437+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:05.437+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:05.470+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:05.471+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:05.471+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:05.471+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23158, above threshold 10. [2023-03-11T19:39:06.516+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58412871 ms [2023-03-11T19:39:06.520+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:06.520+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:06.525+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:06.526+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:06.532+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:06.533+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:06.538+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:06.539+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:06.574+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:06.575+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:06.575+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:06.575+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23159, above threshold 10. [2023-03-11T19:39:07.612+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58413967 ms [2023-03-11T19:39:07.616+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:07.616+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:07.625+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:07.625+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:07.642+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:07.642+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:07.647+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:07.647+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:07.657+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:07.658+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:07.658+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:07.658+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23160, above threshold 10. [2023-03-11T19:39:08.662+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58415017 ms [2023-03-11T19:39:08.666+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:08.666+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:08.704+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:08.704+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:08.723+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:08.723+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:08.739+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:08.739+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:08.755+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:08.755+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:08.755+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:08.755+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23161, above threshold 10. [2023-03-11T19:39:09.761+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58416116 ms [2023-03-11T19:39:09.769+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:09.770+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:09.780+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:09.780+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:09.788+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:09.788+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:09.797+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:09.797+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:09.817+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:09.817+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:09.817+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:09.817+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23162, above threshold 10. [2023-03-11T19:39:10.819+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58417174 ms [2023-03-11T19:39:10.825+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:10.825+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:10.830+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:10.831+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:10.870+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:10.870+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:10.901+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:10.901+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:10.912+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:10.913+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:10.913+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:10.913+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23163, above threshold 10. [2023-03-11T19:39:11.961+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58418316 ms [2023-03-11T19:39:11.964+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:11.964+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:11.976+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:11.976+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:11.985+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:11.985+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:11.992+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:11.992+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:12.000+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:12.002+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:12.002+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:12.002+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23164, above threshold 10. [2023-03-11T19:39:13.011+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58419366 ms [2023-03-11T19:39:13.015+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:13.015+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:13.022+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:13.022+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:13.029+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:13.029+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:13.034+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:13.034+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:13.045+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:13.045+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:13.045+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:13.045+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23165, above threshold 10. [2023-03-11T19:39:14.061+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58420416 ms [2023-03-11T19:39:14.064+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:14.064+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:14.072+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:14.072+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:14.077+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:14.077+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:14.086+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:14.086+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:14.092+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:14.092+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:14.092+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:14.092+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23166, above threshold 10. [2023-03-11T19:39:15.111+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58421466 ms [2023-03-11T19:39:15.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:15.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:15.129+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:15.129+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:15.142+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:15.142+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:15.147+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:15.147+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:15.150+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:15.151+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:15.151+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:15.151+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23167, above threshold 10. [2023-03-11T19:39:16.162+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58422517 ms [2023-03-11T19:39:16.171+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:16.171+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:16.175+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:16.175+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:16.181+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:16.181+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:16.188+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:16.188+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:16.191+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:16.192+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:16.192+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:16.192+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23168, above threshold 10. [2023-03-11T19:39:17.211+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58423566 ms [2023-03-11T19:39:17.212+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:17.212+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:17.216+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:17.216+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:17.221+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:17.221+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:17.226+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:17.226+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:17.231+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:17.231+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:17.231+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:17.231+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23169, above threshold 10. [2023-03-11T19:39:18.261+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58424616 ms [2023-03-11T19:39:18.265+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:18.265+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:18.269+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:18.269+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:18.281+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:18.281+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:18.297+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:18.297+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:18.301+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:18.302+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:18.302+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:18.302+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23170, above threshold 10. [2023-03-11T19:39:19.314+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58425669 ms [2023-03-11T19:39:19.319+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:19.320+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:19.332+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:19.333+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:19.338+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:19.339+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:19.341+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:19.342+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:19.345+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:19.346+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:19.346+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:19.347+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23171, above threshold 10. [2023-03-11T19:39:20.365+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58426720 ms [2023-03-11T19:39:20.374+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:20.375+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:20.383+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:20.383+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:20.388+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:20.388+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:20.406+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:20.406+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:20.426+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:20.427+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:20.427+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:20.427+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23172, above threshold 10. [2023-03-11T19:39:21.462+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58427817 ms [2023-03-11T19:39:21.466+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:21.466+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:21.470+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:21.470+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:21.476+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:21.476+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:21.491+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:21.491+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:21.495+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:21.496+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:21.496+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:21.496+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23173, above threshold 10. [2023-03-11T19:39:22.511+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58428866 ms [2023-03-11T19:39:22.516+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:22.516+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:22.530+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:22.530+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:22.541+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:22.541+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:22.551+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:22.551+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:22.566+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:22.566+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:22.566+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:22.566+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23174, above threshold 10. [2023-03-11T19:39:23.616+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58429971 ms [2023-03-11T19:39:23.622+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:23.622+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:23.639+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:23.639+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:23.659+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:23.659+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:23.669+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:23.669+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:23.682+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:23.682+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:23.682+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:23.682+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23175, above threshold 10. [2023-03-11T19:39:24.711+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58431066 ms [2023-03-11T19:39:24.717+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:24.717+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:24.726+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:24.727+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:24.776+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:24.777+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:24.790+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:24.790+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:24.798+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:24.799+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:24.800+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:24.800+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23176, above threshold 10. [2023-03-11T19:39:25.811+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58432166 ms [2023-03-11T19:39:25.815+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:25.815+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:25.822+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:25.822+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:25.825+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:25.826+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:25.829+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:25.830+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:25.833+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:25.833+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:25.833+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:25.834+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23177, above threshold 10. [2023-03-11T19:39:26.861+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58433216 ms [2023-03-11T19:39:26.874+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:26.874+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:26.895+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:26.902+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:26.922+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:26.922+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:26.944+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:26.945+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:26.954+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:26.954+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:26.954+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:26.954+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23178, above threshold 10. [2023-03-11T19:39:27.961+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58434316 ms [2023-03-11T19:39:27.965+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:27.966+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:28.007+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:28.008+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:28.027+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:28.027+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:28.044+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:28.045+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:28.077+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:28.078+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:28.078+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:28.078+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23179, above threshold 10. [2023-03-11T19:39:29.111+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58435466 ms [2023-03-11T19:39:29.120+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:29.120+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:29.137+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:29.137+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:29.161+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:29.161+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:29.191+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:29.191+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:29.250+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:29.251+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:29.251+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:29.251+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23180, above threshold 10. [2023-03-11T19:39:30.261+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58436616 ms [2023-03-11T19:39:30.264+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:30.265+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:30.284+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:30.284+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:30.297+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:30.297+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:30.324+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:30.324+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:30.331+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:30.332+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:30.332+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:30.332+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23181, above threshold 10. [2023-03-11T19:39:31.361+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58437716 ms [2023-03-11T19:39:31.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:31.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:31.395+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:31.400+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:31.408+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:31.409+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:31.445+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:31.445+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:31.473+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:31.473+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:31.473+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:31.473+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23182, above threshold 10. [2023-03-11T19:39:32.513+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 487 msgs to /events/POLICY-PDP-PAP. Oldest: 58438868 ms [2023-03-11T19:39:32.526+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:32.526+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:32.546+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:32.546+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:32.572+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:32.572+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:32.579+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:32.579+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:32.583+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:32.583+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:32.583+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 487 message to send. [2023-03-11T19:39:32.583+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23183, above threshold 10. [2023-03-11T19:39:32.887+00:00|WARN|PdpMessageHandler|Timer-0] not registered: object:engine/apex/handler [2023-03-11T19:39:32.888+00:00|INFO|network|Timer-0] [OUT|DMAAP|POLICY-PDP-PAP] {"pdpType":"apex","state":"PASSIVE","healthy":"HEALTHY","description":"Pdp Heartbeat","statistics":{"pdpInstanceId":"apex-7fd94bb6-8411-46e5-8859-f3d0db7bc3be","timeStamp":"2023-03-11T19:39:32.888096Z","pdpGroupName":"defaultGroup","policyExecutedCount":0,"policyExecutedSuccessCount":0,"policyExecutedFailCount":0,"policyDeployCount":0,"policyDeploySuccessCount":0,"policyDeployFailCount":0,"policyUndeployCount":0,"policyUndeploySuccessCount":0,"policyUndeployFailCount":0},"messageName":"PDP_STATUS","requestId":"d81468b4-148a-4a78-8cba-d3b20107aafa","timestampMs":1678563572887,"name":"apex-7fd94bb6-8411-46e5-8859-f3d0db7bc3be","pdpGroup":"defaultGroup"} [2023-03-11T19:39:33.612+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58439967 ms [2023-03-11T19:39:33.616+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:33.616+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:33.634+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:33.634+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:33.644+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:33.645+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:33.660+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:33.660+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:33.680+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:33.680+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:33.680+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:33.680+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23184, above threshold 10. [2023-03-11T19:39:34.711+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58441066 ms [2023-03-11T19:39:34.715+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:34.715+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:34.738+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:34.739+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:34.744+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:34.745+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:34.749+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:34.750+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:34.766+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:34.766+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:34.767+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:34.768+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23185, above threshold 10. [2023-03-11T19:39:35.811+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58442166 ms [2023-03-11T19:39:35.814+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:35.814+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:35.836+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:35.836+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:35.842+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:35.842+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:35.850+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:35.850+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:35.855+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:35.856+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:35.856+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:35.856+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23186, above threshold 10. [2023-03-11T19:39:36.862+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58443217 ms [2023-03-11T19:39:36.865+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:36.865+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:36.873+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:36.873+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:36.881+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:36.881+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:36.893+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:36.894+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:36.900+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:36.901+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:36.901+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:36.901+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23187, above threshold 10. [2023-03-11T19:39:37.911+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58444266 ms [2023-03-11T19:39:37.914+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:37.914+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:37.924+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:37.924+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:37.950+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:37.950+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:37.961+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:37.961+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:37.988+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:37.988+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:37.988+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:37.988+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23188, above threshold 10. [2023-03-11T19:39:39.011+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58445366 ms [2023-03-11T19:39:39.014+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:39.015+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:39.026+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:39.026+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:39.057+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:39.057+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:39.081+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:39.081+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:39.101+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:39.102+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:39.102+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:39.102+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23189, above threshold 10. [2023-03-11T19:39:40.111+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58446466 ms [2023-03-11T19:39:40.114+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:40.115+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:40.128+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:40.128+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:40.133+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:40.134+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:40.138+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:40.138+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:40.145+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:40.145+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:40.146+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:40.146+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23190, above threshold 10. [2023-03-11T19:39:41.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58447516 ms [2023-03-11T19:39:41.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:41.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:41.223+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:41.223+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:41.230+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:41.230+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:41.236+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:41.236+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:41.245+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:41.245+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:41.245+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:41.245+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23191, above threshold 10. [2023-03-11T19:39:42.262+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58448617 ms [2023-03-11T19:39:42.265+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:42.265+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:42.304+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:42.304+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:42.345+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:42.346+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:42.366+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:42.366+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:42.397+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:42.397+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:42.397+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:42.397+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23192, above threshold 10. [2023-03-11T19:39:43.411+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58449766 ms [2023-03-11T19:39:43.414+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:43.414+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:43.423+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:43.423+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:43.431+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:43.431+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:43.447+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:43.447+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:43.453+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:43.453+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:43.453+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:43.453+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23193, above threshold 10. [2023-03-11T19:39:44.461+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58450816 ms [2023-03-11T19:39:44.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:44.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:44.470+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:44.471+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:44.475+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:44.476+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:44.480+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:44.481+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:44.485+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:44.485+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:44.486+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:44.487+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23194, above threshold 10. [2023-03-11T19:39:45.154+00:00|WARN|HttpClient|DMAAP-source-POLICY-PDP-PAP] Error executing HTTP request. Read timed out; blacklisting for 2 minutes [2023-03-11T19:39:45.154+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of Read timed out [2023-03-11T19:39:45.154+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-03-11T19:39:45.511+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58451866 ms [2023-03-11T19:39:45.514+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:45.515+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:45.530+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:45.530+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:45.556+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:45.557+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:45.596+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:45.597+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:45.625+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:45.626+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:45.626+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:45.626+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23195, above threshold 10. [2023-03-11T19:39:46.661+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58453016 ms [2023-03-11T19:39:46.664+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:46.665+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:46.697+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:46.697+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:46.702+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:46.703+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:46.711+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:46.711+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:46.752+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:46.757+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:46.759+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:46.759+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23196, above threshold 10. [2023-03-11T19:39:47.761+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58454116 ms [2023-03-11T19:39:47.766+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:47.766+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:47.791+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:47.793+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:47.810+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:47.810+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:47.823+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:47.823+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:47.832+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:47.833+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:47.835+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:47.835+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23197, above threshold 10. [2023-03-11T19:39:48.861+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58455216 ms [2023-03-11T19:39:48.871+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:48.872+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:48.896+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:48.897+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:48.919+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:48.919+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:48.940+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:48.941+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:48.953+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:48.954+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:48.954+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:48.954+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23198, above threshold 10. [2023-03-11T19:39:49.961+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58456316 ms [2023-03-11T19:39:49.982+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:49.982+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:49.997+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:49.998+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:50.007+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:50.007+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:50.012+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:50.012+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:50.023+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:50.024+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:50.026+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:50.026+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23199, above threshold 10. [2023-03-11T19:39:51.064+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58457419 ms [2023-03-11T19:39:51.090+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:51.090+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:51.097+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:51.097+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:51.101+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:51.102+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:51.108+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:51.108+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:51.117+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:51.118+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:51.118+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:51.118+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23200, above threshold 10. [2023-03-11T19:39:52.164+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58458519 ms [2023-03-11T19:39:52.180+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:52.181+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:52.187+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:52.188+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:52.195+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:52.196+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:52.204+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:52.208+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:52.219+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:52.220+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:52.221+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:52.221+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23201, above threshold 10. [2023-03-11T19:39:53.261+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58459616 ms [2023-03-11T19:39:53.264+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:53.265+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:53.276+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:53.277+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:53.280+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:53.283+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:53.299+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:53.300+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:53.313+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:53.314+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:53.314+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:53.314+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23202, above threshold 10. [2023-03-11T19:39:54.361+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58460716 ms [2023-03-11T19:39:54.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:54.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:54.372+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:54.373+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:54.379+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:54.381+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:54.394+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:54.395+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:54.414+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:54.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:54.419+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:54.419+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23203, above threshold 10. [2023-03-11T19:39:55.465+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58461820 ms [2023-03-11T19:39:55.470+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:55.470+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:55.478+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:55.479+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:55.491+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:55.491+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:55.504+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:55.504+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:55.521+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:55.521+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:55.521+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:55.521+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23204, above threshold 10. [2023-03-11T19:39:56.561+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58462916 ms [2023-03-11T19:39:56.564+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:56.564+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:56.583+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:56.583+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:56.594+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:56.594+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:56.603+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:56.603+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:56.612+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:56.613+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:56.613+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:56.613+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23205, above threshold 10. [2023-03-11T19:39:57.661+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58464016 ms [2023-03-11T19:39:57.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:57.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:57.678+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:57.678+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:57.695+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:57.696+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:57.713+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:57.713+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:57.740+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:57.745+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:57.746+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:57.746+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23206, above threshold 10. [2023-03-11T19:39:58.761+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58465116 ms [2023-03-11T19:39:58.767+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:58.767+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:58.798+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:58.799+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:58.807+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:58.807+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:58.812+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:58.812+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:58.858+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:58.858+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:58.858+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:58.858+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23207, above threshold 10. [2023-03-11T19:39:59.861+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58466216 ms [2023-03-11T19:39:59.861+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:39:59.861+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:39:59.878+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:59.878+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:59.884+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:39:59.884+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:59.887+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:39:59.887+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:39:59.897+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:39:59.898+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:39:59.898+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:39:59.898+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23208, above threshold 10. [2023-03-11T19:40:00.155+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=a3b578ba-0c27-4bd3-b245-ce44c0f07415, consumerInstance=onap-policy-apex-pdp-0, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=1, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=1]]]]: cannot fetch java.net.SocketTimeoutException: Read timed out at java.base/java.net.SocketInputStream.socketRead0(Native Method) at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168) at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140) at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137) at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153) at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138) at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56) at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259) at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163) at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157) at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:00.155+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 [2023-03-11T19:40:00.155+00:00|WARN|HostSelector|DMAAP-source-POLICY-PDP-PAP] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:00.155+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/a3b578ba-0c27-4bd3-b245-ce44c0f07415/onap-policy-apex-pdp-0?timeout=15000 (anonymous) ... [2023-03-11T19:40:00.911+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58467266 ms [2023-03-11T19:40:00.918+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:00.918+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:00.926+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:00.926+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:00.946+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:00.946+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:00.958+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:00.958+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:00.971+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:00.971+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:00.971+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:00.971+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23209, above threshold 10. [2023-03-11T19:40:02.011+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58468366 ms [2023-03-11T19:40:02.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:02.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:02.019+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:02.019+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:02.025+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:02.025+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:02.042+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:02.042+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:02.054+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:02.055+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:02.055+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:02.055+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23210, above threshold 10. [2023-03-11T19:40:03.061+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58469416 ms [2023-03-11T19:40:03.062+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:03.062+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:03.076+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:03.076+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:03.079+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:03.079+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:03.084+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:03.084+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:03.090+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:03.090+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:03.090+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:03.090+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23211, above threshold 10. [2023-03-11T19:40:04.111+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58470466 ms [2023-03-11T19:40:04.112+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:04.112+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:04.120+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:04.120+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:04.138+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:04.138+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:04.142+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:04.142+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:04.147+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:04.148+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:04.148+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:04.148+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23212, above threshold 10. [2023-03-11T19:40:05.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58471516 ms [2023-03-11T19:40:05.162+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:05.162+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:05.177+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:05.177+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:05.184+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:05.184+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:05.189+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:05.189+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:05.195+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:05.195+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:05.195+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:05.196+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23213, above threshold 10. [2023-03-11T19:40:06.211+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58472566 ms [2023-03-11T19:40:06.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:06.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:06.219+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:06.219+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:06.222+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:06.222+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:06.227+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:06.227+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:06.233+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:06.233+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:06.233+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:06.234+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23214, above threshold 10. [2023-03-11T19:40:07.261+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58473616 ms [2023-03-11T19:40:07.261+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:07.261+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:07.274+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:07.274+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:07.278+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:07.278+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:07.284+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:07.284+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:07.288+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:07.288+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:07.288+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:07.288+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23215, above threshold 10. [2023-03-11T19:40:08.311+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58474666 ms [2023-03-11T19:40:08.312+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:08.312+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:08.321+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:08.321+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:08.325+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:08.325+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:08.329+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:08.329+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:08.333+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:08.334+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:08.334+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:08.334+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23216, above threshold 10. [2023-03-11T19:40:09.361+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58475716 ms [2023-03-11T19:40:09.361+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:09.361+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:09.373+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:09.373+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:09.379+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:09.379+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:09.389+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:09.389+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:09.400+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:09.401+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:09.402+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:09.402+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23217, above threshold 10. [2023-03-11T19:40:10.411+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58476766 ms [2023-03-11T19:40:10.412+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:10.412+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:10.417+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:10.417+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:10.422+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:10.422+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:10.425+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:10.425+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:10.430+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:10.431+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:10.431+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:10.431+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23218, above threshold 10. [2023-03-11T19:40:11.461+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58477816 ms [2023-03-11T19:40:11.462+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:11.462+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:11.466+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:11.467+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:11.473+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:11.473+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:11.480+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:11.481+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:11.489+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:11.489+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:11.489+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:11.490+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23219, above threshold 10. [2023-03-11T19:40:12.511+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58478866 ms [2023-03-11T19:40:12.512+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:12.512+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:12.518+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:12.518+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:12.533+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:12.533+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:12.562+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:12.562+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:12.594+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:12.594+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:12.595+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:12.595+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23220, above threshold 10. [2023-03-11T19:40:13.611+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58479966 ms [2023-03-11T19:40:13.611+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:13.611+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:13.616+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:13.616+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:13.623+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:13.623+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:13.628+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:13.628+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:13.646+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:13.646+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:13.646+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:13.646+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23221, above threshold 10. [2023-03-11T19:40:14.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58481015 ms [2023-03-11T19:40:14.662+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:14.662+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:14.685+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:14.686+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:14.691+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:14.691+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:14.695+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:14.695+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:14.710+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:14.710+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:14.710+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:14.710+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23222, above threshold 10. [2023-03-11T19:40:15.711+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58482066 ms [2023-03-11T19:40:15.712+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:15.712+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:15.718+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:15.718+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:15.728+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:15.728+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:15.733+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:15.733+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:15.736+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:15.737+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:15.737+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:15.737+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23223, above threshold 10. [2023-03-11T19:40:16.762+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58483117 ms [2023-03-11T19:40:16.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:16.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:16.772+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:16.772+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:16.776+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:16.776+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:16.802+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:16.802+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:16.831+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Broken pipe (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:16.832+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Broken pipe (Write failed) java.net.SocketException: Broken pipe (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89) at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:708) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) [2023-03-11T19:40:16.832+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 488 message to send. [2023-03-11T19:40:16.832+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 23224, above threshold 10. [2023-03-11T19:40:17.861+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 488 msgs to /events/POLICY-PDP-PAP. Oldest: 58484216 ms [2023-03-11T19:40:17.861+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-03-11T19:40:17.861+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-03-11T19:40:17.865+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:17.865+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:17.870+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Broken pipe (Write failed) [2023-03-11T19:40:17.870+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:17.887+00:00|INFO|RetryExec|pool-2-thread-1] I/O exception (java.net.SocketException) caught when processing request to {}->http://message-router:3904: Connection reset by peer (Write failed) [2023-03-11T19:40:17.887+00:00|INFO|RetryExec|pool-2-thread-1] Retrying request to {}->http://message-router:3904 [2023-03-11T19:40:17.911+00:00|WARN|HttpClient|pool-2-thread-1] Error executing HTTP request. Connection reset by peer (Write failed); blacklisting for 2 minutes [2023-03-11T19:40:17.911+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Connection reset by peer (Write failed) java.net.SocketException: Connection reset by peer (Write failed) at java.base/java.net.SocketOutputStream.socketWrite0(Native Method) at java.base/java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:110) at java.base/java.net.SocketOutputStream.write(SocketOutputStream.java:150) at org.apache.http.impl.io.SessionOutputBufferImpl.streamWrite(SessionOutputBufferImpl.java:124) at org.apache.http.impl.io.SessionOutputBufferImpl.write(SessionOutputBufferImpl.java:160) at org.apache.http.impl.io.ContentLengthOutputStream.write(ContentLengthOutputStream.java:113) at org.apache.http.entity.ByteArrayEntity.writeTo(ByteArrayEntity.java:112) at org.apache.http.impl.DefaultBHttpClientConnection.sendRequestEntity(DefaultBHttpClientConnection.java:156) at org.apache.http.impl.conn.CPoolProxy.sendRequestEntity(CPoolProxy.java:152) at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:238) at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272) at org.apache.http.impl.execchai