[2023-01-13T06:38:04.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:04.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:04.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:04.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:04.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6726 message to send. [2023-01-13T06:38:04.565+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25629, above threshold 10. [2023-01-13T06:38:04.646+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:05.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6727 msgs to /events/POLICY-PDP-PAP. Oldest: 26907237 ms [2023-01-13T06:38:05.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:05.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:05.614+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:05.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:05.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6727 message to send. [2023-01-13T06:38:05.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25630, above threshold 10. [2023-01-13T06:38:06.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6727 msgs to /events/POLICY-PDP-PAP. Oldest: 26908287 ms [2023-01-13T06:38:06.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:06.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:06.665+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:06.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:06.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6727 message to send. [2023-01-13T06:38:06.665+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25631, above threshold 10. [2023-01-13T06:38:07.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6727 msgs to /events/POLICY-PDP-PAP. Oldest: 26909337 ms [2023-01-13T06:38:07.714+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:07.714+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:07.717+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:07.717+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:07.717+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6727 message to send. [2023-01-13T06:38:07.717+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25632, above threshold 10. [2023-01-13T06:38:08.646+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:08.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6728 msgs to /events/POLICY-PDP-PAP. Oldest: 26910387 ms [2023-01-13T06:38:08.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:08.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:08.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:08.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:08.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6728 message to send. [2023-01-13T06:38:08.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25633, above threshold 10. [2023-01-13T06:38:09.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6728 msgs to /events/POLICY-PDP-PAP. Oldest: 26911437 ms [2023-01-13T06:38:09.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:09.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:09.814+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:09.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:09.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6728 message to send. [2023-01-13T06:38:09.815+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25634, above threshold 10. [2023-01-13T06:38:10.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6728 msgs to /events/POLICY-PDP-PAP. Oldest: 26912487 ms [2023-01-13T06:38:10.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:10.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:10.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:10.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:10.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6728 message to send. [2023-01-13T06:38:10.865+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25635, above threshold 10. [2023-01-13T06:38:11.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6728 msgs to /events/POLICY-PDP-PAP. Oldest: 26913537 ms [2023-01-13T06:38:11.915+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:11.915+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:11.918+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:11.918+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:11.918+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6728 message to send. [2023-01-13T06:38:11.919+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25636, above threshold 10. [2023-01-13T06:38:12.646+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:12.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6729 msgs to /events/POLICY-PDP-PAP. Oldest: 26914587 ms [2023-01-13T06:38:12.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:12.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:12.966+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:12.967+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:12.967+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6729 message to send. [2023-01-13T06:38:12.967+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25637, above threshold 10. [2023-01-13T06:38:14.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6729 msgs to /events/POLICY-PDP-PAP. Oldest: 26915637 ms [2023-01-13T06:38:14.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:14.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:14.015+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:14.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:14.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6729 message to send. [2023-01-13T06:38:14.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25638, above threshold 10. [2023-01-13T06:38:14.596+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:38:14.597+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:38:14.597+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:38:14.598+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:14.599+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:14.599+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:38:15.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6729 msgs to /events/POLICY-PDP-PAP. Oldest: 26916687 ms [2023-01-13T06:38:15.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:15.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:15.064+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:15.064+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:15.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6729 message to send. [2023-01-13T06:38:15.065+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25639, above threshold 10. [2023-01-13T06:38:16.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6729 msgs to /events/POLICY-PDP-PAP. Oldest: 26917737 ms [2023-01-13T06:38:16.112+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:16.112+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:16.114+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:16.114+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:16.114+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6729 message to send. [2023-01-13T06:38:16.114+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25640, above threshold 10. [2023-01-13T06:38:16.646+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:17.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6730 msgs to /events/POLICY-PDP-PAP. Oldest: 26918787 ms [2023-01-13T06:38:17.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:17.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:17.165+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:17.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:17.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6730 message to send. [2023-01-13T06:38:17.165+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25641, above threshold 10. [2023-01-13T06:38:18.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6730 msgs to /events/POLICY-PDP-PAP. Oldest: 26919837 ms [2023-01-13T06:38:18.212+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:18.212+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:18.215+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:18.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:18.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6730 message to send. [2023-01-13T06:38:18.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25642, above threshold 10. [2023-01-13T06:38:19.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6730 msgs to /events/POLICY-PDP-PAP. Oldest: 26920887 ms [2023-01-13T06:38:19.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:19.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:19.264+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:19.264+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:19.264+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6730 message to send. [2023-01-13T06:38:19.264+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25643, above threshold 10. [2023-01-13T06:38:20.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6730 msgs to /events/POLICY-PDP-PAP. Oldest: 26921937 ms [2023-01-13T06:38:20.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:20.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:20.315+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:20.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:20.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6730 message to send. [2023-01-13T06:38:20.315+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25644, above threshold 10. [2023-01-13T06:38:20.647+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:21.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6731 msgs to /events/POLICY-PDP-PAP. Oldest: 26922987 ms [2023-01-13T06:38:21.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:21.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:21.365+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:21.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:21.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6731 message to send. [2023-01-13T06:38:21.365+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25645, above threshold 10. [2023-01-13T06:38:22.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6731 msgs to /events/POLICY-PDP-PAP. Oldest: 26924037 ms [2023-01-13T06:38:22.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:22.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:22.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:22.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:22.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6731 message to send. [2023-01-13T06:38:22.415+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25646, above threshold 10. [2023-01-13T06:38:23.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6731 msgs to /events/POLICY-PDP-PAP. Oldest: 26925087 ms [2023-01-13T06:38:23.464+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:23.464+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:23.466+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:23.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:23.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6731 message to send. [2023-01-13T06:38:23.466+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25647, above threshold 10. [2023-01-13T06:38:24.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6731 msgs to /events/POLICY-PDP-PAP. Oldest: 26926137 ms [2023-01-13T06:38:24.512+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:24.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:24.514+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:24.514+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:24.514+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6731 message to send. [2023-01-13T06:38:24.514+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25648, above threshold 10. [2023-01-13T06:38:24.647+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:25.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6732 msgs to /events/POLICY-PDP-PAP. Oldest: 26927187 ms [2023-01-13T06:38:25.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:25.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:25.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:25.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:25.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6732 message to send. [2023-01-13T06:38:25.564+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25649, above threshold 10. [2023-01-13T06:38:26.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6732 msgs to /events/POLICY-PDP-PAP. Oldest: 26928237 ms [2023-01-13T06:38:26.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:26.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:26.614+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:26.614+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:26.614+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6732 message to send. [2023-01-13T06:38:26.614+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25650, above threshold 10. [2023-01-13T06:38:27.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6732 msgs to /events/POLICY-PDP-PAP. Oldest: 26929287 ms [2023-01-13T06:38:27.664+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:27.664+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:27.665+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:27.666+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:27.666+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6732 message to send. [2023-01-13T06:38:27.666+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25651, above threshold 10. [2023-01-13T06:38:28.650+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:28.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6733 msgs to /events/POLICY-PDP-PAP. Oldest: 26930337 ms [2023-01-13T06:38:28.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:28.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:28.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:28.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:28.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6733 message to send. [2023-01-13T06:38:28.714+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25652, above threshold 10. [2023-01-13T06:38:29.599+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:38:29.599+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:38:29.599+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:38:29.601+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:29.601+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:29.601+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:38:29.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6733 msgs to /events/POLICY-PDP-PAP. Oldest: 26931387 ms [2023-01-13T06:38:29.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:29.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:29.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:29.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:29.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6733 message to send. [2023-01-13T06:38:29.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25653, above threshold 10. [2023-01-13T06:38:30.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6733 msgs to /events/POLICY-PDP-PAP. Oldest: 26932437 ms [2023-01-13T06:38:30.812+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:30.812+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:30.814+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:30.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:30.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6733 message to send. [2023-01-13T06:38:30.814+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25654, above threshold 10. [2023-01-13T06:38:31.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6733 msgs to /events/POLICY-PDP-PAP. Oldest: 26933487 ms [2023-01-13T06:38:31.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:31.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:31.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:31.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:31.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6733 message to send. [2023-01-13T06:38:31.864+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25655, above threshold 10. [2023-01-13T06:38:32.650+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:32.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6734 msgs to /events/POLICY-PDP-PAP. Oldest: 26934537 ms [2023-01-13T06:38:32.912+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:32.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:32.914+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:32.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:32.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6734 message to send. [2023-01-13T06:38:32.914+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25656, above threshold 10. [2023-01-13T06:38:33.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6734 msgs to /events/POLICY-PDP-PAP. Oldest: 26935587 ms [2023-01-13T06:38:33.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:33.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:33.966+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:33.966+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:33.966+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6734 message to send. [2023-01-13T06:38:33.966+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25657, above threshold 10. [2023-01-13T06:38:35.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6734 msgs to /events/POLICY-PDP-PAP. Oldest: 26936637 ms [2023-01-13T06:38:35.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:35.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:35.016+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:35.016+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:35.016+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6734 message to send. [2023-01-13T06:38:35.016+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25658, above threshold 10. [2023-01-13T06:38:36.061+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6734 msgs to /events/POLICY-PDP-PAP. Oldest: 26937688 ms [2023-01-13T06:38:36.064+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:36.064+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:36.067+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:36.067+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:36.067+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6734 message to send. [2023-01-13T06:38:36.067+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25659, above threshold 10. [2023-01-13T06:38:36.650+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:37.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6735 msgs to /events/POLICY-PDP-PAP. Oldest: 26938737 ms [2023-01-13T06:38:37.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:37.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:37.114+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:37.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:37.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6735 message to send. [2023-01-13T06:38:37.115+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25660, above threshold 10. [2023-01-13T06:38:38.161+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6735 msgs to /events/POLICY-PDP-PAP. Oldest: 26939788 ms [2023-01-13T06:38:38.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:38.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:38.165+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:38.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:38.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6735 message to send. [2023-01-13T06:38:38.165+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25661, above threshold 10. [2023-01-13T06:38:39.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6735 msgs to /events/POLICY-PDP-PAP. Oldest: 26940837 ms [2023-01-13T06:38:39.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:39.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:39.214+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:39.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:39.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6735 message to send. [2023-01-13T06:38:39.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25662, above threshold 10. [2023-01-13T06:38:40.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6735 msgs to /events/POLICY-PDP-PAP. Oldest: 26941887 ms [2023-01-13T06:38:40.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:40.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:40.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:40.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:40.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6735 message to send. [2023-01-13T06:38:40.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25663, above threshold 10. [2023-01-13T06:38:40.651+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:41.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6736 msgs to /events/POLICY-PDP-PAP. Oldest: 26942937 ms [2023-01-13T06:38:41.312+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:41.312+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:41.314+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:41.314+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:41.314+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6736 message to send. [2023-01-13T06:38:41.314+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25664, above threshold 10. [2023-01-13T06:38:42.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6736 msgs to /events/POLICY-PDP-PAP. Oldest: 26943987 ms [2023-01-13T06:38:42.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:42.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:42.364+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:42.364+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:42.364+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6736 message to send. [2023-01-13T06:38:42.364+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25665, above threshold 10. [2023-01-13T06:38:43.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6736 msgs to /events/POLICY-PDP-PAP. Oldest: 26945037 ms [2023-01-13T06:38:43.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:43.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:43.416+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:43.416+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:43.416+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6736 message to send. [2023-01-13T06:38:43.416+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25666, above threshold 10. [2023-01-13T06:38:44.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6736 msgs to /events/POLICY-PDP-PAP. Oldest: 26946087 ms [2023-01-13T06:38:44.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:44.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:44.465+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:44.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:44.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6736 message to send. [2023-01-13T06:38:44.466+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25667, above threshold 10. [2023-01-13T06:38:44.602+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:38:44.602+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:38:44.602+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:38:44.604+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:44.604+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:44.604+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:38:44.651+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:45.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6737 msgs to /events/POLICY-PDP-PAP. Oldest: 26947137 ms [2023-01-13T06:38:45.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:45.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:45.515+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:45.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:45.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6737 message to send. [2023-01-13T06:38:45.515+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25668, above threshold 10. [2023-01-13T06:38:46.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6737 msgs to /events/POLICY-PDP-PAP. Oldest: 26948187 ms [2023-01-13T06:38:46.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:46.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:46.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:46.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:46.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6737 message to send. [2023-01-13T06:38:46.565+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25669, above threshold 10. [2023-01-13T06:38:47.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6737 msgs to /events/POLICY-PDP-PAP. Oldest: 26949237 ms [2023-01-13T06:38:47.614+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:47.615+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:47.616+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:47.616+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:47.617+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6737 message to send. [2023-01-13T06:38:47.617+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25670, above threshold 10. [2023-01-13T06:38:48.651+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:48.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6738 msgs to /events/POLICY-PDP-PAP. Oldest: 26950287 ms [2023-01-13T06:38:48.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:48.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:48.665+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:48.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:48.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6738 message to send. [2023-01-13T06:38:48.665+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25671, above threshold 10. [2023-01-13T06:38:49.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6738 msgs to /events/POLICY-PDP-PAP. Oldest: 26951337 ms [2023-01-13T06:38:49.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:49.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:49.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:49.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:49.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6738 message to send. [2023-01-13T06:38:49.714+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25672, above threshold 10. [2023-01-13T06:38:50.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6738 msgs to /events/POLICY-PDP-PAP. Oldest: 26952387 ms [2023-01-13T06:38:50.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:50.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:50.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:50.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:50.765+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6738 message to send. [2023-01-13T06:38:50.765+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25673, above threshold 10. [2023-01-13T06:38:51.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6738 msgs to /events/POLICY-PDP-PAP. Oldest: 26953437 ms [2023-01-13T06:38:51.814+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:51.814+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:51.816+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:51.817+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:51.817+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6738 message to send. [2023-01-13T06:38:51.817+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25674, above threshold 10. [2023-01-13T06:38:52.651+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:52.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6739 msgs to /events/POLICY-PDP-PAP. Oldest: 26954487 ms [2023-01-13T06:38:52.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:52.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:52.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:52.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:52.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6739 message to send. [2023-01-13T06:38:52.864+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25675, above threshold 10. [2023-01-13T06:38:53.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6739 msgs to /events/POLICY-PDP-PAP. Oldest: 26955537 ms [2023-01-13T06:38:53.913+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:53.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:53.915+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:53.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:53.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6739 message to send. [2023-01-13T06:38:53.915+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25676, above threshold 10. [2023-01-13T06:38:54.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6739 msgs to /events/POLICY-PDP-PAP. Oldest: 26956587 ms [2023-01-13T06:38:54.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:54.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:54.967+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:54.968+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:54.968+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6739 message to send. [2023-01-13T06:38:54.968+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25677, above threshold 10. [2023-01-13T06:38:56.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6739 msgs to /events/POLICY-PDP-PAP. Oldest: 26957637 ms [2023-01-13T06:38:56.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:56.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:56.014+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:56.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:56.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6739 message to send. [2023-01-13T06:38:56.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25678, above threshold 10. [2023-01-13T06:38:56.652+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:38:57.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6740 msgs to /events/POLICY-PDP-PAP. Oldest: 26958687 ms [2023-01-13T06:38:57.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:57.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:57.065+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:57.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:57.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6740 message to send. [2023-01-13T06:38:57.065+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25679, above threshold 10. [2023-01-13T06:38:58.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6740 msgs to /events/POLICY-PDP-PAP. Oldest: 26959737 ms [2023-01-13T06:38:58.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:58.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:58.114+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:58.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:58.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6740 message to send. [2023-01-13T06:38:58.115+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25680, above threshold 10. [2023-01-13T06:38:59.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6740 msgs to /events/POLICY-PDP-PAP. Oldest: 26960787 ms [2023-01-13T06:38:59.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:38:59.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:38:59.166+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:59.166+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:38:59.167+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6740 message to send. [2023-01-13T06:38:59.167+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25681, above threshold 10. [2023-01-13T06:38:59.604+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:38:59.604+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:38:59.604+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:38:59.607+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:59.607+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:38:59.607+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:39:00.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6740 msgs to /events/POLICY-PDP-PAP. Oldest: 26961837 ms [2023-01-13T06:39:00.212+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:00.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:00.214+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:00.214+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:00.214+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6740 message to send. [2023-01-13T06:39:00.214+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25682, above threshold 10. [2023-01-13T06:39:00.652+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:01.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6741 msgs to /events/POLICY-PDP-PAP. Oldest: 26962887 ms [2023-01-13T06:39:01.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:01.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:01.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:01.266+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:01.266+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6741 message to send. [2023-01-13T06:39:01.266+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25683, above threshold 10. [2023-01-13T06:39:02.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6741 msgs to /events/POLICY-PDP-PAP. Oldest: 26963937 ms [2023-01-13T06:39:02.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:02.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:02.315+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:02.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:02.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6741 message to send. [2023-01-13T06:39:02.315+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25684, above threshold 10. [2023-01-13T06:39:03.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6741 msgs to /events/POLICY-PDP-PAP. Oldest: 26964987 ms [2023-01-13T06:39:03.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:03.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:03.364+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:03.364+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:03.364+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6741 message to send. [2023-01-13T06:39:03.364+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25685, above threshold 10. [2023-01-13T06:39:04.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6741 msgs to /events/POLICY-PDP-PAP. Oldest: 26966037 ms [2023-01-13T06:39:04.412+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:04.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:04.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:04.414+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:04.414+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6741 message to send. [2023-01-13T06:39:04.414+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25686, above threshold 10. [2023-01-13T06:39:04.652+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:05.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6742 msgs to /events/POLICY-PDP-PAP. Oldest: 26967087 ms [2023-01-13T06:39:05.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:05.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:05.465+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:05.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:05.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6742 message to send. [2023-01-13T06:39:05.465+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25687, above threshold 10. [2023-01-13T06:39:06.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6742 msgs to /events/POLICY-PDP-PAP. Oldest: 26968137 ms [2023-01-13T06:39:06.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:06.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:06.514+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:06.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:06.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6742 message to send. [2023-01-13T06:39:06.515+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25688, above threshold 10. [2023-01-13T06:39:07.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6742 msgs to /events/POLICY-PDP-PAP. Oldest: 26969187 ms [2023-01-13T06:39:07.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:07.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:07.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:07.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:07.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6742 message to send. [2023-01-13T06:39:07.564+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25689, above threshold 10. [2023-01-13T06:39:08.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6742 msgs to /events/POLICY-PDP-PAP. Oldest: 26970237 ms [2023-01-13T06:39:08.614+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:08.614+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:08.615+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:08.616+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:08.616+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6742 message to send. [2023-01-13T06:39:08.616+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25690, above threshold 10. [2023-01-13T06:39:08.652+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:09.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6743 msgs to /events/POLICY-PDP-PAP. Oldest: 26971287 ms [2023-01-13T06:39:09.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:09.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:09.665+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:09.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:09.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6743 message to send. [2023-01-13T06:39:09.665+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25691, above threshold 10. [2023-01-13T06:39:10.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6743 msgs to /events/POLICY-PDP-PAP. Oldest: 26972337 ms [2023-01-13T06:39:10.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:10.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:10.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:10.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:10.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6743 message to send. [2023-01-13T06:39:10.714+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25692, above threshold 10. [2023-01-13T06:39:11.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6743 msgs to /events/POLICY-PDP-PAP. Oldest: 26973387 ms [2023-01-13T06:39:11.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:11.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:11.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:11.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:11.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6743 message to send. [2023-01-13T06:39:11.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25693, above threshold 10. [2023-01-13T06:39:12.652+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:12.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6744 msgs to /events/POLICY-PDP-PAP. Oldest: 26974437 ms [2023-01-13T06:39:12.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:12.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:12.815+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:12.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:12.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6744 message to send. [2023-01-13T06:39:12.815+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25694, above threshold 10. [2023-01-13T06:39:13.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6744 msgs to /events/POLICY-PDP-PAP. Oldest: 26975487 ms [2023-01-13T06:39:13.862+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:13.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:13.866+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:13.866+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:13.866+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6744 message to send. [2023-01-13T06:39:13.866+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25695, above threshold 10. [2023-01-13T06:39:14.607+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:39:14.607+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:39:14.607+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:39:14.609+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:14.610+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:14.610+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:39:14.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6744 msgs to /events/POLICY-PDP-PAP. Oldest: 26976537 ms [2023-01-13T06:39:14.913+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:14.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:14.914+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:14.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:14.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6744 message to send. [2023-01-13T06:39:14.915+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25696, above threshold 10. [2023-01-13T06:39:15.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6744 msgs to /events/POLICY-PDP-PAP. Oldest: 26977587 ms [2023-01-13T06:39:15.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:15.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:15.964+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:15.964+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:15.964+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6744 message to send. [2023-01-13T06:39:15.964+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25697, above threshold 10. [2023-01-13T06:39:16.653+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:17.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6745 msgs to /events/POLICY-PDP-PAP. Oldest: 26978637 ms [2023-01-13T06:39:17.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:17.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:17.015+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:17.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:17.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6745 message to send. [2023-01-13T06:39:17.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25698, above threshold 10. [2023-01-13T06:39:18.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6745 msgs to /events/POLICY-PDP-PAP. Oldest: 26979687 ms [2023-01-13T06:39:18.064+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:18.064+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:18.066+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:18.066+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:18.066+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6745 message to send. [2023-01-13T06:39:18.066+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25699, above threshold 10. [2023-01-13T06:39:19.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6745 msgs to /events/POLICY-PDP-PAP. Oldest: 26980737 ms [2023-01-13T06:39:19.112+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:19.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:19.114+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:19.114+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:19.114+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6745 message to send. [2023-01-13T06:39:19.114+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25700, above threshold 10. [2023-01-13T06:39:20.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6745 msgs to /events/POLICY-PDP-PAP. Oldest: 26981787 ms [2023-01-13T06:39:20.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:20.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:20.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:20.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:20.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6745 message to send. [2023-01-13T06:39:20.164+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25701, above threshold 10. [2023-01-13T06:39:20.653+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:21.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6746 msgs to /events/POLICY-PDP-PAP. Oldest: 26982837 ms [2023-01-13T06:39:21.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:21.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:21.214+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:21.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:21.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6746 message to send. [2023-01-13T06:39:21.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25702, above threshold 10. [2023-01-13T06:39:22.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6746 msgs to /events/POLICY-PDP-PAP. Oldest: 26983887 ms [2023-01-13T06:39:22.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:22.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:22.264+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:22.264+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:22.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6746 message to send. [2023-01-13T06:39:22.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25703, above threshold 10. [2023-01-13T06:39:23.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6746 msgs to /events/POLICY-PDP-PAP. Oldest: 26984937 ms [2023-01-13T06:39:23.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:23.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:23.314+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:23.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:23.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6746 message to send. [2023-01-13T06:39:23.315+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25704, above threshold 10. [2023-01-13T06:39:24.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6746 msgs to /events/POLICY-PDP-PAP. Oldest: 26985987 ms [2023-01-13T06:39:24.364+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:24.364+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:24.366+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:24.366+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:24.366+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6746 message to send. [2023-01-13T06:39:24.366+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25705, above threshold 10. [2023-01-13T06:39:24.653+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:25.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6747 msgs to /events/POLICY-PDP-PAP. Oldest: 26987037 ms [2023-01-13T06:39:25.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:25.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:25.415+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:25.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:25.416+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6747 message to send. [2023-01-13T06:39:25.416+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25706, above threshold 10. [2023-01-13T06:39:26.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6747 msgs to /events/POLICY-PDP-PAP. Oldest: 26988087 ms [2023-01-13T06:39:26.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:26.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:26.465+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:26.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:26.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6747 message to send. [2023-01-13T06:39:26.465+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25707, above threshold 10. [2023-01-13T06:39:27.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6747 msgs to /events/POLICY-PDP-PAP. Oldest: 26989137 ms [2023-01-13T06:39:27.514+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:27.514+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:27.516+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:27.516+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:27.516+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6747 message to send. [2023-01-13T06:39:27.516+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25708, above threshold 10. [2023-01-13T06:39:28.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6747 msgs to /events/POLICY-PDP-PAP. Oldest: 26990187 ms [2023-01-13T06:39:28.568+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:28.568+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:28.569+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:28.570+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:28.570+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6747 message to send. [2023-01-13T06:39:28.570+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25709, above threshold 10. [2023-01-13T06:39:28.653+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:29.610+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:39:29.610+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:39:29.610+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:39:29.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6748 msgs to /events/POLICY-PDP-PAP. Oldest: 26991237 ms [2023-01-13T06:39:29.612+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:29.612+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:29.612+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:39:29.612+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:29.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:29.614+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:29.614+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:29.614+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6748 message to send. [2023-01-13T06:39:29.614+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25710, above threshold 10. [2023-01-13T06:39:30.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6748 msgs to /events/POLICY-PDP-PAP. Oldest: 26992287 ms [2023-01-13T06:39:30.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:30.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:30.665+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:30.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:30.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6748 message to send. [2023-01-13T06:39:30.665+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25711, above threshold 10. [2023-01-13T06:39:31.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6748 msgs to /events/POLICY-PDP-PAP. Oldest: 26993337 ms [2023-01-13T06:39:31.712+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:31.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:31.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:31.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:31.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6748 message to send. [2023-01-13T06:39:31.715+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25712, above threshold 10. [2023-01-13T06:39:32.653+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:32.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6749 msgs to /events/POLICY-PDP-PAP. Oldest: 26994387 ms [2023-01-13T06:39:32.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:32.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:32.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:32.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:32.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6749 message to send. [2023-01-13T06:39:32.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25713, above threshold 10. [2023-01-13T06:39:33.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6749 msgs to /events/POLICY-PDP-PAP. Oldest: 26995437 ms [2023-01-13T06:39:33.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:33.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:33.814+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:33.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:33.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6749 message to send. [2023-01-13T06:39:33.815+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25714, above threshold 10. [2023-01-13T06:39:34.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6749 msgs to /events/POLICY-PDP-PAP. Oldest: 26996487 ms [2023-01-13T06:39:34.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:34.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:34.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:34.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:34.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6749 message to send. [2023-01-13T06:39:34.864+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25715, above threshold 10. [2023-01-13T06:39:35.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6749 msgs to /events/POLICY-PDP-PAP. Oldest: 26997537 ms [2023-01-13T06:39:35.913+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:35.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:35.915+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:35.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:35.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6749 message to send. [2023-01-13T06:39:35.915+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25716, above threshold 10. [2023-01-13T06:39:36.654+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:36.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6750 msgs to /events/POLICY-PDP-PAP. Oldest: 26998587 ms [2023-01-13T06:39:36.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:36.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:36.965+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:36.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:36.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6750 message to send. [2023-01-13T06:39:36.965+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25717, above threshold 10. [2023-01-13T06:39:38.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6750 msgs to /events/POLICY-PDP-PAP. Oldest: 26999637 ms [2023-01-13T06:39:38.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:38.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:38.015+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:38.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:38.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6750 message to send. [2023-01-13T06:39:38.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25718, above threshold 10. [2023-01-13T06:39:39.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6750 msgs to /events/POLICY-PDP-PAP. Oldest: 27000687 ms [2023-01-13T06:39:39.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:39.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:39.067+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:39.067+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:39.067+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6750 message to send. [2023-01-13T06:39:39.067+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25719, above threshold 10. [2023-01-13T06:39:40.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6750 msgs to /events/POLICY-PDP-PAP. Oldest: 27001737 ms [2023-01-13T06:39:40.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:40.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:40.116+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:40.117+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:40.117+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6750 message to send. [2023-01-13T06:39:40.117+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25720, above threshold 10. [2023-01-13T06:39:40.654+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:41.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6751 msgs to /events/POLICY-PDP-PAP. Oldest: 27002787 ms [2023-01-13T06:39:41.162+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:41.162+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:41.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:41.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:41.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6751 message to send. [2023-01-13T06:39:41.164+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25721, above threshold 10. [2023-01-13T06:39:42.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6751 msgs to /events/POLICY-PDP-PAP. Oldest: 27003837 ms [2023-01-13T06:39:42.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:42.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:42.215+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:42.216+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:42.216+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6751 message to send. [2023-01-13T06:39:42.216+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25722, above threshold 10. [2023-01-13T06:39:43.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6751 msgs to /events/POLICY-PDP-PAP. Oldest: 27004887 ms [2023-01-13T06:39:43.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:43.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:43.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:43.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:43.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6751 message to send. [2023-01-13T06:39:43.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25723, above threshold 10. [2023-01-13T06:39:44.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6751 msgs to /events/POLICY-PDP-PAP. Oldest: 27005937 ms [2023-01-13T06:39:44.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:44.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:44.316+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:44.317+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:44.317+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6751 message to send. [2023-01-13T06:39:44.317+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25724, above threshold 10. [2023-01-13T06:39:44.612+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:39:44.613+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:39:44.613+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:39:44.615+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:44.615+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:44.615+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:39:44.654+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:45.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6752 msgs to /events/POLICY-PDP-PAP. Oldest: 27006987 ms [2023-01-13T06:39:45.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:45.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:45.365+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:45.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:45.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6752 message to send. [2023-01-13T06:39:45.365+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25725, above threshold 10. [2023-01-13T06:39:46.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6752 msgs to /events/POLICY-PDP-PAP. Oldest: 27008037 ms [2023-01-13T06:39:46.412+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:46.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:46.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:46.414+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:46.414+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6752 message to send. [2023-01-13T06:39:46.414+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25726, above threshold 10. [2023-01-13T06:39:47.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6752 msgs to /events/POLICY-PDP-PAP. Oldest: 27009087 ms [2023-01-13T06:39:47.462+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:47.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:47.464+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:47.464+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:47.464+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6752 message to send. [2023-01-13T06:39:47.464+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25727, above threshold 10. [2023-01-13T06:39:48.511+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6752 msgs to /events/POLICY-PDP-PAP. Oldest: 27010138 ms [2023-01-13T06:39:48.516+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:48.516+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:48.518+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:48.518+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:48.518+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6752 message to send. [2023-01-13T06:39:48.518+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25728, above threshold 10. [2023-01-13T06:39:48.655+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:49.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6753 msgs to /events/POLICY-PDP-PAP. Oldest: 27011187 ms [2023-01-13T06:39:49.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:49.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:49.567+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:49.567+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:49.567+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6753 message to send. [2023-01-13T06:39:49.567+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25729, above threshold 10. [2023-01-13T06:39:50.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6753 msgs to /events/POLICY-PDP-PAP. Oldest: 27012237 ms [2023-01-13T06:39:50.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:50.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:50.614+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:50.614+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:50.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6753 message to send. [2023-01-13T06:39:50.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25730, above threshold 10. [2023-01-13T06:39:51.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6753 msgs to /events/POLICY-PDP-PAP. Oldest: 27013287 ms [2023-01-13T06:39:51.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:51.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:51.667+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:51.667+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:51.667+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6753 message to send. [2023-01-13T06:39:51.667+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25731, above threshold 10. [2023-01-13T06:39:52.655+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:52.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6754 msgs to /events/POLICY-PDP-PAP. Oldest: 27014337 ms [2023-01-13T06:39:52.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:52.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:52.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:52.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:52.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6754 message to send. [2023-01-13T06:39:52.714+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25732, above threshold 10. [2023-01-13T06:39:53.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6754 msgs to /events/POLICY-PDP-PAP. Oldest: 27015387 ms [2023-01-13T06:39:53.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:53.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:53.765+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:53.765+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:53.765+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6754 message to send. [2023-01-13T06:39:53.765+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25733, above threshold 10. [2023-01-13T06:39:54.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6754 msgs to /events/POLICY-PDP-PAP. Oldest: 27016437 ms [2023-01-13T06:39:54.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:54.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:54.815+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:54.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:54.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6754 message to send. [2023-01-13T06:39:54.816+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25734, above threshold 10. [2023-01-13T06:39:55.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6754 msgs to /events/POLICY-PDP-PAP. Oldest: 27017487 ms [2023-01-13T06:39:55.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:55.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:55.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:55.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:55.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6754 message to send. [2023-01-13T06:39:55.865+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25735, above threshold 10. [2023-01-13T06:39:56.655+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:39:56.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6755 msgs to /events/POLICY-PDP-PAP. Oldest: 27018537 ms [2023-01-13T06:39:56.912+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:56.912+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:56.914+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:56.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:56.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6755 message to send. [2023-01-13T06:39:56.914+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25736, above threshold 10. [2023-01-13T06:39:57.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6755 msgs to /events/POLICY-PDP-PAP. Oldest: 27019587 ms [2023-01-13T06:39:57.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:57.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:57.965+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:57.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:57.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6755 message to send. [2023-01-13T06:39:57.965+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25737, above threshold 10. [2023-01-13T06:39:59.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6755 msgs to /events/POLICY-PDP-PAP. Oldest: 27020637 ms [2023-01-13T06:39:59.014+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:39:59.014+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:39:59.016+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:59.016+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:39:59.016+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6755 message to send. [2023-01-13T06:39:59.016+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25738, above threshold 10. [2023-01-13T06:39:59.615+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:39:59.615+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:39:59.615+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:39:59.617+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:59.617+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:39:59.617+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:40:00.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6755 msgs to /events/POLICY-PDP-PAP. Oldest: 27021687 ms [2023-01-13T06:40:00.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:00.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:00.064+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:00.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:00.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6755 message to send. [2023-01-13T06:40:00.065+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25739, above threshold 10. [2023-01-13T06:40:00.655+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:01.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6756 msgs to /events/POLICY-PDP-PAP. Oldest: 27022737 ms [2023-01-13T06:40:01.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:01.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:01.115+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:01.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:01.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6756 message to send. [2023-01-13T06:40:01.115+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25740, above threshold 10. [2023-01-13T06:40:02.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6756 msgs to /events/POLICY-PDP-PAP. Oldest: 27023787 ms [2023-01-13T06:40:02.162+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:02.162+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:02.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:02.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:02.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6756 message to send. [2023-01-13T06:40:02.164+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25741, above threshold 10. [2023-01-13T06:40:03.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6756 msgs to /events/POLICY-PDP-PAP. Oldest: 27024837 ms [2023-01-13T06:40:03.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:03.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:03.214+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:03.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:03.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6756 message to send. [2023-01-13T06:40:03.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25742, above threshold 10. [2023-01-13T06:40:04.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6756 msgs to /events/POLICY-PDP-PAP. Oldest: 27025887 ms [2023-01-13T06:40:04.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:04.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:04.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:04.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:04.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6756 message to send. [2023-01-13T06:40:04.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25743, above threshold 10. [2023-01-13T06:40:04.655+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:05.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6757 msgs to /events/POLICY-PDP-PAP. Oldest: 27026937 ms [2023-01-13T06:40:05.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:05.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:05.315+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:05.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:05.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6757 message to send. [2023-01-13T06:40:05.315+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25744, above threshold 10. [2023-01-13T06:40:06.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6757 msgs to /events/POLICY-PDP-PAP. Oldest: 27027987 ms [2023-01-13T06:40:06.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:06.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:06.365+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:06.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:06.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6757 message to send. [2023-01-13T06:40:06.366+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25745, above threshold 10. [2023-01-13T06:40:07.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6757 msgs to /events/POLICY-PDP-PAP. Oldest: 27029037 ms [2023-01-13T06:40:07.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:07.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:07.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:07.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:07.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6757 message to send. [2023-01-13T06:40:07.415+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25746, above threshold 10. [2023-01-13T06:40:08.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6757 msgs to /events/POLICY-PDP-PAP. Oldest: 27030087 ms [2023-01-13T06:40:08.462+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:08.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:08.464+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:08.464+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:08.464+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6757 message to send. [2023-01-13T06:40:08.464+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25747, above threshold 10. [2023-01-13T06:40:08.656+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:09.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6758 msgs to /events/POLICY-PDP-PAP. Oldest: 27031137 ms [2023-01-13T06:40:09.515+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:09.515+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:09.516+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:09.517+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:09.517+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6758 message to send. [2023-01-13T06:40:09.517+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25748, above threshold 10. [2023-01-13T06:40:10.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6758 msgs to /events/POLICY-PDP-PAP. Oldest: 27032187 ms [2023-01-13T06:40:10.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:10.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:10.565+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:10.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:10.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6758 message to send. [2023-01-13T06:40:10.565+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25749, above threshold 10. [2023-01-13T06:40:11.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6758 msgs to /events/POLICY-PDP-PAP. Oldest: 27033237 ms [2023-01-13T06:40:11.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:11.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:11.615+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:11.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:11.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6758 message to send. [2023-01-13T06:40:11.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25750, above threshold 10. [2023-01-13T06:40:12.656+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:12.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6759 msgs to /events/POLICY-PDP-PAP. Oldest: 27034287 ms [2023-01-13T06:40:12.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:12.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:12.664+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:12.664+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:12.665+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6759 message to send. [2023-01-13T06:40:12.665+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25751, above threshold 10. [2023-01-13T06:40:13.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6759 msgs to /events/POLICY-PDP-PAP. Oldest: 27035337 ms [2023-01-13T06:40:13.714+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:13.714+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:13.716+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:13.716+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:13.716+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6759 message to send. [2023-01-13T06:40:13.716+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25752, above threshold 10. [2023-01-13T06:40:14.618+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:40:14.618+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:40:14.618+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:40:14.620+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:14.620+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:14.620+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:40:14.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6759 msgs to /events/POLICY-PDP-PAP. Oldest: 27036387 ms [2023-01-13T06:40:14.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:14.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:14.767+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:14.767+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:14.767+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6759 message to send. [2023-01-13T06:40:14.767+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25753, above threshold 10. [2023-01-13T06:40:15.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6759 msgs to /events/POLICY-PDP-PAP. Oldest: 27037437 ms [2023-01-13T06:40:15.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:15.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:15.814+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:15.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:15.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6759 message to send. [2023-01-13T06:40:15.814+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25754, above threshold 10. [2023-01-13T06:40:16.656+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:16.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6760 msgs to /events/POLICY-PDP-PAP. Oldest: 27038487 ms [2023-01-13T06:40:16.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:16.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:16.865+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:16.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:16.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6760 message to send. [2023-01-13T06:40:16.865+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25755, above threshold 10. [2023-01-13T06:40:17.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6760 msgs to /events/POLICY-PDP-PAP. Oldest: 27039537 ms [2023-01-13T06:40:17.913+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:17.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:17.915+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:17.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:17.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6760 message to send. [2023-01-13T06:40:17.915+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25756, above threshold 10. [2023-01-13T06:40:18.962+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6760 msgs to /events/POLICY-PDP-PAP. Oldest: 27040589 ms [2023-01-13T06:40:18.964+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:18.964+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:18.966+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:18.966+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:18.966+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6760 message to send. [2023-01-13T06:40:18.966+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25757, above threshold 10. [2023-01-13T06:40:20.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6760 msgs to /events/POLICY-PDP-PAP. Oldest: 27041637 ms [2023-01-13T06:40:20.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:20.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:20.015+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:20.016+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:20.016+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6760 message to send. [2023-01-13T06:40:20.016+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25758, above threshold 10. [2023-01-13T06:40:20.656+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:21.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6761 msgs to /events/POLICY-PDP-PAP. Oldest: 27042687 ms [2023-01-13T06:40:21.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:21.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:21.067+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:21.067+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:21.067+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6761 message to send. [2023-01-13T06:40:21.067+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25759, above threshold 10. [2023-01-13T06:40:22.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6761 msgs to /events/POLICY-PDP-PAP. Oldest: 27043737 ms [2023-01-13T06:40:22.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:22.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:22.116+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:22.116+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:22.116+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6761 message to send. [2023-01-13T06:40:22.116+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25760, above threshold 10. [2023-01-13T06:40:23.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6761 msgs to /events/POLICY-PDP-PAP. Oldest: 27044787 ms [2023-01-13T06:40:23.162+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:23.162+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:23.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:23.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:23.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6761 message to send. [2023-01-13T06:40:23.164+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25761, above threshold 10. [2023-01-13T06:40:24.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6761 msgs to /events/POLICY-PDP-PAP. Oldest: 27045837 ms [2023-01-13T06:40:24.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:24.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:24.214+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:24.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:24.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6761 message to send. [2023-01-13T06:40:24.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25762, above threshold 10. [2023-01-13T06:40:24.656+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:25.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6762 msgs to /events/POLICY-PDP-PAP. Oldest: 27046887 ms [2023-01-13T06:40:25.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:25.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:25.264+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:25.264+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:25.264+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6762 message to send. [2023-01-13T06:40:25.264+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25763, above threshold 10. [2023-01-13T06:40:26.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6762 msgs to /events/POLICY-PDP-PAP. Oldest: 27047937 ms [2023-01-13T06:40:26.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:26.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:26.314+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:26.314+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:26.314+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6762 message to send. [2023-01-13T06:40:26.314+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25764, above threshold 10. [2023-01-13T06:40:27.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6762 msgs to /events/POLICY-PDP-PAP. Oldest: 27048987 ms [2023-01-13T06:40:27.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:27.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:27.364+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:27.364+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:27.364+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6762 message to send. [2023-01-13T06:40:27.364+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25765, above threshold 10. [2023-01-13T06:40:28.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6762 msgs to /events/POLICY-PDP-PAP. Oldest: 27050037 ms [2023-01-13T06:40:28.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:28.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:28.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:28.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:28.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6762 message to send. [2023-01-13T06:40:28.415+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25766, above threshold 10. [2023-01-13T06:40:28.656+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:29.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6763 msgs to /events/POLICY-PDP-PAP. Oldest: 27051087 ms [2023-01-13T06:40:29.464+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:29.464+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:29.466+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:29.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:29.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6763 message to send. [2023-01-13T06:40:29.466+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25767, above threshold 10. [2023-01-13T06:40:29.620+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:40:29.621+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:40:29.621+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:40:29.623+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:29.623+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:29.623+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:40:30.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6763 msgs to /events/POLICY-PDP-PAP. Oldest: 27052137 ms [2023-01-13T06:40:30.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:30.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:30.515+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:30.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:30.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6763 message to send. [2023-01-13T06:40:30.515+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25768, above threshold 10. [2023-01-13T06:40:31.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6763 msgs to /events/POLICY-PDP-PAP. Oldest: 27053187 ms [2023-01-13T06:40:31.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:31.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:31.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:31.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:31.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6763 message to send. [2023-01-13T06:40:31.564+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25769, above threshold 10. [2023-01-13T06:40:32.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6763 msgs to /events/POLICY-PDP-PAP. Oldest: 27054237 ms [2023-01-13T06:40:32.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:32.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:32.615+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:32.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:32.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6763 message to send. [2023-01-13T06:40:32.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25770, above threshold 10. [2023-01-13T06:40:32.657+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:33.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6764 msgs to /events/POLICY-PDP-PAP. Oldest: 27055287 ms [2023-01-13T06:40:33.662+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:33.662+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:33.664+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:33.664+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:33.664+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6764 message to send. [2023-01-13T06:40:33.664+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25771, above threshold 10. [2023-01-13T06:40:34.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6764 msgs to /events/POLICY-PDP-PAP. Oldest: 27056337 ms [2023-01-13T06:40:34.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:34.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:34.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:34.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:34.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6764 message to send. [2023-01-13T06:40:34.715+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25772, above threshold 10. [2023-01-13T06:40:35.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6764 msgs to /events/POLICY-PDP-PAP. Oldest: 27057387 ms [2023-01-13T06:40:35.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:35.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:35.765+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:35.765+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:35.765+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6764 message to send. [2023-01-13T06:40:35.765+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25773, above threshold 10. [2023-01-13T06:40:36.657+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:36.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6765 msgs to /events/POLICY-PDP-PAP. Oldest: 27058437 ms [2023-01-13T06:40:36.812+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:36.812+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:36.816+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:36.816+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:36.816+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6765 message to send. [2023-01-13T06:40:36.816+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25774, above threshold 10. [2023-01-13T06:40:37.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6765 msgs to /events/POLICY-PDP-PAP. Oldest: 27059487 ms [2023-01-13T06:40:37.862+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:37.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:37.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:37.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:37.864+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6765 message to send. [2023-01-13T06:40:37.864+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25775, above threshold 10. [2023-01-13T06:40:38.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6765 msgs to /events/POLICY-PDP-PAP. Oldest: 27060537 ms [2023-01-13T06:40:38.912+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:38.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:38.914+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:38.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:38.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6765 message to send. [2023-01-13T06:40:38.915+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25776, above threshold 10. [2023-01-13T06:40:39.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6765 msgs to /events/POLICY-PDP-PAP. Oldest: 27061587 ms [2023-01-13T06:40:39.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:39.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:39.965+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:39.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:39.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6765 message to send. [2023-01-13T06:40:39.965+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25777, above threshold 10. [2023-01-13T06:40:40.657+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:41.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6766 msgs to /events/POLICY-PDP-PAP. Oldest: 27062637 ms [2023-01-13T06:40:41.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:41.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:41.015+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:41.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:41.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6766 message to send. [2023-01-13T06:40:41.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25778, above threshold 10. [2023-01-13T06:40:42.061+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6766 msgs to /events/POLICY-PDP-PAP. Oldest: 27063688 ms [2023-01-13T06:40:42.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:42.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:42.065+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:42.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:42.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6766 message to send. [2023-01-13T06:40:42.065+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25779, above threshold 10. [2023-01-13T06:40:43.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6766 msgs to /events/POLICY-PDP-PAP. Oldest: 27064737 ms [2023-01-13T06:40:43.114+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:43.114+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:43.116+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:43.116+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:43.116+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6766 message to send. [2023-01-13T06:40:43.116+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25780, above threshold 10. [2023-01-13T06:40:44.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6766 msgs to /events/POLICY-PDP-PAP. Oldest: 27065787 ms [2023-01-13T06:40:44.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:44.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:44.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:44.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:44.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6766 message to send. [2023-01-13T06:40:44.165+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25781, above threshold 10. [2023-01-13T06:40:44.623+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:40:44.623+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:40:44.623+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:40:44.626+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:44.626+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:44.626+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:40:44.657+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:45.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6767 msgs to /events/POLICY-PDP-PAP. Oldest: 27066837 ms [2023-01-13T06:40:45.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:45.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:45.217+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:45.217+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:45.218+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6767 message to send. [2023-01-13T06:40:45.218+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25782, above threshold 10. [2023-01-13T06:40:46.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6767 msgs to /events/POLICY-PDP-PAP. Oldest: 27067887 ms [2023-01-13T06:40:46.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:46.264+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:46.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:46.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:46.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6767 message to send. [2023-01-13T06:40:46.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25783, above threshold 10. [2023-01-13T06:40:47.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6767 msgs to /events/POLICY-PDP-PAP. Oldest: 27068937 ms [2023-01-13T06:40:47.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:47.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:47.315+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:47.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:47.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6767 message to send. [2023-01-13T06:40:47.315+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25784, above threshold 10. [2023-01-13T06:40:48.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6767 msgs to /events/POLICY-PDP-PAP. Oldest: 27069987 ms [2023-01-13T06:40:48.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:48.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:48.365+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:48.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:48.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6767 message to send. [2023-01-13T06:40:48.365+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25785, above threshold 10. [2023-01-13T06:40:48.658+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:49.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6768 msgs to /events/POLICY-PDP-PAP. Oldest: 27071037 ms [2023-01-13T06:40:49.416+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:49.416+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:49.420+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:49.420+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:49.420+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6768 message to send. [2023-01-13T06:40:49.420+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25786, above threshold 10. [2023-01-13T06:40:50.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6768 msgs to /events/POLICY-PDP-PAP. Oldest: 27072087 ms [2023-01-13T06:40:50.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:50.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:50.466+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:50.467+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:50.467+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6768 message to send. [2023-01-13T06:40:50.467+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25787, above threshold 10. [2023-01-13T06:40:51.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6768 msgs to /events/POLICY-PDP-PAP. Oldest: 27073137 ms [2023-01-13T06:40:51.512+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:51.512+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:51.519+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:51.520+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:51.520+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6768 message to send. [2023-01-13T06:40:51.520+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25788, above threshold 10. [2023-01-13T06:40:52.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6768 msgs to /events/POLICY-PDP-PAP. Oldest: 27074187 ms [2023-01-13T06:40:52.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:52.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:52.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:52.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:52.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6768 message to send. [2023-01-13T06:40:52.564+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25789, above threshold 10. [2023-01-13T06:40:52.658+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:53.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6769 msgs to /events/POLICY-PDP-PAP. Oldest: 27075237 ms [2023-01-13T06:40:53.614+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:53.614+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:53.615+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:53.616+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:53.616+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6769 message to send. [2023-01-13T06:40:53.616+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25790, above threshold 10. [2023-01-13T06:40:54.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6769 msgs to /events/POLICY-PDP-PAP. Oldest: 27076287 ms [2023-01-13T06:40:54.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:54.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:54.665+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:54.666+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:54.666+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6769 message to send. [2023-01-13T06:40:54.666+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25791, above threshold 10. [2023-01-13T06:40:55.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6769 msgs to /events/POLICY-PDP-PAP. Oldest: 27077337 ms [2023-01-13T06:40:55.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:55.714+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:55.715+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:55.716+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:55.716+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6769 message to send. [2023-01-13T06:40:55.716+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25792, above threshold 10. [2023-01-13T06:40:56.658+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:40:56.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6770 msgs to /events/POLICY-PDP-PAP. Oldest: 27078387 ms [2023-01-13T06:40:56.762+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:56.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:56.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:56.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:56.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6770 message to send. [2023-01-13T06:40:56.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25793, above threshold 10. [2023-01-13T06:40:57.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6770 msgs to /events/POLICY-PDP-PAP. Oldest: 27079437 ms [2023-01-13T06:40:57.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:57.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:57.815+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:57.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:57.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6770 message to send. [2023-01-13T06:40:57.815+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25794, above threshold 10. [2023-01-13T06:40:58.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6770 msgs to /events/POLICY-PDP-PAP. Oldest: 27080487 ms [2023-01-13T06:40:58.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:58.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:58.865+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:58.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:58.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6770 message to send. [2023-01-13T06:40:58.865+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25795, above threshold 10. [2023-01-13T06:40:59.627+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:40:59.627+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:40:59.627+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:40:59.629+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:59.629+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:59.629+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:40:59.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6770 msgs to /events/POLICY-PDP-PAP. Oldest: 27081537 ms [2023-01-13T06:40:59.913+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:40:59.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:40:59.914+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:40:59.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:40:59.914+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6770 message to send. [2023-01-13T06:40:59.914+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25796, above threshold 10. [2023-01-13T06:41:00.658+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:00.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6771 msgs to /events/POLICY-PDP-PAP. Oldest: 27082587 ms [2023-01-13T06:41:00.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:00.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:00.964+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:00.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:00.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6771 message to send. [2023-01-13T06:41:00.965+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25797, above threshold 10. [2023-01-13T06:41:02.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6771 msgs to /events/POLICY-PDP-PAP. Oldest: 27083637 ms [2023-01-13T06:41:02.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:02.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:02.015+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:02.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:02.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6771 message to send. [2023-01-13T06:41:02.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25798, above threshold 10. [2023-01-13T06:41:03.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6771 msgs to /events/POLICY-PDP-PAP. Oldest: 27084687 ms [2023-01-13T06:41:03.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:03.064+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:03.065+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:03.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:03.065+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6771 message to send. [2023-01-13T06:41:03.065+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25799, above threshold 10. [2023-01-13T06:41:04.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6771 msgs to /events/POLICY-PDP-PAP. Oldest: 27085737 ms [2023-01-13T06:41:04.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:04.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:04.115+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:04.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:04.115+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6771 message to send. [2023-01-13T06:41:04.115+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25800, above threshold 10. [2023-01-13T06:41:04.658+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:05.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6772 msgs to /events/POLICY-PDP-PAP. Oldest: 27086787 ms [2023-01-13T06:41:05.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:05.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:05.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:05.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:05.164+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6772 message to send. [2023-01-13T06:41:05.164+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25801, above threshold 10. [2023-01-13T06:41:06.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6772 msgs to /events/POLICY-PDP-PAP. Oldest: 27087837 ms [2023-01-13T06:41:06.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:06.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:06.215+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:06.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:06.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6772 message to send. [2023-01-13T06:41:06.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25802, above threshold 10. [2023-01-13T06:41:07.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6772 msgs to /events/POLICY-PDP-PAP. Oldest: 27088887 ms [2023-01-13T06:41:07.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:07.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:07.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:07.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:07.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6772 message to send. [2023-01-13T06:41:07.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25803, above threshold 10. [2023-01-13T06:41:08.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6772 msgs to /events/POLICY-PDP-PAP. Oldest: 27089937 ms [2023-01-13T06:41:08.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:08.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:08.315+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:08.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:08.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6772 message to send. [2023-01-13T06:41:08.315+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25804, above threshold 10. [2023-01-13T06:41:08.659+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:09.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6773 msgs to /events/POLICY-PDP-PAP. Oldest: 27090987 ms [2023-01-13T06:41:09.366+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:09.366+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:09.367+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:09.367+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:09.367+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6773 message to send. [2023-01-13T06:41:09.367+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25805, above threshold 10. [2023-01-13T06:41:10.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6773 msgs to /events/POLICY-PDP-PAP. Oldest: 27092037 ms [2023-01-13T06:41:10.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:10.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:10.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:10.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:10.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6773 message to send. [2023-01-13T06:41:10.415+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25806, above threshold 10. [2023-01-13T06:41:11.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6773 msgs to /events/POLICY-PDP-PAP. Oldest: 27093087 ms [2023-01-13T06:41:11.464+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:11.464+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:11.465+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:11.465+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:11.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6773 message to send. [2023-01-13T06:41:11.466+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25807, above threshold 10. [2023-01-13T06:41:12.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6773 msgs to /events/POLICY-PDP-PAP. Oldest: 27094137 ms [2023-01-13T06:41:12.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:12.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:12.514+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:12.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:12.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6773 message to send. [2023-01-13T06:41:12.515+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25808, above threshold 10. [2023-01-13T06:41:12.659+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:13.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6774 msgs to /events/POLICY-PDP-PAP. Oldest: 27095187 ms [2023-01-13T06:41:13.562+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:13.562+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:13.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:13.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:13.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6774 message to send. [2023-01-13T06:41:13.564+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25809, above threshold 10. [2023-01-13T06:41:14.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6774 msgs to /events/POLICY-PDP-PAP. Oldest: 27096237 ms [2023-01-13T06:41:14.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:14.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:14.615+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:14.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:14.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6774 message to send. [2023-01-13T06:41:14.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25810, above threshold 10. [2023-01-13T06:41:14.629+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:41:14.629+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:41:14.629+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:41:14.631+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:14.631+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:14.631+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:41:15.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6774 msgs to /events/POLICY-PDP-PAP. Oldest: 27097287 ms [2023-01-13T06:41:15.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:15.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:15.666+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:15.667+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:15.667+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6774 message to send. [2023-01-13T06:41:15.667+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25811, above threshold 10. [2023-01-13T06:41:16.659+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:16.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6775 msgs to /events/POLICY-PDP-PAP. Oldest: 27098337 ms [2023-01-13T06:41:16.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:16.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:16.715+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:16.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:16.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6775 message to send. [2023-01-13T06:41:16.715+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25812, above threshold 10. [2023-01-13T06:41:17.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6775 msgs to /events/POLICY-PDP-PAP. Oldest: 27099387 ms [2023-01-13T06:41:17.762+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:17.762+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:17.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:17.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:17.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6775 message to send. [2023-01-13T06:41:17.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25813, above threshold 10. [2023-01-13T06:41:18.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6775 msgs to /events/POLICY-PDP-PAP. Oldest: 27100437 ms [2023-01-13T06:41:18.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:18.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:18.814+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:18.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:18.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6775 message to send. [2023-01-13T06:41:18.815+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25814, above threshold 10. [2023-01-13T06:41:19.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6775 msgs to /events/POLICY-PDP-PAP. Oldest: 27101487 ms [2023-01-13T06:41:19.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:19.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:19.865+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:19.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:19.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6775 message to send. [2023-01-13T06:41:19.865+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25815, above threshold 10. [2023-01-13T06:41:20.659+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:20.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6776 msgs to /events/POLICY-PDP-PAP. Oldest: 27102537 ms [2023-01-13T06:41:20.913+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:20.913+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:20.914+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:20.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:20.915+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6776 message to send. [2023-01-13T06:41:20.915+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25816, above threshold 10. [2023-01-13T06:41:21.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6776 msgs to /events/POLICY-PDP-PAP. Oldest: 27103587 ms [2023-01-13T06:41:21.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:21.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:21.965+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:21.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:21.965+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6776 message to send. [2023-01-13T06:41:21.965+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25817, above threshold 10. [2023-01-13T06:41:23.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6776 msgs to /events/POLICY-PDP-PAP. Oldest: 27104637 ms [2023-01-13T06:41:23.012+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:23.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:23.014+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:23.014+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:23.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6776 message to send. [2023-01-13T06:41:23.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25818, above threshold 10. [2023-01-13T06:41:24.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6776 msgs to /events/POLICY-PDP-PAP. Oldest: 27105687 ms [2023-01-13T06:41:24.063+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:24.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:24.064+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:24.064+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:24.064+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6776 message to send. [2023-01-13T06:41:24.064+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25819, above threshold 10. [2023-01-13T06:41:24.659+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:25.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6777 msgs to /events/POLICY-PDP-PAP. Oldest: 27106737 ms [2023-01-13T06:41:25.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:25.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:25.114+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:25.114+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:25.114+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6777 message to send. [2023-01-13T06:41:25.115+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25820, above threshold 10. [2023-01-13T06:41:26.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6777 msgs to /events/POLICY-PDP-PAP. Oldest: 27107787 ms [2023-01-13T06:41:26.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:26.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:26.165+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:26.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:26.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6777 message to send. [2023-01-13T06:41:26.165+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25821, above threshold 10. [2023-01-13T06:41:27.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6777 msgs to /events/POLICY-PDP-PAP. Oldest: 27108837 ms [2023-01-13T06:41:27.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:27.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:27.215+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:27.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:27.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6777 message to send. [2023-01-13T06:41:27.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25822, above threshold 10. [2023-01-13T06:41:28.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6777 msgs to /events/POLICY-PDP-PAP. Oldest: 27109887 ms [2023-01-13T06:41:28.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:28.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:28.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:28.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:28.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6777 message to send. [2023-01-13T06:41:28.265+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25823, above threshold 10. [2023-01-13T06:41:28.659+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:29.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6778 msgs to /events/POLICY-PDP-PAP. Oldest: 27110937 ms [2023-01-13T06:41:29.313+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:29.313+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:29.315+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:29.315+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:29.316+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6778 message to send. [2023-01-13T06:41:29.316+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25824, above threshold 10. [2023-01-13T06:41:29.632+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:41:29.632+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:41:29.632+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:41:29.634+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:29.634+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:29.634+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:41:30.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6778 msgs to /events/POLICY-PDP-PAP. Oldest: 27111987 ms [2023-01-13T06:41:30.365+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:30.365+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:30.366+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:30.366+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:30.366+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6778 message to send. [2023-01-13T06:41:30.366+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25825, above threshold 10. [2023-01-13T06:41:31.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6778 msgs to /events/POLICY-PDP-PAP. Oldest: 27113037 ms [2023-01-13T06:41:31.414+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:31.414+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:31.417+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:31.417+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:31.417+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6778 message to send. [2023-01-13T06:41:31.417+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25826, above threshold 10. [2023-01-13T06:41:32.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6778 msgs to /events/POLICY-PDP-PAP. Oldest: 27114087 ms [2023-01-13T06:41:32.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:32.463+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:32.464+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:32.464+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:32.464+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6778 message to send. [2023-01-13T06:41:32.464+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25827, above threshold 10. [2023-01-13T06:41:32.660+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:33.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6779 msgs to /events/POLICY-PDP-PAP. Oldest: 27115137 ms [2023-01-13T06:41:33.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:33.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:33.514+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:33.514+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:33.514+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6779 message to send. [2023-01-13T06:41:33.514+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25828, above threshold 10. [2023-01-13T06:41:34.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6779 msgs to /events/POLICY-PDP-PAP. Oldest: 27116187 ms [2023-01-13T06:41:34.562+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:34.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:34.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:34.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:34.564+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6779 message to send. [2023-01-13T06:41:34.564+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25829, above threshold 10. [2023-01-13T06:41:35.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6779 msgs to /events/POLICY-PDP-PAP. Oldest: 27117237 ms [2023-01-13T06:41:35.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:35.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:35.614+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:35.614+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:35.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6779 message to send. [2023-01-13T06:41:35.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25830, above threshold 10. [2023-01-13T06:41:36.660+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:36.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6780 msgs to /events/POLICY-PDP-PAP. Oldest: 27118287 ms [2023-01-13T06:41:36.663+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:36.663+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:36.664+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:36.664+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:36.664+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6780 message to send. [2023-01-13T06:41:36.664+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25831, above threshold 10. [2023-01-13T06:41:37.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6780 msgs to /events/POLICY-PDP-PAP. Oldest: 27119337 ms [2023-01-13T06:41:37.713+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:37.713+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:37.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:37.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:37.715+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6780 message to send. [2023-01-13T06:41:37.715+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25832, above threshold 10. [2023-01-13T06:41:38.760+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6780 msgs to /events/POLICY-PDP-PAP. Oldest: 27120387 ms [2023-01-13T06:41:38.763+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:38.763+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:38.764+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:38.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:38.764+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6780 message to send. [2023-01-13T06:41:38.764+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25833, above threshold 10. [2023-01-13T06:41:39.810+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6780 msgs to /events/POLICY-PDP-PAP. Oldest: 27121437 ms [2023-01-13T06:41:39.813+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:39.813+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:39.814+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:39.814+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:39.815+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6780 message to send. [2023-01-13T06:41:39.815+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25834, above threshold 10. [2023-01-13T06:41:40.660+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:40.860+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6781 msgs to /events/POLICY-PDP-PAP. Oldest: 27122487 ms [2023-01-13T06:41:40.863+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:40.863+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:40.864+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:40.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:40.865+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6781 message to send. [2023-01-13T06:41:40.865+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25835, above threshold 10. [2023-01-13T06:41:41.910+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6781 msgs to /events/POLICY-PDP-PAP. Oldest: 27123537 ms [2023-01-13T06:41:41.914+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:41.914+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:41.915+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:41.916+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:41.916+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6781 message to send. [2023-01-13T06:41:41.916+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25836, above threshold 10. [2023-01-13T06:41:42.960+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6781 msgs to /events/POLICY-PDP-PAP. Oldest: 27124587 ms [2023-01-13T06:41:42.963+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:42.963+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:42.966+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:42.966+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:42.966+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6781 message to send. [2023-01-13T06:41:42.966+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25837, above threshold 10. [2023-01-13T06:41:44.010+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6781 msgs to /events/POLICY-PDP-PAP. Oldest: 27125637 ms [2023-01-13T06:41:44.013+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:44.013+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:44.014+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:44.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:44.015+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6781 message to send. [2023-01-13T06:41:44.015+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25838, above threshold 10. [2023-01-13T06:41:44.635+00:00|ERROR|SingleThreadedBusTopicSource|DMAAP-source-POLICY-PDP-PAP] SingleThreadedDmaapTopicSource [userName=null, password=-, getTopicCommInfrastructure()=DMAAP, toString()=SingleThreadedBusTopicSource [consumerGroup=2fd10729-ab99-456e-a6b0-5ed98d3c44f2, consumerInstance=onap-policy-xacml-pdp-7dcdcdcf66-4p7hn, fetchTimeout=15000, fetchLimit=-1, consumer=CambriaConsumerWrapper [fetchTimeout=15000], alive=true, locked=false, uebThread=Thread[DMAAP-source-POLICY-PDP-PAP,5,main], topicListeners=2, toString()=BusTopicBase [apiKey=null, apiSecret=null, useHttps=false, allowSelfSignedCerts=false, toString()=TopicBase [servers=[message-router], topic=POLICY-PDP-PAP, effectiveTopic=POLICY-PDP-PAP, #recentEvents=0, locked=false, #topicListeners=2]]]]: cannot fetch java.io.IOException: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:123) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:64) at org.onap.policy.common.endpoints.event.comm.bus.internal.BusConsumer$CambriaConsumerWrapper.fetch(BusConsumer.java:212) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.fetchAllMessages(SingleThreadedBusTopicSource.java:244) at org.onap.policy.common.endpoints.event.comm.bus.internal.SingleThreadedBusTopicSource.run(SingleThreadedBusTopicSource.java:234) at java.base/java.lang.Thread.run(Unknown Source) Caused by: com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:384) at com.att.nsa.apiClient.http.HttpClient.get(HttpClient.java:368) at com.att.nsa.cambria.client.impl.CambriaConsumerImpl.fetch(CambriaConsumerImpl.java:87) ... 5 common frames omitted [2023-01-13T06:41:44.635+00:00|INFO|CambriaConsumerImpl|DMAAP-source-POLICY-PDP-PAP] UEB GET /events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 [2023-01-13T06:41:44.635+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] GET http://message-router:3904/events/POLICY-PDP-PAP/2fd10729-ab99-456e-a6b0-5ed98d3c44f2/onap-policy-xacml-pdp-7dcdcdcf66-4p7hn?timeout=15000 (anonymous) ... [2023-01-13T06:41:44.637+00:00|INFO|HttpClient|DMAAP-source-POLICY-PDP-PAP] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:44.638+00:00|ERROR|BusConsumer$CambriaConsumerWrapper|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: cannot fetch because of com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:44.638+00:00|INFO|BusConsumer$FetchingBusConsumer|DMAAP-source-POLICY-PDP-PAP] CambriaConsumerWrapper [fetchTimeout=15000]: backoff for 15000ms [2023-01-13T06:41:44.660+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:45.060+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6782 msgs to /events/POLICY-PDP-PAP. Oldest: 27126687 ms [2023-01-13T06:41:45.062+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:45.063+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:45.066+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:45.066+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:45.066+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6782 message to send. [2023-01-13T06:41:45.066+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25839, above threshold 10. [2023-01-13T06:41:46.110+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6782 msgs to /events/POLICY-PDP-PAP. Oldest: 27127737 ms [2023-01-13T06:41:46.113+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:46.113+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:46.117+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:46.118+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:46.118+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6782 message to send. [2023-01-13T06:41:46.118+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25840, above threshold 10. [2023-01-13T06:41:47.160+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6782 msgs to /events/POLICY-PDP-PAP. Oldest: 27128787 ms [2023-01-13T06:41:47.163+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:47.163+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:47.164+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:47.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:47.165+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6782 message to send. [2023-01-13T06:41:47.165+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25841, above threshold 10. [2023-01-13T06:41:48.210+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6782 msgs to /events/POLICY-PDP-PAP. Oldest: 27129837 ms [2023-01-13T06:41:48.213+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:48.213+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:48.215+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:48.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:48.215+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6782 message to send. [2023-01-13T06:41:48.215+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25842, above threshold 10. [2023-01-13T06:41:48.660+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:49.260+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6783 msgs to /events/POLICY-PDP-PAP. Oldest: 27130887 ms [2023-01-13T06:41:49.263+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:49.263+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:49.265+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:49.265+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:49.266+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6783 message to send. [2023-01-13T06:41:49.266+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25843, above threshold 10. [2023-01-13T06:41:50.310+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6783 msgs to /events/POLICY-PDP-PAP. Oldest: 27131937 ms [2023-01-13T06:41:50.316+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:50.316+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:50.320+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:50.320+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:50.321+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6783 message to send. [2023-01-13T06:41:50.321+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25844, above threshold 10. [2023-01-13T06:41:51.360+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6783 msgs to /events/POLICY-PDP-PAP. Oldest: 27132987 ms [2023-01-13T06:41:51.363+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:51.363+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:51.365+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:51.365+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:51.366+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6783 message to send. [2023-01-13T06:41:51.366+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25845, above threshold 10. [2023-01-13T06:41:52.410+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6783 msgs to /events/POLICY-PDP-PAP. Oldest: 27134037 ms [2023-01-13T06:41:52.413+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:52.413+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:52.414+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:52.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:52.415+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6783 message to send. [2023-01-13T06:41:52.415+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25846, above threshold 10. [2023-01-13T06:41:52.661+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:53.460+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6784 msgs to /events/POLICY-PDP-PAP. Oldest: 27135087 ms [2023-01-13T06:41:53.463+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:53.464+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:53.466+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:53.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:53.466+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6784 message to send. [2023-01-13T06:41:53.466+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25847, above threshold 10. [2023-01-13T06:41:54.510+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6784 msgs to /events/POLICY-PDP-PAP. Oldest: 27136137 ms [2023-01-13T06:41:54.513+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:54.513+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:54.514+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:54.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:54.515+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6784 message to send. [2023-01-13T06:41:54.515+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25848, above threshold 10. [2023-01-13T06:41:55.560+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6784 msgs to /events/POLICY-PDP-PAP. Oldest: 27137187 ms [2023-01-13T06:41:55.563+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:55.563+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:55.564+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:55.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:55.565+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6784 message to send. [2023-01-13T06:41:55.565+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25849, above threshold 10. [2023-01-13T06:41:56.610+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6784 msgs to /events/POLICY-PDP-PAP. Oldest: 27138237 ms [2023-01-13T06:41:56.613+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:56.613+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:56.615+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:56.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:56.615+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6784 message to send. [2023-01-13T06:41:56.615+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25850, above threshold 10. [2023-01-13T06:41:56.661+00:00|INFO|network|pool-3-thread-1] [OUT|DMAAP|POLICY-PDP-PAP] {"messageName":"PDP_TOPIC_CHECK","requestId":"31d4ad6e-e27e-421d-839c-a48d518a8325","timestampMs":1673564978316,"name":"xacml-7fff302f-dddd-4a3c-baff-cca5a96214dc"} [2023-01-13T06:41:57.660+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6785 msgs to /events/POLICY-PDP-PAP. Oldest: 27139287 ms [2023-01-13T06:41:57.664+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:57.664+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:57.666+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:57.666+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:57.666+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6785 message to send. [2023-01-13T06:41:57.666+00:00|ERROR|CambriaSimplerBatchPublisher|pool-2-thread-1] PUB_CHRONIC_FAILURE: Send failure count is 25851, above threshold 10. [2023-01-13T06:41:58.710+00:00|INFO|CambriaSimplerBatchPublisher|pool-2-thread-1] sending 6785 msgs to /events/POLICY-PDP-PAP. Oldest: 27140337 ms [2023-01-13T06:41:58.712+00:00|WARN|HostSelector|pool-2-thread-1] All hosts were blacklisted; reverting to full set of hosts. [2023-01-13T06:41:58.712+00:00|INFO|HttpClient|pool-2-thread-1] POST http://message-router:3904/events/POLICY-PDP-PAP (anonymous) ... [2023-01-13T06:41:58.714+00:00|INFO|HttpClient|pool-2-thread-1] --> HTTP/1.1 503 Service Unavailable [2023-01-13T06:41:58.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] HTTP/1.1 503 Service Unavailable com.att.nsa.apiClient.http.HttpException: HTTP/1.1 503 Service Unavailable at com.att.nsa.apiClient.http.HttpClient.runCall(HttpClient.java:816) at com.att.nsa.apiClient.http.HttpClient.post(HttpClient.java:456) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.sendBatch(CambriaSimplerBatchPublisher.java:342) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.send(CambriaSimplerBatchPublisher.java:251) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher.access$100(CambriaSimplerBatchPublisher.java:31) at com.att.nsa.cambria.client.impl.CambriaSimplerBatchPublisher$1.run(CambriaSimplerBatchPublisher.java:411) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) [2023-01-13T06:41:58.714+00:00|WARN|CambriaSimplerBatchPublisher|pool-2-thread-1] Send failed, 6785 message to send. [2023-01-13T06:41:58.714+00:00|ERROR|CambriaSimplerBatchPublishe