at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:52:57.332 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:52:57,332 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:52:57:332", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.68.101"} "04:52:57.332 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:52:57:332", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.68.101"} "2022-05-29 04:52:57,332 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:52:57.332 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:52:57,486 [qtp1022308509-35518] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:52:57.486 [qtp1022308509-35518] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:52:57,487 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:52:57.487 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:52:57,489 [qtp1022308509-35518] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:52:57.489 [qtp1022308509-35518] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:52:57,490 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:52:57.490 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:52:58,327 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:52:58.327 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:52:58,330 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:52:58.330 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:52:58,483 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:52:58.483 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:52:58,483 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:52:58.483 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:52:58,487 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:52:58.487 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:52:58,487 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:52:58.487 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:00,740 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:00.740 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:00,742 [qtp1022308509-35515] INFO org.onap.dmaap.service.TopicRestService - Authenticating the user before fetching the topics "04:53:00.742 [qtp1022308509-35515] INFO org.onap.dmaap.service.TopicRestService - Authenticating the user before fetching the topics "2022-05-29 04:53:00,742 [qtp1022308509-35515] INFO org.onap.dmaap.service.TopicRestService - Fetching all Topics "04:53:00.742 [qtp1022308509-35515] INFO org.onap.dmaap.service.TopicRestService - Fetching all Topics "2022-05-29 04:53:00,742 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.TopicServiceImpl - Fetching list of all the topics. "04:53:00.742 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.TopicServiceImpl - Fetching list of all the topics. "2022-05-29 04:53:00,742 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.utils.ConfigurationReader - Retrieving list of all the topics. "04:53:00.742 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.utils.ConfigurationReader - Retrieving list of all the topics. "2022-05-29 04:53:00,742 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.utils.ConfigurationReader - Retrieving all topics from root: /brokers/topics "04:53:00.742 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.utils.ConfigurationReader - Retrieving all topics from root: /brokers/topics "2022-05-29 04:53:00,794 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.TopicServiceImpl - Returning list of all the topics. "04:53:00.794 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.TopicServiceImpl - Returning list of all the topics. "2022-05-29 04:53:00,794 [qtp1022308509-35515] INFO org.onap.dmaap.service.TopicRestService - Returning List of all Topics "04:53:00.794 [qtp1022308509-35515] INFO org.onap.dmaap.service.TopicRestService - Returning List of all Topics "2022-05-29 04:53:00,795 [qtp1022308509-35515] DEBUG org.onap.dmaap - Body= ** "04:53:00.795 [qtp1022308509-35515] DEBUG org.onap.dmaap - Body= ** "2022-05-29 04:53:00,795 [qtp1022308509-35515] DEBUG org.onap.dmaap.service - Body= ** "04:53:00.795 [qtp1022308509-35515] DEBUG org.onap.dmaap.service - Body= ** "2022-05-29 04:53:01,414 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:01.414 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:01,420 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 5 ms for AAI-EVENT jmsgrp sdnc2 "04:53:01.420 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 5 ms for AAI-EVENT jmsgrp sdnc2 "2022-05-29 04:53:01,420 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "04:53:01.420 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "2022-05-29 04:53:01,420 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.2 empty replies/minute. "04:53:01.420 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.2 empty replies/minute. "2022-05-29 04:53:01,421 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:01.421 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:01,421 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 ""2022-05-29 04:53:01,421 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:01.421 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:01,421 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:01.421 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:01,422 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:01:422", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "04:53:01.422 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:01:422", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "2022-05-29 04:53:01,422 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:01.422 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:01,702 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:01.702 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:01,703 [qtp1022308509-35509] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching details of api key: XBArgOhNqRe3z708 "04:53:01.703 [qtp1022308509-35509] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching details of api key: XBArgOhNqRe3z708 "2022-05-29 04:53:01,707 [qtp1022308509-35509] INFO class org.onap.dmaap.dmf.mr.service.impl.ApiKeysServiceImpl - ========== ApiKeysServiceImpl: getApiKey : {"aux":{"description":"ASDC Key for sdc-AUTO1","email":""},"key":"XBArgOhNqRe3z708"}=========== "04:53:01.707 [qtp1022308509-35509] INFO class org.onap.dmaap.dmf.mr.service.impl.ApiKeysServiceImpl - ========== ApiKeysServiceImpl: getApiKey : {"aux":{"description":"ASDC Key for sdc-AUTO1","email":""},"key":"XBArgOhNqRe3z708"}=========== "2022-05-29 04:53:01,707 [qtp1022308509-35509] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching specific API key is Successful "04:53:01.707 [qtp1022308509-35509] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching specific API key is Successful "2022-05-29 04:53:02,002 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:02.002 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:02,006 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:02.006 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:02,499 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication ""2022-05-29 04:53:02,499 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:02.499 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication 04:53:02.499 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:02,502 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:02.502 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:02,502 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:02.502 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:02,906 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:02.906 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:02,915 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.71.60): authenticated lYPYer9wsEfizoiA "04:53:02.915 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.71.60): authenticated lYPYer9wsEfizoiA "2022-05-29 04:53:02,915 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] Time taken in getEvents Authorization 8 ms for SDC-DISTR-NOTIF-TOPIC-AUTO multicloud-k8s-group multicloud-k8s-id "04:53:02.915 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] Time taken in getEvents Authorization 8 ms for SDC-DISTR-NOTIF-TOPIC-AUTO multicloud-k8s-group multicloud-k8s-id "2022-05-29 04:53:02,916 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.60 "04:53:02.916 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.60 "2022-05-29 04:53:02,916 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id: 3.0 empty replies/minute. "04:53:02.916 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id: 3.0 empty replies/minute. "2022-05-29 04:53:02,916 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "04:53:02.916 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "2022-05-29 04:53:02,916 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id ""2022-05-29 04:53:02,916 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "04:53:02.916 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "2022-05-29 04:53:02,917 [qtp1022308509-35502] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:02.917 [qtp1022308509-35502] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:02,917 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:02:917", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"multicloud-k8s-group/multicloud-k8s-id", "subscriberIp":"10.233.71.60"} "04:53:02.917 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:02:917", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"multicloud-k8s-group/multicloud-k8s-id", "subscriberIp":"10.233.71.60"} "2022-05-29 04:53:02,917 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:02.917 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:03,337 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:03.337 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:03,341 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:03.341 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:03,493 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:03.493 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:03,493 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:03.493 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:03,496 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:03.496 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:03,497 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:03.497 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:05,190 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:05.190 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:05,191 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:05.191 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:05,191 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:05.191 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:05,192 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:05.192 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:05,194 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:05.194 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:05,195 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:05.195 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:05,195 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:05.195 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:05,196 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:05.196 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:05,773 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:05.773 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:05,778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 4 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "04:53:05.778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 4 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "2022-05-29 04:53:05,778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.75.89 "04:53:05.778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.75.89 "2022-05-29 04:53:05,778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 6.0 empty replies/minute. "04:53:05.778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 6.0 empty replies/minute. "2022-05-29 04:53:05,778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "04:53:05.778 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-05-29 04:53:05,778 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 ""2022-05-29 04:53:05,778 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "04:53:05.778 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-05-29 04:53:05,779 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:05.779 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:05,779 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:05:779", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.75.89"} "04:53:05.779 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:05:779", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.75.89"} "2022-05-29 04:53:05,779 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:05.779 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:06,429 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:06.429 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:06,434 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 4 ms for AAI-EVENT jmsgrp sdnc2 "04:53:06.434 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 4 ms for AAI-EVENT jmsgrp sdnc2 "2022-05-29 04:53:06,435 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "04:53:06.435 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "2022-05-29 04:53:06,435 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.4 empty replies/minute. "04:53:06.435 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.4 empty replies/minute. "2022-05-29 04:53:06,435 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:06.435 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:06,435 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 ""2022-05-29 04:53:06,435 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:06.435 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:06,436 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:06.436 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:06,436 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:436", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "04:53:06.436 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:436", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "2022-05-29 04:53:06,436 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:06.436 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:06,819 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:06.819 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:06,819 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:06.819 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:06,821 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:06.821 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:06,821 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:06.821 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:06,825 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 3320287d-778c-4eab-9fb4-2c842aeffbb8 onap-policy-pap-7ffcb875db-mvjxg "04:53:06.825 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 3320287d-778c-4eab-9fb4-2c842aeffbb8 onap-policy-pap-7ffcb875db-mvjxg "2022-05-29 04:53:06,825 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "04:53:06.825 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "2022-05-29 04:53:06,825 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg: 3.8 empty replies/minute. "04:53:06.825 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg: 3.8 empty replies/minute. "2022-05-29 04:53:06,826 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "04:53:06.826 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "2022-05-29 04:53:06,826 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg ""2022-05-29 04:53:06,826 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "04:53:06.826 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "2022-05-29 04:53:06,826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP policy-pap fe2ab631-81d2-4bfe-adf2-b39ac511e75e "04:53:06.826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP policy-pap fe2ab631-81d2-4bfe-adf2-b39ac511e75e "2022-05-29 04:53:06,826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] Time taken in getEvents Authorization 4 ms for POLICY-PDP-PAP 9c1f47a5-34b9-4b42-ba12-66c3ef66ed87 onap-policy-apex-pdp-0 "04:53:06.826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] Time taken in getEvents Authorization 4 ms for POLICY-PDP-PAP 9c1f47a5-34b9-4b42-ba12-66c3ef66ed87 onap-policy-apex-pdp-0 "2022-05-29 04:53:06,826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "04:53:06.826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "2022-05-29 04:53:06,826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e: 4.0 empty replies/minute. "04:53:06.826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e: 4.0 empty replies/minute. "2022-05-29 04:53:06,826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.72.167 "04:53:06.826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.72.167 "2022-05-29 04:53:06,826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e ""2022-05-29 04:53:06,826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0: 3.8 empty replies/minute. "04:53:06.826 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e "2022-05-29 04:53:06,826 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e ""2022-05-29 04:53:06,826 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e "04:53:06.826 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e 04:53:06.826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0: 3.8 empty replies/minute. "2022-05-29 04:53:06,826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd] Time taken in getEvents Authorization 4 ms for POLICY-ACRUNTIME-PARTICIPANT 75f4b428-dd10-4c9b-be72-f24a8941576c onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd "04:53:06.826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd] Time taken in getEvents Authorization 4 ms for POLICY-ACRUNTIME-PARTICIPANT 75f4b428-dd10-4c9b-be72-f24a8941576c onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd "2022-05-29 04:53:06,826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "04:53:06.826 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "2022-05-29 04:53:06,826 [qtp1022308509-35516] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 ""2022-05-29 04:53:06,826 [qtp1022308509-35516] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "04:53:06.826 [qtp1022308509-35516] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "2022-05-29 04:53:06,826 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:06.826 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:06,826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.186 "04:53:06.826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.186 "2022-05-29 04:53:06,826 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:826", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg", "subscriberIp":"10.233.76.128"} "04:53:06.826 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:826", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg", "subscriberIp":"10.233.76.128"} "2022-05-29 04:53:06,826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd: 3.8 empty replies/minute. "04:53:06.826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd: 3.8 empty replies/minute. "2022-05-29 04:53:06,826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd "04:53:06.826 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd "2022-05-29 04:53:06,826 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:06.826 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:06,826 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd ""2022-05-29 04:53:06,826 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd "04:53:06.826 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::75f4b428-dd10-4c9b-be72-f24a8941576c::onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd "2022-05-29 04:53:06,827 [qtp1022308509-35516] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:06,827 [qtp1022308509-35522] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:06.827 [qtp1022308509-35516] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:06.827 [qtp1022308509-35522] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:06,827 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:827", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0", "subscriberIp":"10.233.72.167"} ""2022-05-29 04:53:06,827 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:827", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e", "subscriberIp":"10.233.76.128"} "04:53:06.827 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:827", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0", "subscriberIp":"10.233.72.167"} 04:53:06.827 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:827", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e", "subscriberIp":"10.233.76.128"} "2022-05-29 04:53:06,827 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:06.827 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:06,827 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:06.827 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:06,827 [qtp1022308509-35502] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:06.827 [qtp1022308509-35502] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:06,827 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:827", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd", "subscriberIp":"10.233.68.186"} "04:53:06.827 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:06:827", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"75f4b428-dd10-4c9b-be72-f24a8941576c/onap-policy-clamp-runtime-acm-5b565d5bdd-fkzbd", "subscriberIp":"10.233.68.186"} "2022-05-29 04:53:06,828 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:06.828 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,014 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.014 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,017 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.017 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,130 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.130 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,133 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.133 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,337 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.337 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 5 ms for AAI-EVENT NBICG1 NBIC1 "04:53:07.343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 5 ms for AAI-EVENT NBICG1 NBIC1 "2022-05-29 04:53:07,343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.68.101 "04:53:07.343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.68.101 "2022-05-29 04:53:07,343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.4 empty replies/minute. "04:53:07.343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.4 empty replies/minute. "2022-05-29 04:53:07,343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "04:53:07.343 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-05-29 04:53:07,343 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 ""2022-05-29 04:53:07,343 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "04:53:07.343 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-05-29 04:53:07,344 [qtp1022308509-35521] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:07.344 [qtp1022308509-35521] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:07,344 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:344", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.68.101"} "04:53:07.344 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:344", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.68.101"} "2022-05-29 04:53:07,344 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.344 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,396 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.396 [qtp1022308509-35520] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,396 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.396 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,396 [qtp1022308509-35524] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.396 [qtp1022308509-35524] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,397 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.397 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT 99c14cf7-f742-40f5-b279-e5b31a3688bf onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 "04:53:07.401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT 99c14cf7-f742-40f5-b279-e5b31a3688bf onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 "2022-05-29 04:53:07,401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.71.244 "04:53:07.401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.71.244 "2022-05-29 04:53:07,401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8: 3.8 empty replies/minute. "04:53:07.401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8: 3.8 empty replies/minute. "2022-05-29 04:53:07,401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 "04:53:07.401 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 "2022-05-29 04:53:07,401 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 ""2022-05-29 04:53:07,401 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 "04:53:07.401 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::99c14cf7-f742-40f5-b279-e5b31a3688bf::onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8 "2022-05-29 04:53:07,402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT dac432df-77a3-4e6e-a208-52aa18fca93a onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "04:53:07.402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT dac432df-77a3-4e6e-a208-52aa18fca93a onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "2022-05-29 04:53:07,402 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 1bc56a20-db24-49e8-9129-6fb09a7a7982 onap-policy-drools-pdp-0 "04:53:07.402 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 1bc56a20-db24-49e8-9129-6fb09a7a7982 onap-policy-drools-pdp-0 "2022-05-29 04:53:07,402 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt] Time taken in getEvents Authorization 4 ms for POLICY-PDP-PAP 30a6aa9a-b9c9-4623-978b-f775079b20d3 onap-policy-xacml-pdp-859895d849-4tflt "04:53:07.402 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt] Time taken in getEvents Authorization 4 ms for POLICY-PDP-PAP 30a6aa9a-b9c9-4623-978b-f775079b20d3 onap-policy-xacml-pdp-859895d849-4tflt "2022-05-29 04:53:07,402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.75.212 "04:53:07.402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.75.212 "2022-05-29 04:53:07,402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb: 3.8 empty replies/minute. "04:53:07.402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb: 3.8 empty replies/minute. "2022-05-29 04:53:07,402 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0] fetch: timeout=15000, limit=100, filter= from Remote host 10.233.74.108 ""2022-05-29 04:53:07,402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "04:53:07.402 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0] fetch: timeout=15000, limit=100, filter= from Remote host 10.233.74.108 04:53:07.402 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "2022-05-29 04:53:07,402 [qtp1022308509-35521] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:07,402 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.67.25 "04:53:07.402 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.67.25 "2022-05-29 04:53:07,402 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0: 3.8 empty replies/minute. "04:53:07.402 [qtp1022308509-35521] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:07,402 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt: 3.8 empty replies/minute. "04:53:07.402 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0: 3.8 empty replies/minute. "2022-05-29 04:53:07,403 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:402", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8", "subscriberIp":"10.233.71.244"} "04:53:07.403 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:402", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"99c14cf7-f742-40f5-b279-e5b31a3688bf/onap-policy-clamp-ac-pf-ppnt-5c44ddc4d5-676n8", "subscriberIp":"10.233.71.244"} "2022-05-29 04:53:07,402 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "04:53:07.402 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt: 3.8 empty replies/minute. "2022-05-29 04:53:07,402 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "04:53:07.402 [qtp1022308509-35520] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::dac432df-77a3-4e6e-a208-52aa18fca93a::onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb "2022-05-29 04:53:07,403 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0 ""2022-05-29 04:53:07,403 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt "04:53:07.403 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0 "2022-05-29 04:53:07,403 [qtp1022308509-35524] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0 "04:53:07.403 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt "2022-05-29 04:53:07,403 [qtp1022308509-35524] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0 "04:53:07.403 [qtp1022308509-35524] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::1bc56a20-db24-49e8-9129-6fb09a7a7982::onap-policy-drools-pdp-0 "2022-05-29 04:53:07,403 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. ""2022-05-29 04:53:07,403 [qtp1022308509-35509] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt "04:53:07.403 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,403 [qtp1022308509-35509] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt "04:53:07.403 [qtp1022308509-35509] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::30a6aa9a-b9c9-4623-978b-f775079b20d3::onap-policy-xacml-pdp-859895d849-4tflt "2022-05-29 04:53:07,403 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:07.403 [qtp1022308509-35520] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:07,403 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:403", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb", "subscriberIp":"10.233.75.212"} "04:53:07.403 [qtp1022308509-35520] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:403", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"dac432df-77a3-4e6e-a208-52aa18fca93a/onap-policy-clamp-ac-http-ppnt-75d95cdbd4-9ddfb", "subscriberIp":"10.233.75.212"} "2022-05-29 04:53:07,403 [qtp1022308509-35524] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 66 common frames omitted 04:53:07.403 [qtp1022308509-35524] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 66 common frames omitted "2022-05-29 04:53:07,404 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:404", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0", "subscriberIp":"10.233.74.108"} "04:53:07.404 [qtp1022308509-35524] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:404", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"1bc56a20-db24-49e8-9129-6fb09a7a7982/onap-policy-drools-pdp-0", "subscriberIp":"10.233.74.108"} "2022-05-29 04:53:07,404 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.404 [qtp1022308509-35520] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,404 [qtp1022308509-35524] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.404 [qtp1022308509-35524] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,404 [qtp1022308509-35509] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:07.404 [qtp1022308509-35509] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:07,404 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:404", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt", "subscriberIp":"10.233.67.25"} "04:53:07.404 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:404", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"30a6aa9a-b9c9-4623-978b-f775079b20d3/onap-policy-xacml-pdp-859895d849-4tflt", "subscriberIp":"10.233.67.25"} "2022-05-29 04:53:07,404 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.404 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,510 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.510 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,510 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.510 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,514 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.514 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,519 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.519 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:07,756 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:07.756 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:07,768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp] Time taken in getEvents Authorization 11 ms for POLICY-ACRUNTIME-PARTICIPANT 11920468-19bb-481a-8940-24f275c5de15 onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp "04:53:07.768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp] Time taken in getEvents Authorization 11 ms for POLICY-ACRUNTIME-PARTICIPANT 11920468-19bb-481a-8940-24f275c5de15 onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp "2022-05-29 04:53:07,768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.69.146 "04:53:07.768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.69.146 "2022-05-29 04:53:07,768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp: 3.8 empty replies/minute. "04:53:07.768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp: 3.8 empty replies/minute. "2022-05-29 04:53:07,768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp "04:53:07.768 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp "2022-05-29 04:53:07,768 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp ""2022-05-29 04:53:07,768 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp "04:53:07.768 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::11920468-19bb-481a-8940-24f275c5de15::onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp "2022-05-29 04:53:07,769 [qtp1022308509-35523] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:07.769 [qtp1022308509-35523] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:07,769 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:769", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp", "subscriberIp":"10.233.69.146"} "04:53:07.769 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:07:769", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"11920468-19bb-481a-8940-24f275c5de15/onap-policy-clamp-ac-k8s-ppnt-5945f96d85-7tmsp", "subscriberIp":"10.233.69.146"} "2022-05-29 04:53:07,769 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:07.769 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:08,349 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:08.349 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:08,352 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:08.352 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:08,504 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:08.504 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:08,505 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:08.505 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:08,508 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:08.508 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:08,508 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:08.508 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:10,121 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:10.121 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:10,133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.69.160): authenticated e2VIXGKGVWNrBhEx "04:53:10.133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.69.160): authenticated e2VIXGKGVWNrBhEx "2022-05-29 04:53:10,133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/ves-openapi-manager/ves-openapi-manager] Time taken in getEvents Authorization 11 ms for SDC-DISTR-NOTIF-TOPIC-AUTO ves-openapi-manager ves-openapi-manager "04:53:10.133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/ves-openapi-manager/ves-openapi-manager] Time taken in getEvents Authorization 11 ms for SDC-DISTR-NOTIF-TOPIC-AUTO ves-openapi-manager ves-openapi-manager "2022-05-29 04:53:10,133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/ves-openapi-manager/ves-openapi-manager] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.69.160 "04:53:10.133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/ves-openapi-manager/ves-openapi-manager] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.69.160 "2022-05-29 04:53:10,133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager: 3.0 empty replies/minute. "04:53:10.133 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager: 3.0 empty replies/minute. "2022-05-29 04:53:10,134 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager "04:53:10.134 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager "2022-05-29 04:53:10,134 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager ""2022-05-29 04:53:10,134 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager "04:53:10.134 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::ves-openapi-manager::ves-openapi-manager "2022-05-29 04:53:10,134 [qtp1022308509-35522] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/ves-openapi-manager/ves-openapi-manager] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:10.134 [qtp1022308509-35522] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/ves-openapi-manager/ves-openapi-manager] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:10,134 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:10:134", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"ves-openapi-manager/ves-openapi-manager", "subscriberIp":"10.233.69.160"} "04:53:10.134 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:10:134", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"ves-openapi-manager/ves-openapi-manager", "subscriberIp":"10.233.69.160"} "2022-05-29 04:53:10,135 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:10.135 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:10,986 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:10.986 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:10,996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.77.254): authenticated WFld8ExOqXT5JksR "04:53:10.996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.77.254): authenticated WFld8ExOqXT5JksR "2022-05-29 04:53:10,996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/policy-group/policy-id] Time taken in getEvents Authorization 9 ms for SDC-DISTR-NOTIF-TOPIC-AUTO policy-group policy-id "04:53:10.996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/policy-group/policy-id] Time taken in getEvents Authorization 9 ms for SDC-DISTR-NOTIF-TOPIC-AUTO policy-group policy-id "2022-05-29 04:53:10,996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/policy-group/policy-id] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.77.254 "04:53:10.996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/policy-group/policy-id] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.77.254 "2022-05-29 04:53:10,996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id: 2.8 empty replies/minute. "04:53:10.996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id: 2.8 empty replies/minute. "2022-05-29 04:53:10,996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id "04:53:10.996 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id "2022-05-29 04:53:10,996 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id ""2022-05-29 04:53:10,996 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id "04:53:10.996 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::policy-group::policy-id "2022-05-29 04:53:10,997 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/policy-group/policy-id] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:10.997 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/policy-group/policy-id] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:10,997 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:10:997", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"policy-group/policy-id", "subscriberIp":"10.233.77.254"} "04:53:10.997 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:10:997", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"policy-group/policy-id", "subscriberIp":"10.233.77.254"} "2022-05-29 04:53:10,997 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:10.997 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:11,443 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:11.443 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:11,460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 16 ms for AAI-EVENT jmsgrp sdnc2 "04:53:11.460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 16 ms for AAI-EVENT jmsgrp sdnc2 "2022-05-29 04:53:11,460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "04:53:11.460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "2022-05-29 04:53:11,460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.6 empty replies/minute. "04:53:11.460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.6 empty replies/minute. "2022-05-29 04:53:11,460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:11.460 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:11,460 [qtp1022308509-35516] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 ""2022-05-29 04:53:11,460 [qtp1022308509-35516] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:11.460 [qtp1022308509-35516] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:11,461 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:11.461 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:11,461 [qtp1022308509-35516] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:11.461 [qtp1022308509-35516] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:11,461 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:11:461", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "04:53:11.461 [qtp1022308509-35516] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:11:461", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "2022-05-29 04:53:11,461 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:11.461 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:11,469 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.68.153): authenticated R2OxPtTkqwNlxlrt "04:53:11.469 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.68.153): authenticated R2OxPtTkqwNlxlrt "2022-05-29 04:53:11,470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] Time taken in getEvents Authorization 8 ms for SDC-DISTR-NOTIF-TOPIC-AUTO cds cds "04:53:11.470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] Time taken in getEvents Authorization 8 ms for SDC-DISTR-NOTIF-TOPIC-AUTO cds cds "2022-05-29 04:53:11,470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.68.153 "04:53:11.470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.68.153 "2022-05-29 04:53:11,470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds: 4.0 empty replies/minute. "04:53:11.470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds: 4.0 empty replies/minute. "2022-05-29 04:53:11,470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "04:53:11.470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "2022-05-29 04:53:11,470 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds ""2022-05-29 04:53:11,470 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "04:53:11.470 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "2022-05-29 04:53:11,470 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:11.470 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:11,470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:11:470", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"cds/cds", "subscriberIp":"10.233.68.153"} "04:53:11.470 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:11:470", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"cds/cds", "subscriberIp":"10.233.68.153"} "2022-05-29 04:53:11,471 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:11.471 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:12,025 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:12.025 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:12,029 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:12.029 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:12,525 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:12.525 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:12,527 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:12.527 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:12,529 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:12.529 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:12,529 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:12.529 [qtp1022308509-35502] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:13,360 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:13.360 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:13,365 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:13.365 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:13,516 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:13.516 [qtp1022308509-35516] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:13,516 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:13.516 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:13,520 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:13.520 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:13,520 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:13.520 [qtp1022308509-35516] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:14,605 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:14.605 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:14,606 [qtp1022308509-35523] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-ACRUNTIME-PARTICIPANT "04:53:14.606 [qtp1022308509-35523] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-ACRUNTIME-PARTICIPANT "2022-05-29 04:53:14,610 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-ACRUNTIME-PARTICIPANT from server 10.233.75.212 "04:53:14.610 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-ACRUNTIME-PARTICIPANT from server 10.233.75.212 "2022-05-29 04:53:14,611 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-ACRUNTIME-PARTICIPANT "04:53:14.611 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-ACRUNTIME-PARTICIPANT "2022-05-29 04:53:15,783 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:15.783 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:15,789 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 4 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "04:53:15.789 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 4 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "2022-05-29 04:53:15,790 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.75.89 "04:53:15.790 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.75.89 "2022-05-29 04:53:15,790 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 5.0 empty replies/minute. "04:53:15.790 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 5.0 empty replies/minute. "2022-05-29 04:53:15,790 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "04:53:15.790 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-05-29 04:53:15,790 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 ""2022-05-29 04:53:15,790 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "04:53:15.790 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-05-29 04:53:15,791 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:15.791 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:15,791 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:15:791", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.75.89"} "04:53:15.791 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:15:791", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.75.89"} "2022-05-29 04:53:15,791 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:15.791 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:16,469 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:16.469 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:16,475 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 4 ms for AAI-EVENT jmsgrp sdnc2 "04:53:16.475 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 4 ms for AAI-EVENT jmsgrp sdnc2 "2022-05-29 04:53:16,476 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "04:53:16.476 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "2022-05-29 04:53:16,476 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.8 empty replies/minute. "04:53:16.476 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 9.8 empty replies/minute. "2022-05-29 04:53:16,476 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:16.476 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:16,476 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 ""2022-05-29 04:53:16,476 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:16.476 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:16,477 [qtp1022308509-35523] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:16.477 [qtp1022308509-35523] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:16,477 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:16:477", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "04:53:16.477 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:16:477", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "2022-05-29 04:53:16,477 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:16.477 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:16,526 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:16.526 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:16,530 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:16.530 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:16,702 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:16.702 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:16,703 [qtp1022308509-35523] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching details of api key: XBArgOhNqRe3z708 "04:53:16.703 [qtp1022308509-35523] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching details of api key: XBArgOhNqRe3z708 "2022-05-29 04:53:16,707 [qtp1022308509-35523] INFO class org.onap.dmaap.dmf.mr.service.impl.ApiKeysServiceImpl - ========== ApiKeysServiceImpl: getApiKey : {"aux":{"description":"ASDC Key for sdc-AUTO1","email":""},"key":"XBArgOhNqRe3z708"}=========== "04:53:16.707 [qtp1022308509-35523] INFO class org.onap.dmaap.dmf.mr.service.impl.ApiKeysServiceImpl - ========== ApiKeysServiceImpl: getApiKey : {"aux":{"description":"ASDC Key for sdc-AUTO1","email":""},"key":"XBArgOhNqRe3z708"}=========== "2022-05-29 04:53:16,707 [qtp1022308509-35523] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching specific API key is Successful "04:53:16.707 [qtp1022308509-35523] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching specific API key is Successful "2022-05-29 04:53:17,005 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:17.005 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:17,014 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.76.232): authenticated lq1iQruhBbrBYrS8 "04:53:17.014 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.76.232): authenticated lq1iQruhBbrBYrS8 "2022-05-29 04:53:17,014 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/clamp/clamp] Time taken in getEvents Authorization 8 ms for SDC-DISTR-NOTIF-TOPIC-AUTO clamp clamp "04:53:17.014 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/clamp/clamp] Time taken in getEvents Authorization 8 ms for SDC-DISTR-NOTIF-TOPIC-AUTO clamp clamp "2022-05-29 04:53:17,014 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/clamp/clamp] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.76.232 "04:53:17.014 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/clamp/clamp] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.76.232 "2022-05-29 04:53:17,015 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp: 1.8 empty replies/minute. "04:53:17.015 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp: 1.8 empty replies/minute. "2022-05-29 04:53:17,015 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp "04:53:17.015 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp "2022-05-29 04:53:17,015 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp ""2022-05-29 04:53:17,015 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp "04:53:17.015 [qtp1022308509-35515] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::clamp::clamp "2022-05-29 04:53:17,016 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/clamp/clamp] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:17.016 [qtp1022308509-35515] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/clamp/clamp] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:17,016 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:17:016", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"clamp/clamp", "subscriberIp":"10.233.76.232"} "04:53:17.016 [qtp1022308509-35515] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:17:016", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"clamp/clamp", "subscriberIp":"10.233.76.232"} "2022-05-29 04:53:17,016 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:17.016 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:17,035 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:17.035 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:17,038 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:17.038 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:17,142 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:17.142 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:17,146 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:17.146 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:17,349 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:17.349 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:17,355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 5 ms for AAI-EVENT NBICG1 NBIC1 "04:53:17.355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 5 ms for AAI-EVENT NBICG1 NBIC1 "2022-05-29 04:53:17,355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.68.101 "04:53:17.355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.68.101 "2022-05-29 04:53:17,355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.6 empty replies/minute. "04:53:17.355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.6 empty replies/minute. "2022-05-29 04:53:17,355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "04:53:17.355 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-05-29 04:53:17,355 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 ""2022-05-29 04:53:17,355 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "04:53:17.355 [qtp1022308509-35522] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-05-29 04:53:17,356 [qtp1022308509-35522] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 04:53:17.356 [qtp1022308509-35522] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:17,356 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:17:356", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.68.101"} "04:53:17.356 [qtp1022308509-35522] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:17:356", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.68.101"} "2022-05-29 04:53:17,356 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:17.356 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:17,537 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:17.537 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:17,537 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:17.537 [qtp1022308509-35522] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:17,540 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:17.540 [qtp1022308509-35522] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:17,540 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:17.540 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:18,374 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:18.374 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:18,378 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:18.378 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:18,571 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:18.571 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:18,571 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:18.571 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:18,574 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:18.574 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:18,574 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:18.574 [qtp1022308509-35523] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:18,763 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:18.763 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:18,774 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.77.98): authenticated OLLNAgoSXUPdW0LR "04:53:18.774 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.security.impl.DMaaPOriginalUebAuthenticator - AUTH-LOG(10.233.77.98): authenticated OLLNAgoSXUPdW0LR "2022-05-29 04:53:18,774 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] Time taken in getEvents Authorization 9 ms for SDC-DISTR-NOTIF-TOPIC-AUTO sdc-OpenSource-Env1-sdnc-dockero sdc-COpenSource-Env11-sdnc-dockero "04:53:18.774 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] Time taken in getEvents Authorization 9 ms for SDC-DISTR-NOTIF-TOPIC-AUTO sdc-OpenSource-Env1-sdnc-dockero sdc-COpenSource-Env11-sdnc-dockero "2022-05-29 04:53:18,775 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.77.98 "04:53:18.775 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.77.98 "2022-05-29 04:53:18,775 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero: 2.0 empty replies/minute. "04:53:18.775 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero: 2.0 empty replies/minute. "2022-05-29 04:53:18,775 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "04:53:18.775 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "2022-05-29 04:53:18,775 [qtp1022308509-35509] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero ""2022-05-29 04:53:18,775 [qtp1022308509-35509] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "04:53:18.775 [qtp1022308509-35509] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "2022-05-29 04:53:18,776 [qtp1022308509-35509] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:18.776 [qtp1022308509-35509] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:18,776 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:18:776", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero", "subscriberIp":"10.233.77.98"} "04:53:18.776 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:18:776", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero", "subscriberIp":"10.233.77.98"} "2022-05-29 04:53:18,777 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:18.777 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:19,872 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:19.872 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:19,873 [qtp1022308509-35509] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-PDP-PAP "04:53:19.873 [qtp1022308509-35509] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-PDP-PAP "2022-05-29 04:53:19,878 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-PDP-PAP from server 10.233.72.167 "04:53:19.878 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-PDP-PAP from server 10.233.72.167 "2022-05-29 04:53:19,878 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-PDP-PAP "04:53:19.878 [qtp1022308509-35509] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-PDP-PAP "2022-05-29 04:53:20,199 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:20.199 [qtp1022308509-35515] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:20,200 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:20.200 [qtp1022308509-35509] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:20,200 [qtp1022308509-35525] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:20.200 [qtp1022308509-35525] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:20,201 [qtp1022308509-35526] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:20.201 [qtp1022308509-35526] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:20,202 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. ""2022-05-29 04:53:20,202 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:20.202 [qtp1022308509-35515] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. 04:53:20.202 [qtp1022308509-35509] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:20,203 [qtp1022308509-35526] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:20.203 [qtp1022308509-35526] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:20,203 [qtp1022308509-35525] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:20.203 [qtp1022308509-35525] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:21,485 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:21.485 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:21,491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 5 ms for AAI-EVENT jmsgrp sdnc2 "04:53:21.491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 5 ms for AAI-EVENT jmsgrp sdnc2 "2022-05-29 04:53:21,491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "04:53:21.491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.217 "2022-05-29 04:53:21,491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 10.0 empty replies/minute. "04:53:21.491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 10.0 empty replies/minute. "2022-05-29 04:53:21,491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:21.491 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:21,491 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 ""2022-05-29 04:53:21,491 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "04:53:21.491 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::jmsgrp::sdnc2 "2022-05-29 04:53:21,492 [qtp1022308509-35521] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 04:53:21.492 [qtp1022308509-35521] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-05-29 04:53:21,492 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:21:492", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "04:53:21.492 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"29-05-2022::04:53:21:492", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.76.217"} "2022-05-29 04:53:21,492 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "04:53:21.492 [qtp1022308509-35521] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-05-29 04:53:21,830 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:21.830 [qtp1022308509-35502] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:21,830 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:21.830 [qtp1022308509-35521] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:21,831 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:21.831 [qtp1022308509-35523] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:21,833 [qtp1022308509-35527] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "04:53:21.833 [qtp1022308509-35527] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-05-29 04:53:21,836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 3320287d-778c-4eab-9fb4-2c842aeffbb8 onap-policy-pap-7ffcb875db-mvjxg "04:53:21.836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 3320287d-778c-4eab-9fb4-2c842aeffbb8 onap-policy-pap-7ffcb875db-mvjxg "2022-05-29 04:53:21,836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP policy-pap fe2ab631-81d2-4bfe-adf2-b39ac511e75e "04:53:21.836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP policy-pap fe2ab631-81d2-4bfe-adf2-b39ac511e75e "2022-05-29 04:53:21,836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "04:53:21.836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "2022-05-29 04:53:21,836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] Time taken in getEvents Authorization 4 ms for POLICY-PDP-PAP 9c1f47a5-34b9-4b42-ba12-66c3ef66ed87 onap-policy-apex-pdp-0 "04:53:21.836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] Time taken in getEvents Authorization 4 ms for POLICY-PDP-PAP 9c1f47a5-34b9-4b42-ba12-66c3ef66ed87 onap-policy-apex-pdp-0 "2022-05-29 04:53:21,836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg: 4.0 empty replies/minute. "04:53:21.836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg: 4.0 empty replies/minute. "2022-05-29 04:53:21,836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "04:53:21.836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/fe2ab631-81d2-4bfe-adf2-b39ac511e75e] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.76.128 "2022-05-29 04:53:21,836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e: 3.4 empty replies/minute. ""2022-05-29 04:53:21,836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "04:53:21.836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e: 3.4 empty replies/minute. "2022-05-29 04:53:21,836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.72.167 "04:53:21.836 [qtp1022308509-35502] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "2022-05-29 04:53:21,836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e "04:53:21.836 [qtp1022308509-35521] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e "2022-05-29 04:53:21,836 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e ""2022-05-29 04:53:21,836 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e "04:53:21.836 [qtp1022308509-35521] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::fe2ab631-81d2-4bfe-adf2-b39ac511e75e "2022-05-29 04:53:21,836 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "04:53:21.836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.72.167 "2022-05-29 04:53:21,836 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "04:53:21.836 [qtp1022308509-35502] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::3320287d-778c-4eab-9fb4-2c842aeffbb8::onap-policy-pap-7ffcb875db-mvjxg "2022-05-29 04:53:21,836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0: 4.0 empty replies/minute. "04:53:21.836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0: 4.0 empty replies/minute. "2022-05-29 04:53:21,836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "04:53:21.836 [qtp1022308509-35523] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "2022-05-29 04:53:21,836 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 ""2022-05-29 04:53:21,836 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "04:53:21.836 [qtp1022308509-35523] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9c1f47a5-34b9-4b42-ba12-66c3ef66ed87::onap-policy-apex-pdp-0 "2022-05-29 04:53:21,837 [qtp1022308509-35502] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/3320287d-778c-4eab-9fb4-2c842aeffbb8/onap-policy-pap-7ffcb875db-mvjxg] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:69) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-05-29 04:53:21,837 [qtp1022308509-35523] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9c1f47a5-34b9-4b42-ba12-66c3ef66ed87/onap-policy-apex-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:132) at sun.reflect.GeneratedMethodAccessor131.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org