""2022-10-21 10:29:25,454 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::993be8a5-8086-4038-b0f2-b122c30b0688::onap-policy-apex-pdp-0 "10:29:25.454 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::993be8a5-8086-4038-b0f2-b122c30b0688::onap-policy-apex-pdp-0 "2022-10-21 10:29:25,455 [qtp1022308509-9974] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:25,455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] Time taken in getEvents Authorization 16 ms for POLICY-PDP-PAP 9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c onap-policy-xacml-pdp-58897cfd88-ndtgz "10:29:25.455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] Time taken in getEvents Authorization 16 ms for POLICY-PDP-PAP 9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c onap-policy-xacml-pdp-58897cfd88-ndtgz 10:29:25.455 [qtp1022308509-9974] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:25,455 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:25:455", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0", "subscriberIp":"10.233.66.12"} ""2022-10-21 10:29:25,455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.65.102 "10:29:25.455 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:25:455", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0", "subscriberIp":"10.233.66.12"} 10:29:25.455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.65.102 "2022-10-21 10:29:25,455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz: 3.4 empty replies/minute. "10:29:25.455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz: 3.4 empty replies/minute. "2022-10-21 10:29:25,455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "10:29:25.455 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "2022-10-21 10:29:25,455 [qtp1022308509-9969] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz ""2022-10-21 10:29:25,455 [qtp1022308509-9969] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "10:29:25.455 [qtp1022308509-9969] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "2022-10-21 10:29:25,455 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:25.455 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:25,455 [qtp1022308509-9969] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:25.455 [qtp1022308509-9969] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:25,456 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:25:456", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz", "subscriberIp":"10.233.65.102"} "10:29:25.456 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:25:456", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz", "subscriberIp":"10.233.65.102"} "2022-10-21 10:29:25,456 [qtp1022308509-9969] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:25.456 [qtp1022308509-9969] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:25,457 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "10:29:25.457 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "2022-10-21 10:29:25,458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 101d33bf-279e-40ce-80dd-23ce8efd46ee onap-policy-drools-pdp-0 "10:29:25.458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 101d33bf-279e-40ce-80dd-23ce8efd46ee onap-policy-drools-pdp-0 "2022-10-21 10:29:25,458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0] fetch: timeout=15000, limit=100, filter= from Remote host 10.233.72.203 "10:29:25.458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0] fetch: timeout=15000, limit=100, filter= from Remote host 10.233.72.203 "2022-10-21 10:29:25,458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0: 3.4 empty replies/minute. "10:29:25.458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0: 3.4 empty replies/minute. "2022-10-21 10:29:25,458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0 "10:29:25.458 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0 "2022-10-21 10:29:25,458 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0 ""2022-10-21 10:29:25,458 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0 "10:29:25.458 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::101d33bf-279e-40ce-80dd-23ce8efd46ee::onap-policy-drools-pdp-0 "2022-10-21 10:29:25,459 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:25.459 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:25,459 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:25:459", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0", "subscriberIp":"10.233.72.203"} "10:29:25.459 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:25:459", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"101d33bf-279e-40ce-80dd-23ce8efd46ee/onap-policy-drools-pdp-0", "subscriberIp":"10.233.72.203"} "2022-10-21 10:29:25,459 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:25.459 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:25,462 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:25.462 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:25,463 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic A1-P "10:29:25.463 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic A1-P "2022-10-21 10:29:25,465 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:25.465 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:25,465 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNC-LCM-READ "10:29:25.465 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNC-LCM-READ "2022-10-21 10:29:25,465 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:25.465 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:25,468 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:25.468 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:26,272 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.272 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,273 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic APPC-CL "10:29:26.273 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic APPC-CL "2022-10-21 10:29:26,276 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.276 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:26,392 [qtp1022308509-9969] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.392 [qtp1022308509-9969] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,392 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.392 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,393 [qtp1022308509-9969] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic APPC-LCM-WRITE "10:29:26.393 [qtp1022308509-9969] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic APPC-LCM-WRITE "2022-10-21 10:29:26,394 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.DCAE_CL_OUTPUT "10:29:26.394 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.DCAE_CL_OUTPUT "2022-10-21 10:29:26,395 [qtp1022308509-9969] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.395 [qtp1022308509-9969] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:26,397 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.397 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,397 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.397 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:26,397 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL-RSP "10:29:26.397 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL-RSP "2022-10-21 10:29:26,401 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.401 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:26,470 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.470 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,471 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "10:29:26.471 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:26,473 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "10:29:26.473 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:26,476 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT 960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7 onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r "10:29:26.476 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT 960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7 onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r "2022-10-21 10:29:26,476 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.69.21 "10:29:26.476 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.69.21 "2022-10-21 10:29:26,476 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r: 4.0 empty replies/minute. "10:29:26.476 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r: 4.0 empty replies/minute. "2022-10-21 10:29:26,477 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r "10:29:26.477 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r "2022-10-21 10:29:26,477 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r ""2022-10-21 10:29:26,477 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r "10:29:26.477 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7::onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r "2022-10-21 10:29:26,478 [qtp1022308509-9970] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:26.478 [qtp1022308509-9970] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:26,479 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:26:479", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r", "subscriberIp":"10.233.69.21"} "10:29:26.479 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:26:479", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"960b57d1-d3d0-4e07-9e96-8d5bdd2d78d7/onap-policy-clamp-ac-http-ppnt-58dd6dcc78-f994r", "subscriberIp":"10.233.69.21"} "2022-10-21 10:29:26,479 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.479 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:26,548 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.548 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,549 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.PMSH_CL_INPUT "10:29:26.549 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.PMSH_CL_INPUT "2022-10-21 10:29:26,552 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.552 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. 2022-10-21 10:29:26.553:INFO:oejshC.ROOT:qtp1022308509-9974: 2022-10-21T10:29:26.553+0000 INFO [cadi] Trans: user=dcae@dcae.onap.org[BAth],ip=10.233.66.74,ms=4.858472,validate=0.065843,code=4.772698,result=IS_AUTHENTICATED "2022-10-21 10:29:26,707 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:26.707 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:26,709 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "10:29:26.709 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:26,712 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "10:29:26.712 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:26,714 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT a613b6d0-8dcd-41de-a559-989804f2e02f onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f "10:29:26.714 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f] Time taken in getEvents Authorization 5 ms for POLICY-ACRUNTIME-PARTICIPANT a613b6d0-8dcd-41de-a559-989804f2e02f onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f "2022-10-21 10:29:26,714 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.12 "10:29:26.714 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.12 "2022-10-21 10:29:26,714 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f: 4.0 empty replies/minute. "10:29:26.714 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f: 4.0 empty replies/minute. "2022-10-21 10:29:26,715 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f "10:29:26.715 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f "2022-10-21 10:29:26,715 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f ""2022-10-21 10:29:26,715 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f "10:29:26.715 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a613b6d0-8dcd-41de-a559-989804f2e02f::onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f "2022-10-21 10:29:26,715 [qtp1022308509-9970] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:26.715 [qtp1022308509-9970] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:26,716 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:26:716", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f", "subscriberIp":"10.233.68.12"} "10:29:26.716 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:26:716", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"a613b6d0-8dcd-41de-a559-989804f2e02f/onap-policy-clamp-ac-k8s-ppnt-68675bf945-8vx8f", "subscriberIp":"10.233.68.12"} "2022-10-21 10:29:26,716 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:26.716 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,069 [qtp1022308509-9981] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.069 [qtp1022308509-9981] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication 2022-10-21 10:29:27.069:INFO:oejshC.ROOT:qtp1022308509-9981: 2022-10-21T10:29:27.069+0000 INFO [cadi] Trans: user=n/a[],ip=10.233.67.235,ms=0.061641,validate=0.058620,code=0.000000,result=TRY_AUTHENTICATING "2022-10-21 10:29:27,125 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.125 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,126 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_PNFREG_OUTPUT "10:29:27.126 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_PNFREG_OUTPUT "2022-10-21 10:29:27,130 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.VES_PNFREG_OUTPUT is equal to : unauthenticated.VES_PNFREG_OUTPUT "10:29:27.130 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.VES_PNFREG_OUTPUT is equal to : unauthenticated.VES_PNFREG_OUTPUT "2022-10-21 10:29:27,132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 5 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "10:29:27.132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 5 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "2022-10-21 10:29:27,132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.55 "10:29:27.132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.55 "2022-10-21 10:29:27,132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 5.0 empty replies/minute. "10:29:27.132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 5.0 empty replies/minute. "2022-10-21 10:29:27,132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "10:29:27.132 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-10-21 10:29:27,132 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 ""2022-10-21 10:29:27,132 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "10:29:27.132 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-10-21 10:29:27,133 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:27.133 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:27,134 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:134", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.71.55"} "10:29:27.134 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:134", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.71.55"} "2022-10-21 10:29:27,134 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.134 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,266 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.266 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,267 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic PCI-NOTIF-TOPIC-NGHBR-LIST-CHANGE-INFO "10:29:27.267 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic PCI-NOTIF-TOPIC-NGHBR-LIST-CHANGE-INFO "2022-10-21 10:29:27,270 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.270 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,271 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.271 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,271 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.271 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,272 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.SEC_FAULT_OUTPUT "10:29:27.272 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.SEC_FAULT_OUTPUT "2022-10-21 10:29:27,272 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.272 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,272 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_MEASUREMENT_OUTPUT "10:29:27.272 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_MEASUREMENT_OUTPUT "2022-10-21 10:29:27,273 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic DCAE_CL_RSP "10:29:27.273 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic DCAE_CL_RSP "2022-10-21 10:29:27,274 [qtp1022308509-9980] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.SEC_FAULT_OUTPUT is equal to : unauthenticated.SEC_FAULT_OUTPUT "10:29:27.274 [qtp1022308509-9980] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.SEC_FAULT_OUTPUT is equal to : unauthenticated.SEC_FAULT_OUTPUT "2022-10-21 10:29:27,276 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.276 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,276 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/sonhms-cg/sonhms-cid] Time taken in getEvents Authorization 4 ms for unauthenticated.SEC_FAULT_OUTPUT sonhms-cg sonhms-cid "10:29:27.276 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/sonhms-cg/sonhms-cid] Time taken in getEvents Authorization 4 ms for unauthenticated.SEC_FAULT_OUTPUT sonhms-cg sonhms-cid "2022-10-21 10:29:27,277 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.277 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,277 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/sonhms-cg/sonhms-cid] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.118 "10:29:27.277 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/sonhms-cg/sonhms-cid] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.118 "2022-10-21 10:29:27,277 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid: 3.0 empty replies/minute. "10:29:27.277 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid: 3.0 empty replies/minute. "2022-10-21 10:29:27,277 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid "10:29:27.277 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid "2022-10-21 10:29:27,277 [qtp1022308509-9980] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid ""2022-10-21 10:29:27,277 [qtp1022308509-9980] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid "10:29:27.277 [qtp1022308509-9980] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::sonhms-cg::sonhms-cid "2022-10-21 10:29:27,278 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/sonhms-cg/sonhms-cid] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:27.278 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/sonhms-cg/sonhms-cid] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:27,279 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:279", "topicId":"unauthenticated.SEC_FAULT_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"sonhms-cg/sonhms-cid", "subscriberIp":"10.233.71.118"} "10:29:27.279 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:279", "topicId":"unauthenticated.SEC_FAULT_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"sonhms-cg/sonhms-cid", "subscriberIp":"10.233.71.118"} "2022-10-21 10:29:27,279 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.279 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,518 [qtp1022308509-9969] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.518 [qtp1022308509-9969] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,519 [qtp1022308509-9969] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "10:29:27.519 [qtp1022308509-9969] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "2022-10-21 10:29:27,522 [qtp1022308509-9969] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "10:29:27.522 [qtp1022308509-9969] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "2022-10-21 10:29:27,524 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 4 ms for AAI-EVENT NBICG1 NBIC1 "10:29:27.524 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 4 ms for AAI-EVENT NBICG1 NBIC1 "2022-10-21 10:29:27,525 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.71.105 "10:29:27.525 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.71.105 "2022-10-21 10:29:27,525 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.4 empty replies/minute. "10:29:27.525 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.4 empty replies/minute. "2022-10-21 10:29:27,525 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "10:29:27.525 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-10-21 10:29:27,525 [qtp1022308509-9969] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 ""2022-10-21 10:29:27,525 [qtp1022308509-9969] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "10:29:27.525 [qtp1022308509-9969] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-10-21 10:29:27,526 [qtp1022308509-9969] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:27.526 [qtp1022308509-9969] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:27,526 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:526", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.71.105"} "10:29:27.526 [qtp1022308509-9969] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:526", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.71.105"} "2022-10-21 10:29:27,527 [qtp1022308509-9969] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.527 [qtp1022308509-9969] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,669 [qtp1022308509-9981] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.669 [qtp1022308509-9981] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,670 [qtp1022308509-9981] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_NOTIFICATION_OUTPUT "10:29:27.670 [qtp1022308509-9981] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_NOTIFICATION_OUTPUT "2022-10-21 10:29:27,672 [qtp1022308509-9981] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.VES_NOTIFICATION_OUTPUT is equal to : unauthenticated.VES_NOTIFICATION_OUTPUT "10:29:27.672 [qtp1022308509-9981] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.VES_NOTIFICATION_OUTPUT is equal to : unauthenticated.VES_NOTIFICATION_OUTPUT "2022-10-21 10:29:27,674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_NOTIFICATION_OUTPUT/sliceanalysisms-cg/sliceanalysisms-cid] Time taken in getEvents Authorization 4 ms for unauthenticated.VES_NOTIFICATION_OUTPUT sliceanalysisms-cg sliceanalysisms-cid "10:29:27.674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_NOTIFICATION_OUTPUT/sliceanalysisms-cg/sliceanalysisms-cid] Time taken in getEvents Authorization 4 ms for unauthenticated.VES_NOTIFICATION_OUTPUT sliceanalysisms-cg sliceanalysisms-cid "2022-10-21 10:29:27,674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_NOTIFICATION_OUTPUT/sliceanalysisms-cg/sliceanalysisms-cid] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.65.83 "10:29:27.674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_NOTIFICATION_OUTPUT/sliceanalysisms-cg/sliceanalysisms-cid] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.65.83 "2022-10-21 10:29:27,674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid: 3.4 empty replies/minute. "10:29:27.674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid: 3.4 empty replies/minute. "2022-10-21 10:29:27,674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid "10:29:27.674 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid "2022-10-21 10:29:27,674 [qtp1022308509-9981] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid ""2022-10-21 10:29:27,674 [qtp1022308509-9981] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid "10:29:27.674 [qtp1022308509-9981] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_NOTIFICATION_OUTPUT::sliceanalysisms-cg::sliceanalysisms-cid "2022-10-21 10:29:27,675 [qtp1022308509-9981] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_NOTIFICATION_OUTPUT/sliceanalysisms-cg/sliceanalysisms-cid] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:27.675 [qtp1022308509-9981] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_NOTIFICATION_OUTPUT/sliceanalysisms-cg/sliceanalysisms-cid] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:27,675 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:675", "topicId":"unauthenticated.VES_NOTIFICATION_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"sliceanalysisms-cg/sliceanalysisms-cid", "subscriberIp":"10.233.65.83"} "10:29:27.675 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:675", "topicId":"unauthenticated.VES_NOTIFICATION_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"sliceanalysisms-cg/sliceanalysisms-cid", "subscriberIp":"10.233.65.83"} "2022-10-21 10:29:27,675 [qtp1022308509-9981] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.675 [qtp1022308509-9981] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,755 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.755 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,756 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "10:29:27.756 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:27,761 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "10:29:27.761 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:27,765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr] Time taken in getEvents Authorization 9 ms for POLICY-ACRUNTIME-PARTICIPANT a071d3ce-2cad-4a49-9fe3-ad760a04759f onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr "10:29:27.765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr] Time taken in getEvents Authorization 9 ms for POLICY-ACRUNTIME-PARTICIPANT a071d3ce-2cad-4a49-9fe3-ad760a04759f onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr "2022-10-21 10:29:27,765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.71.99 "10:29:27.765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.71.99 "2022-10-21 10:29:27,765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr: 4.0 empty replies/minute. "10:29:27.765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr: 4.0 empty replies/minute. "2022-10-21 10:29:27,765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr "10:29:27.765 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr "2022-10-21 10:29:27,765 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr ""2022-10-21 10:29:27,765 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr "10:29:27.765 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::a071d3ce-2cad-4a49-9fe3-ad760a04759f::onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr "2022-10-21 10:29:27,766 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:27.766 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:27,766 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:766", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr", "subscriberIp":"10.233.71.99"} "10:29:27.766 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:766", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"a071d3ce-2cad-4a49-9fe3-ad760a04759f/onap-policy-clamp-ac-pf-ppnt-8d8759fc9-jr9kr", "subscriberIp":"10.233.71.99"} "2022-10-21 10:29:27,766 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.766 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:27,783 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:27.783 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:27,784 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.SEC_FAULT_OUTPUT "10:29:27.784 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.SEC_FAULT_OUTPUT "2022-10-21 10:29:27,786 [qtp1022308509-9974] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.SEC_FAULT_OUTPUT is equal to : unauthenticated.SEC_FAULT_OUTPUT "10:29:27.786 [qtp1022308509-9974] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.SEC_FAULT_OUTPUT is equal to : unauthenticated.SEC_FAULT_OUTPUT "2022-10-21 10:29:27,788 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd] Time taken in getEvents Authorization 4 ms for unauthenticated.SEC_FAULT_OUTPUT homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd "10:29:27.788 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd] Time taken in getEvents Authorization 4 ms for unauthenticated.SEC_FAULT_OUTPUT homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd "2022-10-21 10:29:27,788 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.240 "10:29:27.788 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.240 "2022-10-21 10:29:27,788 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd: 1.0 empty replies/minute. "10:29:27.788 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd: 1.0 empty replies/minute. "2022-10-21 10:29:27,789 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd "10:29:27.789 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd "2022-10-21 10:29:27,789 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd ""2022-10-21 10:29:27,789 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd "10:29:27.789 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.SEC_FAULT_OUTPUT::homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd::homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd "2022-10-21 10:29:27,789 [qtp1022308509-9974] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:27.789 [qtp1022308509-9974] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.SEC_FAULT_OUTPUT/homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:27,789 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:789", "topicId":"unauthenticated.SEC_FAULT_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd", "subscriberIp":"10.233.68.240"} "10:29:27.789 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:27:789", "topicId":"unauthenticated.SEC_FAULT_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"homlesGroupd0601ba4-a7cc-4ff6-bb01-fec45652fcdd/homlesd0601ba4-a7cc-4ff6-bb01-fec45652fcdd", "subscriberIp":"10.233.68.240"} "2022-10-21 10:29:27,789 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:27.789 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:28,075 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:28.075 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:28,076 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "10:29:28.076 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:28,078 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "10:29:28.078 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:28,080 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5] Time taken in getEvents Authorization 4 ms for POLICY-ACRUNTIME-PARTICIPANT f6944b82-7729-4167-b034-c902555807e8 onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 "10:29:28.080 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5] Time taken in getEvents Authorization 4 ms for POLICY-ACRUNTIME-PARTICIPANT f6944b82-7729-4167-b034-c902555807e8 onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 "2022-10-21 10:29:28,081 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.71.203 "10:29:28.081 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.71.203 "2022-10-21 10:29:28,081 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5: 4.0 empty replies/minute. "10:29:28.081 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5: 4.0 empty replies/minute. "2022-10-21 10:29:28,081 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 "10:29:28.081 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 "2022-10-21 10:29:28,081 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 ""2022-10-21 10:29:28,081 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 "10:29:28.081 [qtp1022308509-9970] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-ACRUNTIME-PARTICIPANT::f6944b82-7729-4167-b034-c902555807e8::onap-policy-clamp-runtime-acm-74bf594c44-7s7p5 "2022-10-21 10:29:28,082 [qtp1022308509-9970] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:28.082 [qtp1022308509-9970] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-ACRUNTIME-PARTICIPANT/f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:28,082 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:082", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5", "subscriberIp":"10.233.71.203"} "10:29:28.082 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:082", "topicId":"POLICY-ACRUNTIME-PARTICIPANT", "publisherId":"null", "publisherIp":"null", "subscriberId":"f6944b82-7729-4167-b034-c902555807e8/onap-policy-clamp-runtime-acm-74bf594c44-7s7p5", "subscriberIp":"10.233.71.203"} "2022-10-21 10:29:28,082 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:28.082 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:28,418 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:28.418 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:28,419 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:28.419 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:28,422 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:28.422 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:28,423 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] Time taken in getEvents Authorization 4 ms for SDC-DISTR-NOTIF-TOPIC-AUTO cds cds "10:29:28.423 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] Time taken in getEvents Authorization 4 ms for SDC-DISTR-NOTIF-TOPIC-AUTO cds cds "2022-10-21 10:29:28,424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.182 "10:29:28.424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.182 "2022-10-21 10:29:28,424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds: 3.4 empty replies/minute. "10:29:28.424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds: 3.4 empty replies/minute. "2022-10-21 10:29:28,424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "10:29:28.424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "2022-10-21 10:29:28,424 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds ""2022-10-21 10:29:28,424 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "10:29:28.424 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::cds::cds "2022-10-21 10:29:28,424 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:28.424 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/cds/cds] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:28,424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:424", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"cds/cds", "subscriberIp":"10.233.71.182"} "10:29:28.424 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:424", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"cds/cds", "subscriberIp":"10.233.71.182"} "2022-10-21 10:29:28,425 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:28.425 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:28,571 [qtp1022308509-9981] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:28.571 [qtp1022308509-9981] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:28,572 [qtp1022308509-9981] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:28.572 [qtp1022308509-9981] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:28,583 [qtp1022308509-9981] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:28.583 [qtp1022308509-9981] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:28,585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/aai-ml-group/aai-ml] Time taken in getEvents Authorization 13 ms for SDC-DISTR-NOTIF-TOPIC-AUTO aai-ml-group aai-ml "10:29:28.585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/aai-ml-group/aai-ml] Time taken in getEvents Authorization 13 ms for SDC-DISTR-NOTIF-TOPIC-AUTO aai-ml-group aai-ml "2022-10-21 10:29:28,585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/aai-ml-group/aai-ml] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.65.151 "10:29:28.585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/aai-ml-group/aai-ml] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.65.151 "2022-10-21 10:29:28,585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml: 2.0 empty replies/minute. "10:29:28.585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml: 2.0 empty replies/minute. "2022-10-21 10:29:28,585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml "10:29:28.585 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml "2022-10-21 10:29:28,585 [qtp1022308509-9981] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml ""2022-10-21 10:29:28,585 [qtp1022308509-9981] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml "10:29:28.585 [qtp1022308509-9981] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::aai-ml-group::aai-ml "2022-10-21 10:29:28,586 [qtp1022308509-9981] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/aai-ml-group/aai-ml] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:28.586 [qtp1022308509-9981] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/aai-ml-group/aai-ml] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:28,586 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:586", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"aai-ml-group/aai-ml", "subscriberIp":"10.233.65.151"} "10:29:28.586 [qtp1022308509-9981] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:586", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"aai-ml-group/aai-ml", "subscriberIp":"10.233.65.151"} "2022-10-21 10:29:28,586 [qtp1022308509-9981] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:28.586 [qtp1022308509-9981] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:28,594 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:28.594 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:28,595 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:28.595 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:28,595 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:28.595 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:28,595 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-STATUS-TOPIC-AUTO "10:29:28.595 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-STATUS-TOPIC-AUTO "2022-10-21 10:29:28,598 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-STATUS-TOPIC-AUTO is equal to : SDC-DISTR-STATUS-TOPIC-AUTO "10:29:28.598 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-STATUS-TOPIC-AUTO is equal to : SDC-DISTR-STATUS-TOPIC-AUTO "2022-10-21 10:29:28,599 [qtp1022308509-9980] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:28.599 [qtp1022308509-9980] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:28,599 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-STATUS-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] Time taken in getEvents Authorization 4 ms for SDC-DISTR-STATUS-TOPIC-AUTO SO-OpenSource-Env11 SO-COpenSource-Env11 "10:29:28.599 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-STATUS-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] Time taken in getEvents Authorization 4 ms for SDC-DISTR-STATUS-TOPIC-AUTO SO-OpenSource-Env11 SO-COpenSource-Env11 "2022-10-21 10:29:28,600 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-STATUS-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.72.199 "10:29:28.600 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-STATUS-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.72.199 "2022-10-21 10:29:28,600 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11: 1.0 empty replies/minute. "10:29:28.600 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11: 1.0 empty replies/minute. "2022-10-21 10:29:28,600 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "10:29:28.600 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "2022-10-21 10:29:28,600 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 ""2022-10-21 10:29:28,600 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "10:29:28.600 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-STATUS-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "2022-10-21 10:29:28,600 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-STATUS-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 62 common frames omitted 10:29:28.600 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-STATUS-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 62 common frames omitted "2022-10-21 10:29:28,601 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:601", "topicId":"SDC-DISTR-STATUS-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"SO-OpenSource-Env11/SO-COpenSource-Env11", "subscriberIp":"10.233.72.199"} "10:29:28.601 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:601", "topicId":"SDC-DISTR-STATUS-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"SO-OpenSource-Env11/SO-COpenSource-Env11", "subscriberIp":"10.233.72.199"} "2022-10-21 10:29:28,601 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] Time taken in getEvents Authorization 6 ms for SDC-DISTR-NOTIF-TOPIC-AUTO SO-OpenSource-Env11 SO-COpenSource-Env11 ""2022-10-21 10:29:28,601 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:28.601 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. 10:29:28.601 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] Time taken in getEvents Authorization 6 ms for SDC-DISTR-NOTIF-TOPIC-AUTO SO-OpenSource-Env11 SO-COpenSource-Env11 "2022-10-21 10:29:28,602 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.72.199 "10:29:28.602 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.72.199 "2022-10-21 10:29:28,602 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11: 0.8 empty replies/minute. "10:29:28.602 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11: 0.8 empty replies/minute. "2022-10-21 10:29:28,602 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "10:29:28.602 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "2022-10-21 10:29:28,602 [qtp1022308509-9980] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 ""2022-10-21 10:29:28,602 [qtp1022308509-9980] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "10:29:28.602 [qtp1022308509-9980] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::SO-OpenSource-Env11::SO-COpenSource-Env11 "2022-10-21 10:29:28,603 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:28.603 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/SO-OpenSource-Env11/SO-COpenSource-Env11] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:28,603 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:603", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"SO-OpenSource-Env11/SO-COpenSource-Env11", "subscriberIp":"10.233.72.199"} "10:29:28.603 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:28:603", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"SO-OpenSource-Env11/SO-COpenSource-Env11", "subscriberIp":"10.233.72.199"} "2022-10-21 10:29:28,603 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:28.603 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:29,675 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:29.675 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:29,677 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic RAN-Slice-Mgmt "10:29:29.677 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic RAN-Slice-Mgmt "2022-10-21 10:29:29,679 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:29.679 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:29,683 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:29.683 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:29,684 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic CM-NOTIFICATION "10:29:29.684 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic CM-NOTIFICATION "2022-10-21 10:29:29,687 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:29.687 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:30,053 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:30.053 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:30,054 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic VCPE-DHCP-EVENT "10:29:30.054 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic VCPE-DHCP-EVENT "2022-10-21 10:29:30,057 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:30.057 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:30,062 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:30.062 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:30,063 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL "10:29:30.063 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL "2022-10-21 10:29:30,065 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:30.065 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:30,346 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - ErrorResponse {"httpStatusCode":"429", "mrErrorCode":"5012", "errorMessage":"This client is making too many requests. Please use a long poll setting to decrease the number of requests that result in empty responses. ", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:30:346", "topicId":"AAI-EVENT", "publisherId":"", "publisherIp":"", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.66.172"} "10:29:30.346 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - ErrorResponse {"httpStatusCode":"429", "mrErrorCode":"5012", "errorMessage":"This client is making too many requests. Please use a long poll setting to decrease the number of requests that result in empty responses. ", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:30:346", "topicId":"AAI-EVENT", "publisherId":"", "publisherIp":"", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.66.172"} "2022-10-21 10:29:30,346 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Couldn't respond to client, closing cambria consumer AAI-EVENT jmsgrp sdnc2 503 ****** org.onap.dmaap.dmf.mr.CambriaApiException: 429 {"message":"This client is making too many requests. Please use a long poll setting to decrease the number of requests that result in empty responses. "} "10:29:30.346 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Couldn't respond to client, closing cambria consumer AAI-EVENT jmsgrp sdnc2 503 ****** org.onap.dmaap.dmf.mr.CambriaApiException: 429 {"message":"This client is making too many requests. Please use a long poll setting to decrease the number of requests that result in empty responses. "} "2022-10-21 10:29:30,347 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache - closing Kafka consumer AAI-EVENT::jmsgrp::sdnc2 object null "10:29:30.347 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache - closing Kafka consumer AAI-EVENT::jmsgrp::sdnc2 object null "2022-10-21 10:29:30,347 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Couldn't respond to client, closing cambria consumer429 {"message":"This client is making too many requests. Please use a long poll setting to decrease the number of requests that result in empty responses. "}", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:30:347", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.66.172"} "10:29:30.347 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Couldn't respond to client, closing cambria consumer429 {"message":"This client is making too many requests. Please use a long poll setting to decrease the number of requests that result in empty responses. "}", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:30:347", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"jmsgrp/sdnc2", "subscriberIp":"10.233.66.172"} "2022-10-21 10:29:30,347 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:30.347 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:30,473 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:30.473 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:30,473 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:30.473 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:30,474 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic A1-P "10:29:30.474 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic A1-P "2022-10-21 10:29:30,474 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNC-LCM-READ "10:29:30.474 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNC-LCM-READ "2022-10-21 10:29:30,476 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:30.476 [qtp1022308509-9980] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:30,477 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:30.477 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:30,856 [qtp1022308509-9977] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:30.856 [qtp1022308509-9977] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:30,858 [qtp1022308509-9977] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "10:29:30.858 [qtp1022308509-9977] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "2022-10-21 10:29:30,861 [qtp1022308509-9977] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "10:29:30.861 [qtp1022308509-9977] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "2022-10-21 10:29:30,863 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_pmsh_cg/dcae_pmsh_aai_event] Time taken in getEvents Authorization 5 ms for AAI-EVENT dcae_pmsh_cg dcae_pmsh_aai_event "10:29:30.863 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_pmsh_cg/dcae_pmsh_aai_event] Time taken in getEvents Authorization 5 ms for AAI-EVENT dcae_pmsh_cg dcae_pmsh_aai_event "2022-10-21 10:29:30,863 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_pmsh_cg/dcae_pmsh_aai_event] fetch: timeout=5000, limit=0, filter= from Remote host 10.233.66.74 "10:29:30.863 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_pmsh_cg/dcae_pmsh_aai_event] fetch: timeout=5000, limit=0, filter= from Remote host 10.233.66.74 "2022-10-21 10:29:30,863 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event: 3.0 empty replies/minute. "10:29:30.863 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event: 3.0 empty replies/minute. "2022-10-21 10:29:30,864 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event "10:29:30.864 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event "2022-10-21 10:29:30,864 [qtp1022308509-9977] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event ""2022-10-21 10:29:30,864 [qtp1022308509-9977] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event "10:29:30.864 [qtp1022308509-9977] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_pmsh_cg::dcae_pmsh_aai_event "2022-10-21 10:29:30,864 [qtp1022308509-9977] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_pmsh_cg/dcae_pmsh_aai_event] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.aaf.cadi.filter.SideChain.doFilter(SideChain.java:71) at org.onap.aaf.cadi.filter.CadiFilter.doFilter(CadiFilter.java:284) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:67) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 73 common frames omitted 10:29:30.864 [qtp1022308509-9977] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_pmsh_cg/dcae_pmsh_aai_event] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) 2022-10-21 10:29:30.865:INFO:oejshC.ROOT:qtp1022308509-9977: 2022-10-21T10:29:30.865+0000 INFO [cadi] Trans: user=dcae@dcae.onap.org[BAth],ip=10.233.66.74,ms=8.859035,validate=0.072832,code=8.769497,result=IS_AUTHENTICATED at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.aaf.cadi.filter.SideChain.doFilter(SideChain.java:71) at org.onap.aaf.cadi.filter.CadiFilter.doFilter(CadiFilter.java:284) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:67) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 73 common frames omitted "2022-10-21 10:29:30,864 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:30:864", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"dcae_pmsh_cg/dcae_pmsh_aai_event", "subscriberIp":"10.233.66.74"} "10:29:30.864 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:30:864", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"dcae_pmsh_cg/dcae_pmsh_aai_event", "subscriberIp":"10.233.66.74"} "2022-10-21 10:29:30,865 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:30.865 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:31,184 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:31.184 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:31,185 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_MEASUREMENT_OUTPUT "10:29:31.185 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_MEASUREMENT_OUTPUT "2022-10-21 10:29:31,188 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:31.188 [qtp1022308509-9970] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:32,874 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:32.874 [qtp1022308509-9970] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:32,875 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-ACRUNTIME-PARTICIPANT "10:29:32.875 [qtp1022308509-9970] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:32,878 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "10:29:32.878 [qtp1022308509-9970] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:32,880 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-ACRUNTIME-PARTICIPANT from server 10.233.68.12 "10:29:32.880 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-ACRUNTIME-PARTICIPANT from server 10.233.68.12 "2022-10-21 10:29:32,880 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-ACRUNTIME-PARTICIPANT "10:29:32.880 [qtp1022308509-9970] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:33,691 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:33.691 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:33,692 [qtp1022308509-9979] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching details of api key: JCvtKANLsabmLI3K "10:29:33.692 [qtp1022308509-9979] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching details of api key: JCvtKANLsabmLI3K "2022-10-21 10:29:33,705 [qtp1022308509-9979] INFO class org.onap.dmaap.dmf.mr.service.impl.ApiKeysServiceImpl - ========== ApiKeysServiceImpl: getApiKey : {"aux":{"description":"ASDC Key for sdc-AUTO1","email":""},"key":"JCvtKANLsabmLI3K"}=========== "10:29:33.705 [qtp1022308509-9979] INFO class org.onap.dmaap.dmf.mr.service.impl.ApiKeysServiceImpl - ========== ApiKeysServiceImpl: getApiKey : {"aux":{"description":"ASDC Key for sdc-AUTO1","email":""},"key":"JCvtKANLsabmLI3K"}=========== "2022-10-21 10:29:33,705 [qtp1022308509-9979] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching specific API key is Successful "10:29:33.705 [qtp1022308509-9979] INFO org.onap.dmaap.service.ApiKeysRestService - Fetching specific API key is Successful "2022-10-21 10:29:34,689 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:34.689 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:34,690 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic RAN-Slice-Mgmt "10:29:34.690 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic RAN-Slice-Mgmt "2022-10-21 10:29:34,693 [qtp1022308509-9977] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:34.693 [qtp1022308509-9977] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:34,694 [qtp1022308509-9977] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic CM-NOTIFICATION "10:29:34.694 [qtp1022308509-9977] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic CM-NOTIFICATION "2022-10-21 10:29:34,696 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:34.696 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:34,700 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:34.700 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:34,852 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:34.852 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:34,854 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-ACRUNTIME-PARTICIPANT "10:29:34.854 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Publishing message to topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:34,856 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "10:29:34.856 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-ACRUNTIME-PARTICIPANT is equal to : POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:34,859 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-ACRUNTIME-PARTICIPANT from server 10.233.69.21 "10:29:34.859 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Published 1 msgs in 0 ms for topic POLICY-ACRUNTIME-PARTICIPANT from server 10.233.69.21 "2022-10-21 10:29:34,859 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-ACRUNTIME-PARTICIPANT "10:29:34.859 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - Overall Response time - Published msgs in 5 ms for topic POLICY-ACRUNTIME-PARTICIPANT "2022-10-21 10:29:35,064 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:35.064 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:35,065 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic VCPE-DHCP-EVENT "10:29:35.065 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic VCPE-DHCP-EVENT "2022-10-21 10:29:35,071 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:35.071 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:35,072 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL "10:29:35.072 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL "2022-10-21 10:29:35,082 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:35.082 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:35,090 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:35.090 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:35,355 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:35.355 [qtp1022308509-9980] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:35,357 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "10:29:35.357 [qtp1022308509-9980] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "2022-10-21 10:29:35,359 [qtp1022308509-9980] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "10:29:35.359 [qtp1022308509-9980] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "2022-10-21 10:29:35,375 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 18 ms for AAI-EVENT jmsgrp sdnc2 "10:29:35.375 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] Time taken in getEvents Authorization 18 ms for AAI-EVENT jmsgrp sdnc2 "2022-10-21 10:29:35,375 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.66.172 "10:29:35.375 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/jmsgrp/sdnc2] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.66.172 "2022-10-21 10:29:35,375 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 10.4 empty replies/minute. "10:29:35.375 [qtp1022308509-9980] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 10.4 empty replies/minute. "2022-10-21 10:29:35,375 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 10.4 empty replies/minute, limit is 30.0. "10:29:35.375 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: 10.4 empty replies/minute, limit is 30.0. "2022-10-21 10:29:35,375 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: Slowing response with 6000 ms sleep, then responding in error. "10:29:35.375 [qtp1022308509-9980] WARN org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::jmsgrp::sdnc2: Slowing response with 6000 ms sleep, then responding in error. "2022-10-21 10:29:35,485 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:35.485 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:35,486 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNC-LCM-READ "10:29:35.486 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNC-LCM-READ "2022-10-21 10:29:35,487 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:35.487 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:35,488 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic A1-P "10:29:35.488 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic A1-P "2022-10-21 10:29:35,489 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:35.489 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:35,491 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:35.491 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:36,173 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:36.173 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:36,174 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:36.174 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:36,176 [qtp1022308509-9974] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:36.176 [qtp1022308509-9974] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:36,179 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] Time taken in getEvents Authorization 5 ms for SDC-DISTR-NOTIF-TOPIC-AUTO sdc-OpenSource-Env1-sdnc-dockero sdc-COpenSource-Env11-sdnc-dockero "10:29:36.179 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] Time taken in getEvents Authorization 5 ms for SDC-DISTR-NOTIF-TOPIC-AUTO sdc-OpenSource-Env1-sdnc-dockero sdc-COpenSource-Env11-sdnc-dockero "2022-10-21 10:29:36,179 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.69.6 "10:29:36.179 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.69.6 "2022-10-21 10:29:36,179 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero: 2.2 empty replies/minute. "10:29:36.179 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero: 2.2 empty replies/minute. "2022-10-21 10:29:36,180 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "10:29:36.180 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "2022-10-21 10:29:36,180 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero ""2022-10-21 10:29:36,180 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "10:29:36.180 [qtp1022308509-9974] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::sdc-OpenSource-Env1-sdnc-dockero::sdc-COpenSource-Env11-sdnc-dockero "2022-10-21 10:29:36,181 [qtp1022308509-9974] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:36.181 [qtp1022308509-9974] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:36,181 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:36:181", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero", "subscriberIp":"10.233.69.6"} "10:29:36.181 [qtp1022308509-9974] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:36:181", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"sdc-OpenSource-Env1-sdnc-dockero/sdc-COpenSource-Env11-sdnc-dockero", "subscriberIp":"10.233.69.6"} "2022-10-21 10:29:36,182 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:36.182 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:36,642 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:36.642 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:36,643 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:36.643 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:36,646 [qtp1022308509-9983] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "10:29:36.646 [qtp1022308509-9983] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: SDC-DISTR-NOTIF-TOPIC-AUTO is equal to : SDC-DISTR-NOTIF-TOPIC-AUTO "2022-10-21 10:29:36,649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] Time taken in getEvents Authorization 6 ms for SDC-DISTR-NOTIF-TOPIC-AUTO multicloud-k8s-group multicloud-k8s-id "10:29:36.649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] Time taken in getEvents Authorization 6 ms for SDC-DISTR-NOTIF-TOPIC-AUTO multicloud-k8s-group multicloud-k8s-id "2022-10-21 10:29:36,649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.66.8 "10:29:36.649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.66.8 "2022-10-21 10:29:36,649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id: 3.0 empty replies/minute. "10:29:36.649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id: 3.0 empty replies/minute. "2022-10-21 10:29:36,649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "10:29:36.649 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "2022-10-21 10:29:36,649 [qtp1022308509-9983] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id ""2022-10-21 10:29:36,649 [qtp1022308509-9983] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "10:29:36.649 [qtp1022308509-9983] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. SDC-DISTR-NOTIF-TOPIC-AUTO::multicloud-k8s-group::multicloud-k8s-id "2022-10-21 10:29:36,649 [qtp1022308509-9983] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:36.649 [qtp1022308509-9983] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [SDC-DISTR-NOTIF-TOPIC-AUTO/multicloud-k8s-group/multicloud-k8s-id] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:36,650 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:36:650", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"multicloud-k8s-group/multicloud-k8s-id", "subscriberIp":"10.233.66.8"} "10:29:36.650 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:36:650", "topicId":"SDC-DISTR-NOTIF-TOPIC-AUTO", "publisherId":"null", "publisherIp":"null", "subscriberId":"multicloud-k8s-group/multicloud-k8s-id", "subscriberIp":"10.233.66.8"} "2022-10-21 10:29:36,650 [qtp1022308509-9983] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:36.650 [qtp1022308509-9983] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. 2022-10-21 10:29:37.079:INFO:oejshC.ROOT:qtp1022308509-9979: 2022-10-21T10:29:37.079+0000 INFO [cadi] Trans: user=n/a[],ip=10.233.67.235,ms=0.050588,validate=0.047616,code=0.000000,result=TRY_AUTHENTICATING "2022-10-21 10:29:37,078 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.078 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,141 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.141 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,142 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_PNFREG_OUTPUT "10:29:37.142 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.VES_PNFREG_OUTPUT "2022-10-21 10:29:37,145 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.VES_PNFREG_OUTPUT is equal to : unauthenticated.VES_PNFREG_OUTPUT "10:29:37.145 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: unauthenticated.VES_PNFREG_OUTPUT is equal to : unauthenticated.VES_PNFREG_OUTPUT "2022-10-21 10:29:37,147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 5 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "10:29:37.147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] Time taken in getEvents Authorization 5 ms for unauthenticated.VES_PNFREG_OUTPUT OpenDCAE-c12 c12 "2022-10-21 10:29:37,147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.55 "10:29:37.147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] fetch: timeout=10000, limit=0, filter= from Remote host 10.233.71.55 "2022-10-21 10:29:37,147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 5.2 empty replies/minute. "10:29:37.147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12: 5.2 empty replies/minute. "2022-10-21 10:29:37,147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "10:29:37.147 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-10-21 10:29:37,147 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 ""2022-10-21 10:29:37,147 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "10:29:37.147 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. unauthenticated.VES_PNFREG_OUTPUT::OpenDCAE-c12::c12 "2022-10-21 10:29:37,148 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:37.148 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [unauthenticated.VES_PNFREG_OUTPUT/OpenDCAE-c12/c12] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:37,148 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:37:148", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.71.55"} "10:29:37.148 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:37:148", "topicId":"unauthenticated.VES_PNFREG_OUTPUT", "publisherId":"null", "publisherIp":"null", "subscriberId":"OpenDCAE-c12/c12", "subscriberIp":"10.233.71.55"} "2022-10-21 10:29:37,148 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:37.148 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:37,282 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.282 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,283 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.283 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,283 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.PERFORMANCE_MEASUREMENTS "10:29:37.283 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.PERFORMANCE_MEASUREMENTS "2022-10-21 10:29:37,284 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic DCAE_CL_RSP "10:29:37.284 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic DCAE_CL_RSP "2022-10-21 10:29:37,285 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.285 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,285 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.ML_RESPONSE_TOPIC "10:29:37.285 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.ML_RESPONSE_TOPIC "2022-10-21 10:29:37,286 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:37.286 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:37,287 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:37.287 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:37,289 [qtp1022308509-9983] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:37.289 [qtp1022308509-9983] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:37,532 [qtp1022308509-9977] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.532 [qtp1022308509-9977] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,534 [qtp1022308509-9977] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "10:29:37.534 [qtp1022308509-9977] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "2022-10-21 10:29:37,536 [qtp1022308509-9977] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "10:29:37.536 [qtp1022308509-9977] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "2022-10-21 10:29:37,538 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 4 ms for AAI-EVENT NBICG1 NBIC1 "10:29:37.538 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] Time taken in getEvents Authorization 4 ms for AAI-EVENT NBICG1 NBIC1 "2022-10-21 10:29:37,538 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.71.105 "10:29:37.538 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] fetch: timeout=2000, limit=0, filter= from Remote host 10.233.71.105 "2022-10-21 10:29:37,538 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.6 empty replies/minute. "10:29:37.538 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::NBICG1::NBIC1: 5.6 empty replies/minute. "2022-10-21 10:29:37,539 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "10:29:37.539 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-10-21 10:29:37,539 [qtp1022308509-9977] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 ""2022-10-21 10:29:37,539 [qtp1022308509-9977] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "10:29:37.539 [qtp1022308509-9977] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::NBICG1::NBIC1 "2022-10-21 10:29:37,540 [qtp1022308509-9977] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:37.540 [qtp1022308509-9977] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/NBICG1/NBIC1] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:37,541 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:37:541", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.71.105"} "10:29:37.541 [qtp1022308509-9977] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:37:541", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"NBICG1/NBIC1", "subscriberIp":"10.233.71.105"} "2022-10-21 10:29:37,542 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:37.542 [qtp1022308509-9977] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:37,971 [qtp1022308509-9984] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:37.971 [qtp1022308509-9984] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:37,973 [qtp1022308509-9984] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.PERFORMANCE_MEASUREMENTS "10:29:37.973 [qtp1022308509-9984] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic unauthenticated.PERFORMANCE_MEASUREMENTS "2022-10-21 10:29:37,976 [qtp1022308509-9984] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:37.976 [qtp1022308509-9984] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:38,993 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:38.993 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:38,994 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "10:29:38.994 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic AAI-EVENT "2022-10-21 10:29:38,996 [qtp1022308509-9982] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "10:29:38.996 [qtp1022308509-9982] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: AAI-EVENT is equal to : AAI-EVENT "2022-10-21 10:29:38,998 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent] Time taken in getEvents Authorization 4 ms for AAI-EVENT dcae_ccvpn_cl dcae_ccvpn_cl_aaievent "10:29:38.998 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent] Time taken in getEvents Authorization 4 ms for AAI-EVENT dcae_ccvpn_cl dcae_ccvpn_cl_aaievent "2022-10-21 10:29:38,998 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent] fetch: timeout=15000, limit=100, filter= from Remote host 10.233.65.83 "10:29:38.998 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent] fetch: timeout=15000, limit=100, filter= from Remote host 10.233.65.83 "2022-10-21 10:29:38,998 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent: 3.6 empty replies/minute. "10:29:38.998 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent: 3.6 empty replies/minute. "2022-10-21 10:29:38,999 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent "10:29:38.999 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent "2022-10-21 10:29:38,999 [qtp1022308509-9982] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent ""2022-10-21 10:29:38,999 [qtp1022308509-9982] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent "10:29:38.999 [qtp1022308509-9982] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. AAI-EVENT::dcae_ccvpn_cl::dcae_ccvpn_cl_aaievent "2022-10-21 10:29:38,999 [qtp1022308509-9982] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted 10:29:38.999 [qtp1022308509-9982] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [AAI-EVENT/dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 67 common frames omitted "2022-10-21 10:29:39,000 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:39:000", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent", "subscriberIp":"10.233.65.83"} "10:29:39.000 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:39:000", "topicId":"AAI-EVENT", "publisherId":"null", "publisherIp":"null", "subscriberId":"dcae_ccvpn_cl/dcae_ccvpn_cl_aaievent", "subscriberIp":"10.233.65.83"} "2022-10-21 10:29:39,000 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:39.000 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:39,704 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:39.704 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:39,705 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic CM-NOTIFICATION "10:29:39.705 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic CM-NOTIFICATION "2022-10-21 10:29:39,706 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:39.706 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:39,707 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic RAN-Slice-Mgmt "10:29:39.707 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic RAN-Slice-Mgmt "2022-10-21 10:29:39,709 [qtp1022308509-9983] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:39.709 [qtp1022308509-9983] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:39,711 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:39.711 [qtp1022308509-9979] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:40,089 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.089 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,091 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL "10:29:40.091 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic SDNR-CL "2022-10-21 10:29:40,094 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:40.094 [qtp1022308509-9974] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:40,096 [qtp1022308509-9984] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.096 [qtp1022308509-9984] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,097 [qtp1022308509-9984] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic VCPE-DHCP-EVENT "10:29:40.097 [qtp1022308509-9984] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic VCPE-DHCP-EVENT "2022-10-21 10:29:40,099 [qtp1022308509-9984] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:40.099 [qtp1022308509-9984] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:40,251 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.251 [qtp1022308509-9968] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,252 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "10:29:40.252 [qtp1022308509-9968] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "2022-10-21 10:29:40,254 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "10:29:40.254 [qtp1022308509-9968] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "2022-10-21 10:29:40,257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP ac983dca-a51f-439c-a658-5121fbbaece5 onap-policy-pap-68c6df564-rh7m8 ""2022-10-21 10:29:40,257 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP ac983dca-a51f-439c-a658-5121fbbaece5 onap-policy-pap-68c6df564-rh7m8 10:29:40.257 [qtp1022308509-9982] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.252 "10:29:40.257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.252 "2022-10-21 10:29:40,257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8: 3.6 empty replies/minute. "10:29:40.257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8: 3.6 empty replies/minute. "2022-10-21 10:29:40,257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8 "10:29:40.257 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8 "2022-10-21 10:29:40,257 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8 ""2022-10-21 10:29:40,257 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8 "10:29:40.257 [qtp1022308509-9968] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::ac983dca-a51f-439c-a658-5121fbbaece5::onap-policy-pap-68c6df564-rh7m8 "2022-10-21 10:29:40,257 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "10:29:40.257 [qtp1022308509-9982] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "2022-10-21 10:29:40,258 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:40.258 [qtp1022308509-9968] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:40,259 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:40:258", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8", "subscriberIp":"10.233.68.252"} "10:29:40.259 [qtp1022308509-9968] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:40:258", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"ac983dca-a51f-439c-a658-5121fbbaece5/onap-policy-pap-68c6df564-rh7m8", "subscriberIp":"10.233.68.252"} "2022-10-21 10:29:40,259 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:40.259 [qtp1022308509-9968] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:40,260 [qtp1022308509-9982] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "10:29:40.260 [qtp1022308509-9982] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "2022-10-21 10:29:40,262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP policy-pap 965a3fa0-42a7-4687-87eb-6e33ccade47b "10:29:40.262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP policy-pap 965a3fa0-42a7-4687-87eb-6e33ccade47b "2022-10-21 10:29:40,262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.252 "10:29:40.262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.68.252 "2022-10-21 10:29:40,262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b: 3.6 empty replies/minute. "10:29:40.262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b: 3.6 empty replies/minute. "2022-10-21 10:29:40,262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b "10:29:40.262 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b "2022-10-21 10:29:40,262 [qtp1022308509-9982] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b ""2022-10-21 10:29:40,262 [qtp1022308509-9982] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b "10:29:40.262 [qtp1022308509-9982] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::policy-pap::965a3fa0-42a7-4687-87eb-6e33ccade47b "2022-10-21 10:29:40,262 [qtp1022308509-9982] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted 10:29:40.262 [qtp1022308509-9982] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:762) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:680) at java.lang.Thread.run(Thread.java:748) Caused by: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache.getConsumerFor(KafkaConsumerCache.java:358) at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:145) ... 71 common frames omitted "2022-10-21 10:29:40,263 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:40:263", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b", "subscriberIp":"10.233.68.252"} "10:29:40.263 [qtp1022308509-9982] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - ErrorResponse {"httpStatusCode":"503", "mrErrorCode":"3002", "errorMessage":"Server is temporarily unavailable or busy.Try again later, or try another server in the cluster.org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable.", "helpURL":"http://onap.readthedocs.io", "statusTs":"21-10-2022::10:29:40:263", "topicId":"POLICY-PDP-PAP", "publisherId":"null", "publisherIp":"null", "subscriberId":"policy-pap/965a3fa0-42a7-4687-87eb-6e33ccade47b", "subscriberIp":"10.233.68.252"} "2022-10-21 10:29:40,263 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "10:29:40.263 [qtp1022308509-9982] INFO org.onap.dmaap.DMaaPCambriaExceptionMapper - Reached Cambria Exception Mapper.. "2022-10-21 10:29:40,460 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.460 [qtp1022308509-9983] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,461 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.461 [qtp1022308509-9979] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,462 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "10:29:40.462 [qtp1022308509-9983] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "2022-10-21 10:29:40,462 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "10:29:40.462 [qtp1022308509-9979] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "2022-10-21 10:29:40,463 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "10:29:40.463 [qtp1022308509-9974] INFO org.onap.dmaap.util.DMaaPAuthFilter - inside servlet filter Cambria Auth Headers checking before doing other Authentication "2022-10-21 10:29:40,464 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "10:29:40.464 [qtp1022308509-9974] INFO org.onap.dmaap.service.EventsRestService - Consuming message from topic POLICY-PDP-PAP "2022-10-21 10:29:40,465 [qtp1022308509-9983] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "10:29:40.465 [qtp1022308509-9983] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "2022-10-21 10:29:40,466 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "10:29:40.466 [qtp1022308509-9979] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "2022-10-21 10:29:40,467 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 993be8a5-8086-4038-b0f2-b122c30b0688 onap-policy-apex-pdp-0 "10:29:40.467 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0] Time taken in getEvents Authorization 5 ms for POLICY-PDP-PAP 993be8a5-8086-4038-b0f2-b122c30b0688 onap-policy-apex-pdp-0 "2022-10-21 10:29:40,467 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.66.12 "10:29:40.467 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/993be8a5-8086-4038-b0f2-b122c30b0688/onap-policy-apex-pdp-0] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.66.12 "2022-10-21 10:29:40,468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] Time taken in getEvents Authorization 6 ms for POLICY-PDP-PAP 9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c onap-policy-xacml-pdp-58897cfd88-ndtgz "10:29:40.468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] Time taken in getEvents Authorization 6 ms for POLICY-PDP-PAP 9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c onap-policy-xacml-pdp-58897cfd88-ndtgz "2022-10-21 10:29:40,468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.65.102 "10:29:40.468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] fetch: timeout=15000, limit=0, filter= from Remote host 10.233.65.102 "2022-10-21 10:29:40,468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz: 3.6 empty replies/minute. "10:29:40.468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz: 3.6 empty replies/minute. "2022-10-21 10:29:40,468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "10:29:40.468 [qtp1022308509-9979] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "2022-10-21 10:29:40,468 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz ""2022-10-21 10:29:40,468 [qtp1022308509-9974] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "10:29:40.468 [qtp1022308509-9974] DEBUG org.onap.dmaap.dmf.mr.beans.DMaaPKafkaMetaBroker - TOPIC_NAME: POLICY-PDP-PAP is equal to : POLICY-PDP-PAP "2022-10-21 10:29:40,468 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "10:29:40.468 [qtp1022308509-9979] ERROR org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ####@@@@## Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c::onap-policy-xacml-pdp-58897cfd88-ndtgz "2022-10-21 10:29:40,468 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::993be8a5-8086-4038-b0f2-b122c30b0688::onap-policy-apex-pdp-0: 3.6 empty replies/minute. "10:29:40.468 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPCambriaLimiter - POLICY-PDP-PAP::993be8a5-8086-4038-b0f2-b122c30b0688::onap-policy-apex-pdp-0: 3.6 empty replies/minute. "2022-10-21 10:29:40,469 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::993be8a5-8086-4038-b0f2-b122c30b0688::onap-policy-apex-pdp-0 "10:29:40.469 [qtp1022308509-9983] INFO org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory - ######@@@@### Error occured in Kafka Cachingorg.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. POLICY-PDP-PAP::993be8a5-8086-4038-b0f2-b122c30b0688::onap-policy-apex-pdp-0 "2022-10-21 10:29:40,469 [qtp1022308509-9979] WARN org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl - [POLICY-PDP-PAP/9cdcdff3-f5f5-44a2-af1a-4ba3d19c342c/onap-policy-xacml-pdp-58897cfd88-ndtgz] org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. "org.onap.dmaap.dmf.mr.backends.ConsumerFactory$UnavailableException: org.onap.dmaap.dmf.mr.backends.kafka.KafkaConsumerCache$KafkaConsumerCacheException: The cache service is unavailable. at org.onap.dmaap.dmf.mr.beans.DMaaPKafkaConsumerFactory.getConsumerFor(DMaaPKafkaConsumerFactory.java:152) at org.onap.dmaap.dmf.mr.service.impl.EventsServiceImpl.getEvents(EventsServiceImpl.java:160) at org.onap.dmaap.service.EventsRestService.getEvents(EventsRestService.java:130) at sun.reflect.GeneratedMethodAccessor129.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:179) at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192) at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:103) at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59) at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96) at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308) at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121) at org.apache.camel.component.cxf.cxfbean.CxfBeanDestination.process(CxfBeanDestination.java:83) at org.apache.camel.impl.ProcessorEndpoint.onExchange(ProcessorEndpoint.java:103) at org.apache.camel.impl.ProcessorEndpoint$1.process(ProcessorEndpoint.java:71) at org.apache.camel.util.AsyncProcessorConverterHelper$ProcessorToAsyncProcessorBridge.process(AsyncProcessorConverterHelper.java:61) at org.apache.camel.processor.SendProcessor.process(SendProcessor.java:148) at org.apache.camel.processor.RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:548) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.Pipeline.process(Pipeline.java:138) at org.apache.camel.processor.Pipeline.process(Pipeline.java:101) at org.apache.camel.processor.CamelInternalProcessor.process(CamelInternalProcessor.java:201) at org.apache.camel.processor.DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:97) at org.apache.camel.http.common.CamelServlet.doService(CamelServlet.java:208) at org.apache.camel.http.common.CamelServlet.service(CamelServlet.java:78) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at ajsc.filter.PassthruFilter.doFilter(PassthruFilter.java:26) at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347) at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at com.att.ajsc.csi.writeablerequestfilter.WriteableRequestFilter.doFilter(WriteableRequestFilter.java:41) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.onap.dmaap.util.DMaaPAuthFilter.doFilter(DMaaPAuthFilter.java:70) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1317) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1219) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:531) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:422) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:306) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:160) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYou