Question

JVM going down due to stuck thread (Pega 7.1.8)

Hi All,

We are seeing our production JVM going down due to the below error :-

We are under the assumption that this is exhausting the available threads thereby causing the system to go down. Any idea what might be causing the issue ?

Error logs :------------------------------------------------------------

####<Mar 7, 2016 11:18:53 AM PST> <Error> <WebLogicServer> <slcpegaapp1a> <slcpegaapp1a_unify_L1_1> <[ACTIVE] ExecuteThread: '65' for queue: 'weblogic.kernel.Default (self-tuning)'> <<WLS Kernel>> <> <> <1457378333095> <BEA-000337> <[STUCK] ExecuteThread: '59' for queue: 'weblogic.kernel.Default (self-tuning)' has been busy for "682" seconds working on the request "Workmanager: default, Version: 0, Scheduled=true, Started=true, Started time: 682087 ms

", which is more than the configured time (StuckThreadMaxTime) of "600" seconds in "server-failure-trigger". Stack trace:

                java.util.HashMap.hash(HashMap.java:366)

                java.util.HashMap.getEntry(HashMap.java:466)

                java.util.HashMap.get(HashMap.java:421)

           com.pega.pegarules.data.internal.clipboard.ClipboardPageImpl.setTopLevelPageCache(ClipboardPageImpl.java:2533)

                com.pega.pegarules.data.internal.clipboard.ClipboardPageImpl.setClassName(ClipboardPageImpl.java:863)

                com.pega.pegarules.data.internal.clipboard.ClipboardPageImpl.put(ClipboardPageImpl.java:2162)

                com.pega.pegarules.data.internal.clipboard.ClipboardPageImpl.put(ClipboardPageImpl.java:1917)

                com.pega.pegarules.data.internal.clipboard.ClipboardPageImpl.put(ClipboardPageImpl.java:1907)

                com.pega.pegarules.data.internal.clipboard.ClipboardPageBase.put(ClipboardPageBase.java:1155)

     com.pega.pegarules.data.internal.clipboard.DirectStreamContainer.expandToProperty(DirectStreamContainer.java:543)

com.pega.pegarules.data.internal.clipboard.DirectStreamContainer.expandAllToProperties(DirectStreamContainer.java:406)

                com.pega.pegarules.data.internal.clipboard.ClipboardPageImpl.expandAll(ClipboardPageImpl.java:1146)

com.pega.pegarules.session.internal.mgmt.base.handler.ReadOnlyDataPageHandler.setMiscellaneousProperties(ReadOnlyDataPageHandler.java:282)

com.pega.pegarules.session.internal.mgmt.base.handler.ReadOnlyDataPageHandler.loadDataPage(ReadOnlyDataPageHandler.java:228)

com.pega.pegarules.session.internal.mgmt.base.handler.ReadOnlyDataPageHandler.findDataPage(ReadOnlyDataPageHandler.java:430)

com.pega.pegarules.session.internal.mgmt.base.AbstractPageDirectory.findDataPageInCurrentDir(AbstractPageDirectory.java:920)

      com.pega.pegarules.session.internal.mgmt.base.AbstractPageDirectory.getDirectPage(AbstractPageDirectory.java:713)

                com.pega.pegarules.session.internal.mgmt.base.ThreadPageDir.getDirectPage(ThreadPageDir.java:473)

                com.pega.pegarules.session.internal.mgmt.base.ThreadPageDir.getPage(ThreadPageDir.java:431)

                com.pega.pegarules.session.internal.mgmt.PRThreadImpl.getPage(PRThreadImpl.java:539)

                com.pega.pegarules.session.internal.mgmt.Executable.findPage(Executable.java:2081)

                com.pega.pegarules.session.internal.mgmt.Executable.findPage(Executable.java:2034)

                com.pega.pegarules.session.internal.mgmt.Executable.findPage(Executable.java:2016)

com.pega.pegarules.session.internal.mgmt.authentication.Authentication.getSecuritySettingDeclarePage(Authentication.java:652)

com.pega.pegarules.session.internal.mgmt.authentication.Authentication.getSecuritySettingAsBoolean(Authentication.java:661)

          com.pega.pegarules.session.internal.mgmt.authentication.Authentication.doAuthentication(Authentication.java:574)

                com.pega.pegarules.integration.engine.internal.services.ServiceAPI.handleAuthentication(ServiceAPI.java:1577)

           com.pega.pegarules.session.external.engineinterface.service.EngineAPI.activityExecutionProlog(EngineAPI.java:548)

               com.pega.pegarules.session.external.engineinterface.service.EngineAPI.processRequestInner(EngineAPI.java:383)

                sun.reflect.GeneratedMethodAccessor361.invoke(Unknown Source)

                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

                java.lang.reflect.Method.invoke(Method.java:606)

com.pega.pegarules.session.internal.PRSessionProviderImpl.performTargetActionWithLock(PRSessionProviderImpl.java:1270)

    com.pega.pegarules.session.internal.PRSessionProviderImpl.doWithRequestorLocked(PRSessionProviderImpl.java:1008)

      com.pega.pegarules.session.internal.PRSessionProviderImpl.doWithRequestorLocked(PRSessionProviderImpl.java:841)

                com.pega.pegarules.session.external.engineinterface.service.EngineAPI.processRequest(EngineAPI.java:331)

                com.pega.pegarules.integration.engine.internal.services.soap.SOAPService.invoke(SOAPService.java:579)

    com.pega.pegarules.session.internal.engineinterface.etier.impl.EngineImpl._invokeEngine_privact(EngineImpl.java:315)

                com.pega.pegarules.session.internal.engineinterface.etier.impl.EngineImpl.invokeEngine(EngineImpl.java:263)

                com.pega.pegarules.session.internal.engineinterface.etier.ejb.EngineBean.invokeEngine(EngineBean.java:225)

                sun.reflect.GeneratedMethodAccessor256.invoke(Unknown Source)

                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

                java.lang.reflect.Method.invoke(Method.java:606)

                com.pega.pegarules.internal.bootstrap.PRBootstrap.invokeMethod(PRBootstrap.java:367)

                com.pega.pegarules.internal.bootstrap.PRBootstrap.invokeMethodPropagatingThrowable(PRBootstrap.java:408)

com.pega.pegarules.boot.internal.extbridge.AppServerBridgeToPega.invokeMethodPropagatingThrowable(AppServerBridgeToPega.java:223)

com.pega.pegarules.boot.internal.extbridge.AppServerBridgeToPega.invokeMethodPropagatingException(AppServerBridgeToPega.java:245)

                com.pega.pegarules.internal.etier.ejb.EngineBeanBoot.invokeEngine(EngineBeanBoot.java:168)

                com.pega.pegarules.internal.etier.ejb.EngineBMT_h449u3_ELOImpl.__WL_invoke(Unknown Source)

                weblogic.ejb.container.internal.SessionLocalMethodInvoker.invoke(SessionLocalMethodInvoker.java:33)

                com.pega.pegarules.internal.etier.ejb.EngineBMT_h449u3_ELOImpl.invokeEngine(Unknown Source)

                com.pega.pegarules.priv.context.JNDIEnvironment.invokeEngineInner(JNDIEnvironment.java:278)

                com.pega.pegarules.priv.context.JNDIEnvironment.invokeEngine(JNDIEnvironment.java:223)

                com.pega.pegarules.web.impl.WebStandardImpl.makeEtierRequest(WebStandardImpl.java:574)

                com.pega.pegarules.web.impl.WebStandardImpl.doPost(WebStandardImpl.java:374)

                sun.reflect.GeneratedMethodAccessor426.invoke(Unknown Source)

                sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

                java.lang.reflect.Method.invoke(Method.java:606)

                com.pega.pegarules.internal.bootstrap.PRBootstrap.invokeMethod(PRBootstrap.java:367)

                com.pega.pegarules.internal.bootstrap.PRBootstrap.invokeMethodPropagatingThrowable(PRBootstrap.java:408)

com.pega.pegarules.boot.internal.extbridge.AppServerBridgeToPega.invokeMethodPropagatingThrowable(AppServerBridgeToPega.java:223)

     com.pega.pegarules.boot.internal.extbridge.AppServerBridgeToPega.invokeMethod(AppServerBridgeToPega.java:272)

                com.pega.pegarules.internal.web.servlet.WebStandardBoot.doPost(WebStandardBoot.java:121)

                javax.servlet.http.HttpServlet.service(HttpServlet.java:751)

                javax.servlet.http.HttpServlet.service(HttpServlet.java:844)

                weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:280)

                weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:254)

                weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:136)

                weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:346)

                weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:243)

      weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3432)

                weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3402)

                weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)

                weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)

                weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)

                weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2285)

                weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2201)

                weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2179)

                weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1575)

weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:255)

                weblogic.work.ExecuteThread.execute(ExecuteThread.java:311)

                weblogic.work.ExecuteThread.run(ExecuteThread.java:263)

**Moderation Team has archived post**

This post has been archived for educational purposes. Contents and links will no longer be updated. If you have the same/similar question, please write a new post.

Comments

Keep up to date on this post and subscribe to comments

March 22, 2016 - 6:01pm

Please provide the complete thread dump. When you say JVM going down, you mean the system not responsive instead of sudden jvm crash by itself, right?

February 23, 2017 - 10:27am

 
"http-/0.0.0.0:8080-8" Id=367 in RUNNABLE 
BlockedCount : 480, BlockedTime : -1, WaitedCount : 523, WaitedTime : -1 
    at java.util.HashMap.getNode(HashMap.java:571) 
    at java.util.HashMap.get(HashMap.java:556) 
    at java.util.Collections$SynchronizedMap.get(Collections.java:2584) 
      - locked java.util.Collections$SynchronizedMap@19fad148 
    at com.pega.pegarules.priv.web.ProcessingValve.getToken(ProcessingValve.java:121) 
      - locked com.pega.pegarules.priv.web.ProcessingValve@9f7229 
    at com.pega.pegarules.priv.web.ProcessingValve.waitOnPendingRequests(ProcessingValve.java:89) 
    at com.pega.pegarules.session.internal.mgmt.base.handler.ReadOnlyDataPageHandler.loadDataPage(ReadOnlyDataPageHandler.java:194) 
    at com.pega.pegarules.session.internal.mgmt.base.handler.ReadOnlyDataPageHandler.findDataPage(ReadOnlyDataPageHandler.java:384) 
    at com.pega.pegarules.session.internal.mgmt.base.AbstractPageDirectory.findDataPageInCurrentDir(AbstractPageDirectory.java:920) 
    at com.pega.pegarules.session.internal.mgmt.base.AbstractPageDirectory.getDirectPage(AbstractPageDirectory.java:713) 
    at com.pega.pegarules.session.internal.mgmt.base.ThreadPageDir.getDirectPage(ThreadPageDir.java:473) 
    at com.pega.pegarules.session.internal.mgmt.base.ThreadPageDir.getPageForAutoPopulate(ThreadPageDir.java:446) 
    at com.pega.pegarules.session.internal.mgmt.PRThreadImpl.getPageForAutoPopulate(PRThreadImpl.java:578) 
    at com.pega.pegarules.data.internal.clipboard.PropertyDataPageWrapper.populateDataFromPage(PropertyDataPageWrapper.java:1168) 
    at com.pega.pegarules.data.internal.clipboard.PropertyDataPageWrapper.refresh(PropertyDataPageWrapper.java:963) 
    at com.pega.pegarules.data.internal.clipboard.ClipboardPropertyImpl.getValue(ClipboardPropertyImpl.java:4399) 
    at com.pega.pegarules.data.internal.clipboard.ClipboardPropertyBase.getValue(ClipboardPropertyBase.java:3172) 
    at com.pega.pegarules.data.internal.clipboard.ClipboardPropertyImpl.getPageValue(ClipboardPropertyImpl.java:3488) 
    at com.pega.pegarules.data.internal.clipboard.ClipboardPropertyImpl.getPageValue(ClipboardPropertyImpl.java:3457) 
    at com.pega.pegarules.data.internal.clipboard.PropertyReferenceImpl.resolveEmbeddedPages(PropertyReferenceImpl.java:1851) 
    at com.pega.pegarules.data.internal.clipboard.PropertyReferenceImpl.resolveEmbeddedPages(PropertyReferenceImpl.java:1727) 

February 23, 2017 - 10:28am

it's occurring in our application as well and it's giving 504 Gateway error. Application is not loading at all.

February 23, 2017 - 2:36pm

I got the below refer link about this issue and it may be the root cause .  

https://pdn.pega.com/support-articles/system-hang-loading-read-only-datapages-due-thread-locks.

 

Can somebody provide the background of locking mechanism to load the node level data pages ?

currently prod system is being affected due to this issue. How can we unlock the  thread locks ? is