[ERROR] ACTIVE MQ CONNECTION

active mq connection error...

 

 active mq를 사용한 통신에서 에러가 발생했다.

 

에러 로그는 다음과 같음.

[ERROR] 21-08-31 07:07:16 [impl.QueServiceImpl] - javax.jms.JMSException: Error while attempting to add new Connection to the pool
    at org.apache.activemq.jms.pool.PooledConnectionFactory.createJmsException(PooledConnectionFactory.java:247)
    at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:202)
    at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:181)
    at com.gemiso.zbckoffice.messageque.service.impl.QueServiceImpl.sendMessageExtend(QueServiceImpl.java:118)
    at com.gemiso.archv.workflwImpl.FileInfServiceImpl.getFileSetInfo(FileInfServiceImpl.java:143)
    at com.gemiso.archv.serviceImpl.ArchvWrkServiceImpl.lcMediaInfoRetryInvoke(ArchvWrkServiceImpl.java:776)
    at com.gemiso.archv.serviceImpl.ArchvWrkServiceImpl.doRetryByWorkClassCode(ArchvWrkServiceImpl.java:486)
    at com.gemiso.archv.serviceImpl.ArchvWrkServiceImpl.workRetry(ArchvWrkServiceImpl.java:430)
    at sun.reflect.GeneratedMethodAccessor1522.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:302)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:190)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
    at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:99)
    at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281)
    at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
    at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:208)
    at com.sun.proxy.$Proxy87.workRetry(Unknown Source)
    at com.gemiso.archv.controller.ArchvWrkWebController.workRetry(ArchvWrkWebController.java:252)
    at sun.reflect.GeneratedMethodAccessor1521.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:222)
    at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:137)
    at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:110)
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:814)
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:737)
    at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
    at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:959)
    at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:893)
    at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
    at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:883)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:664)
    at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:742)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.CorsFilter.doFilterInternal(CorsFilter.java:92)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:121)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:478)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:80)
    at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:650)
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:342)
    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:799)
    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66)
    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1457)
    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
    at java.lang.Thread.run(Thread.java:745)
Caused by: javax.jms.JMSException: Could not connect to broker URL: tcp://1*.***.***.***:61616?keepAlive=true&wireFormat.maxInactivityDuration=0&wireFormat.maxInactivityDurationInitalDelay=10000&soTimeout=10000&connectionTimeout=10000. Reason: java.net.ConnectException: Connection refused: connect
    at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:36)
    at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:358)
    at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:303)
    at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:243)
    at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:255)
    at org.apache.activemq.jms.pool.PooledConnectionFactory$1.makeObject(PooledConnectionFactory.java:99)
    at org.apache.activemq.jms.pool.PooledConnectionFactory$1.makeObject(PooledConnectionFactory.java:79)
    at org.apache.commons.pool.impl.GenericKeyedObjectPool.addObject(GenericKeyedObjectPool.java:1668)
    at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:200)
    ... 65 more
Caused by: java.net.ConnectException: Connection refused: connect
    at java.net.DualStackPlainSocketImpl.waitForConnect(Native Method)
    at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:85)
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    at java.net.Socket.connect(Socket.java:589)
    at org.apache.activemq.transport.tcp.TcpTransport.connect(TcpTransport.java:501)
    at org.apache.activemq.transport.tcp.TcpTransport.doStart(TcpTransport.java:464)
    at org.apache.activemq.util.ServiceSupport.start(ServiceSupport.java:55)
    at org.apache.activemq.transport.AbstractInactivityMonitor.start(AbstractInactivityMonitor.java:138)
    at org.apache.activemq.transport.TransportFilter.start(TransportFilter.java:58)
    at org.apache.activemq.transport.WireFormatNegotiator.start(WireFormatNegotiator.java:72)
    at org.apache.activemq.transport.TransportFilter.start(TransportFilter.java:58)
    at org.apache.activemq.transport.TransportFilter.start(TransportFilter.java:58)
    at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:338)
    ... 72 more








ACTOVE MQ 에러 로그


서버 재부팅이후 톰캣(재부팅 안했을 경우 ) 동일하게 커넥션이 안되는 에러가 발생했음. 


2021-08-31 07:19:53,711 | INFO  | Apache ActiveMQ 5.15.0 (localhost, ID:ArchiveJM-53760-1630361993319-0:1) is shutting down | org.apache.activemq.broker.BrokerService | WrapperSimpleAppMain
2021-08-31 07:19:53,713 | INFO  | Connector openwire stopped | org.apache.activemq.broker.TransportConnector | WrapperSimpleAppMain
2021-08-31 07:19:53,715 | INFO  | Connector amqp stopped | org.apache.activemq.broker.TransportConnector | WrapperSimpleAppMain
2021-08-31 07:19:53,716 | INFO  | Connector stomp stopped | org.apache.activemq.broker.TransportConnector | WrapperSimpleAppMain
2021-08-31 07:19:53,718 | INFO  | Connector mqtt stopped | org.apache.activemq.broker.TransportConnector | WrapperSimpleAppMain
2021-08-31 07:19:53,720 | INFO  | Connector ws stopped | org.apache.activemq.broker.TransportConnector | WrapperSimpleAppMain
2021-08-31 07:19:53,726 | INFO  | PListStore:[C:\GArchive\apache-activemq-5.15\bin\win64\..\..\data\localhost\tmp_storage] stopped | org.apache.activemq.store.kahadb.plist.PListStoreImpl | WrapperSimpleAppMain
2021-08-31 07:19:53,727 | INFO  | Stopping async queue tasks | org.apache.activemq.store.kahadb.KahaDBStore | WrapperSimpleAppMain
2021-08-31 07:19:53,728 | INFO  | Stopping async topic tasks | org.apache.activemq.store.kahadb.KahaDBStore | WrapperSimpleAppMain
2021-08-31 07:19:53,730 | INFO  | Stopped KahaDB | org.apache.activemq.store.kahadb.KahaDBStore | WrapperSimpleAppMain
2021-08-31 07:19:53,851 | INFO  | Apache ActiveMQ 5.15.0 (localhost, ID:ArchiveJM-53760-1630361993319-0:1) uptime 0.986 seconds | org.apache.activemq.broker.BrokerService | WrapperSimpleAppMain
2021-08-31 07:19:53,853 | INFO  | Apache ActiveMQ 5.15.0 (localhost, ID:ArchiveJM-53760-1630361993319-0:1) is shutdown | org.apache.activemq.broker.BrokerService | WrapperSimpleAppMain
2021-08-31 07:19:53,854 | INFO  | Closing org.apache.activemq.xbean.XBeanBrokerFactory$1@239c6542: startup date [Tue Aug 31 07:19:51 KST 2021]; root of context hierarchy | org.apache.activemq.xbean.XBeanBrokerFactory$1 | WrapperSimpleAppMain
2021-08-31 07:19:53,858 | WARN  | Exception thrown from LifecycleProcessor on context close | org.apache.activemq.xbean.XBeanBrokerFactory$1 | WrapperSimpleAppMain
java.lang.IllegalStateException: LifecycleProcessor not initialized - call 'refresh' before invoking lifecycle methods via the context: org.apache.activemq.xbean.XBeanBrokerFactory$1@239c6542: startup date [Tue Aug 31 07:19:51 KST 2021]; root of context hierarchy
    at org.springframework.context.support.AbstractApplicationContext.getLifecycleProcessor(AbstractApplicationContext.java:427)[spring-context-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.context.support.AbstractApplicationContext.doClose(AbstractApplicationContext.java:999)[spring-context-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.context.support.AbstractApplicationContext.close(AbstractApplicationContext.java:958)[spring-context-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.apache.activemq.hooks.SpringContextHook.run(SpringContextHook.java:30)[activemq-spring-5.15.0.jar:5.15.0]
    at org.apache.activemq.broker.BrokerService.stop(BrokerService.java:903)[activemq-broker-5.15.0.jar:5.15.0]
    at org.apache.activemq.xbean.XBeanBrokerService.stop(XBeanBrokerService.java:122)[activemq-spring-5.15.0.jar:5.15.0]
    at org.apache.activemq.broker.BrokerService.start(BrokerService.java:642)[activemq-broker-5.15.0.jar:5.15.0]
    at org.apache.activemq.xbean.XBeanBrokerService.afterPropertiesSet(XBeanBrokerService.java:73)[activemq-spring-5.15.0.jar:5.15.0]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.8.0_201]
    at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_201]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_201]
    at java.lang.reflect.Method.invoke(Unknown Source)[:1.8.0_201]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeCustomInitMethod(AbstractAutowireCapableBeanFactory.java:1758)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1695)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1624)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:555)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:483)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:306)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:302)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:197)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:761)[spring-beans-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:867)[spring-context-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:543)[spring-context-4.3.9.RELEASE.jar:4.3.9.RELEASE]
    at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64)[xbean-spring-4.2.jar:4.2]
    at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52)[xbean-spring-4.2.jar:4.2]
    at org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104)[activemq-spring-5.15.0.jar:5.15.0]
    at org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104)[activemq-spring-5.15.0.jar:5.15.0]
    at org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67)[activemq-spring-5.15.0.jar:5.15.0]
    at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71)[activemq-broker-5.15.0.jar:5.15.0]
    at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54)[activemq-broker-5.15.0.jar:5.15.0]
    at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87)[activemq-console-5.15.0.jar:5.15.0]
    at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63)[activemq-console-5.15.0.jar:5.15.0]
    at org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:154)[activemq-console-5.15.0.jar:5.15.0]
    at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63)[activemq-console-5.15.0.jar:5.15.0]
    at org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104)[activemq-console-5.15.0.jar:5.15.0]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.8.0_201]
    at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_201]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_201]
    at java.lang.reflect.Method.invoke(Unknown Source)[:1.8.0_201]
    at org.apache.activemq.console.Main.runTaskClass(Main.java:262)[activemq.jar:5.15.0]
    at org.apache.activemq.console.Main.main(Main.java:115)[activemq.jar:5.15.0]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.8.0_201]
    at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_201]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_201]
    at java.lang.reflect.Method.invoke(Unknown Source)[:1.8.0_201]
    at org.tanukisoftware.wrapper.WrapperSimpleApp.run(WrapperSimpleApp.java:240)[wrapper.jar:3.2.3]
    at java.lang.Thread.run(Unknown Source)[:1.8.0_201]

 

 

당시에는 프로그램에서 일단 통신을 우선해야했기 떄문에 로그 백업 후, 우선 조치

1. tomcat sever stop 

 

2. active mq server stop 

 

3. tomcat server start 

 

4. mq server start 

 

로그  지속적으로 지켜보면서 액터나ㅣ 에이전트에서 tcp 연결 안되는지 확인 필요.

 

원인은 지속적으로 connection을 맺었고, 

 

 

회사에서 쓰는 윈도우 서버에서의 소켓 누수 현상이 원인이 될 수 있음.

 

 

.https://support.microsoft.com/ko-kr/topic/kernel-sockets-leak-on-a-multiprocessor-computer-that-is-running-windows-server-2008-r2-or-windows-7-00056888-7637-2df7-e673-e60f16745e0e

 

운영서버는 Window Server 2008 R2 에 해당한다.