the only thing i was doing was taking my server down, bring it back up, taking it down, bringing it back up. i wasn't even testing failover scenarios explicitly. i was just deploy/redeploying code as part of regular development activities.

2008-09-12 06:34:06,135 ERROR [ClientCommandSenderTask Thread #93] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.security-token=1221208600729-2078898094-559394225472592985, rhq.send-throttle=true, rhq.guaranteed-delivery=true}]; params=[{invocation=NameBasedInvocation[setResourceError], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]. Cause: java.lang.Exception:java.lang.OutOfMemoryError: Java heap space -> java.lang.OutOfMemoryError:Java heap space. Cause: java.lang.Exception: java.lang.OutOfMemoryError: Java heap space
2008-09-12 06:34:06,135 ERROR [ClientCommandSenderTask Thread #95] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.security-token=1221208600729-2078898094-559394225472592985, rhq.send-throttle=true, rhq.guaranteed-delivery=true}]; params=[{invocation=NameBasedInvocation[setResourceError], targetInterfaceName=org.rhq.core.clientapi.server.discovery.DiscoveryServerService}]]. Cause: java.lang.Exception:java.lang.OutOfMemoryError: Java heap space -> java.lang.OutOfMemoryError:Java heap space. Cause: java.lang.Exception: java.lang.OutOfMemoryError: Java heap space
2008-09-12 06:34:06,135 ERROR [ClientCommandSenderTask Thread #94] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.security-token=1221208600729-2078898094-559394225472592985, rhq.guaranteed-delivery=true, rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeMeasurementReport], targetInterfaceName=org.rhq.core.clientapi.server.measurement.MeasurementServerService}]]. Cause: java.lang.Exception:java.lang.OutOfMemoryError: Java heap space -> java.lang.OutOfMemoryError:Java heap space. Cause: java.lang.Exception: java.lang.OutOfMemoryError: Java heap space
2008-09-12 06:34:06,135 ERROR [ClientCommandSenderTask Thread #94] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.security-token=1221208600729-2078898094-559394225472592985, rhq.guaranteed-delivery=true, rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeMeasurementReport], targetInterfaceName=org.rhq.core.clientapi.server.measurement.MeasurementServerService}]]. Cause: java.lang.Exception:java.lang.OutOfMemoryError: Java heap space -> java.lang.OutOfMemoryError:Java heap space. Cause: java.lang.Exception: java.lang.OutOfMemoryError: Java heap space
2008-09-12 06:34:26,291 INFO [MeasurementManager.sender-1] (rhq.core.pc.measurement.MeasurementCollectorRunner)- Measurement collection for [3] metrics took 500ms - sending report to Server...
2008-09-12 06:34:26,307 ERROR [ClientCommandSenderTask Thread #96] (enterprise.communications.command.client.ClientCommandSenderTask)- {ClientCommandSenderTask.send-failed}Failed to send command [Command: type=[remotepojo]; cmd-in-response=[false]; config=[{rhq.security-token=1221208600729-2078898094-559394225472592985, rhq.guaranteed-delivery=true, rhq.send-throttle=true}]; params=[{invocation=NameBasedInvocation[mergeMeasurementReport], targetInterfaceName=org.rhq.core.clientapi.server.measurement.MeasurementServerService}]]. Cause: java.lang.Exception:java.lang.OutOfMemoryError: Java heap space -> java.lang.OutOfMemoryError:Java heap space. Cause: java.lang.Exception: java.lang.OutOfMemoryError: Java heap space
2008-09-12 06:34:26,307 WARN [ResourceContainer.invoker.daemon-55] (jmx.connection.bean.attribute.DAttribute)- Attribute access failure Connection failure Java heap space
org.mc4j.ems.connection.EmsConnectException: Connection failure Java heap space
at org.mc4j.ems.impl.jmx.connection.support.providers.proxy.GenericMBeanServerProxy.invoke(GenericMBeanServerProxy.java:144)
at $Proxy42.getAttribute(Unknown Source)
at org.mc4j.ems.impl.jmx.connection.bean.attribute.DAttribute.refresh(DAttribute.java:199)
at org.rhq.plugins.jbossas.JBossASServerComponent.getValues(JBossASServerComponent.java:327)
at sun.reflect.GeneratedMethodAccessor58.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.rhq.core.pc.inventory.ResourceContainer$ComponentInvocationThread.call(ResourceContainer.java:446)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
Caused by: java.lang.OutOfMemoryError: Java heap space
2008-09-12 06:34:26,307 ERROR [ResourceContainer.invoker.daemon-55] (org.rhq.plugins.jbossas.JBossASServerComponent)- Failed to obtain measurement [jboss.system:type=ServerInfo:FreeMemory]
org.mc4j.ems.connection.EmsException: Could not load attribute value Connection failure Java heap space
at org.mc4j.ems.impl.jmx.connection.bean.attribute.DAttribute.refresh(DAttribute.java:235)
at org.rhq.plugins.jbossas.JBossASServerComponent.getValues(JBossASServerComponent.java:327)
at sun.reflect.GeneratedMethodAccessor58.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.rhq.core.pc.inventory.ResourceContainer$ComponentInvocationThread.call(ResourceContainer.java:446)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
Caused by: org.mc4j.ems.connection.EmsConnectException: Connection failure Java heap space
at org.mc4j.ems.impl.jmx.connection.support.providers.proxy.GenericMBeanServerProxy.invoke(GenericMBeanServerProxy.java:144)
at $Proxy42.getAttribute(Unknown Source)
at org.mc4j.ems.impl.jmx.connection.bean.attribute.DAttribute.refresh(DAttribute.java:199)
... 10 more
Caused by: java.lang.OutOfMemoryError: Java heap space