Error on execution of many queries

Hi,

I’m having issues with the query running on Realtime.

The internal platform has increased the number of queries sent in the same time and the total queries and in some moments I receive these errors.

Currently, I’ve MaxDirectMemorySize=16g and druid.processing.numThreads=15 having 16 cores on server .

If I run the same query a few moments later everything is correct

Should I increase the amount of direct memory size?

2017-06-02 00:02:39,359 ERROR c.g.c.u.c.Futures$CombinedFuture [groupBy_buck_bidding_[2017-06-01T23:50:31.498-04:00/2017-06-02T00:00:00.000-04:00]] input future failed.

java.lang.OutOfMemoryError: Direct buffer memory

at java.nio.Bits.reserveMemory(Bits.java:658) ~[?:1.8.0_73]

at java.nio.DirectByteBuffer.(DirectByteBuffer.java:123) ~[?:1.8.0_73]

at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) ~[?:1.8.0_73]

at io.druid.offheap.OffheapBufferPool$1.get(OffheapBufferPool.java:48) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.offheap.OffheapBufferPool$1.get(OffheapBufferPool.java:38) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.collections.StupidPool.take(StupidPool.java:64) ~[druid-common-0.9.1.1.jar:0.9.1.1]

at io.druid.query.groupby.GroupByQueryEngine.process(GroupByQueryEngine.java:104) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.groupby.GroupByQueryRunnerFactory$GroupByQueryRunner.run(GroupByQueryRunnerFactory.java:110) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.QueryRunnerHelper$2.run(QueryRunnerHelper.java:94) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.client.CachingQueryRunner.run(CachingQueryRunner.java:198) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:126) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:117) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_73]

at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) ~[guava-16.0.1.jar:?]

at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) ~[?:1.8.0_73]

at com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByMergedQueryRunner$1.apply(GroupByMergedQueryRunner.java:115) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1.apply(GroupByMergedQueryRunner.java:107) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at com.google.common.collect.Iterators$8.transform(Iterators.java:794) ~[guava-16.0.1.jar:?]

at com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Iterators.addAll(Iterators.java:357) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:147) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:129) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByMergedQueryRunner.run(GroupByMergedQueryRunner.java:103) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:87) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:171) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:41) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:162) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:80) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:127) [druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:117) [druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_73]

at io.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:271) [druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_73]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_73]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_73]

2017-06-02 00:02:39,359 ERROR c.g.c.u.c.Futures$CombinedFuture [processing-6] input future failed.

java.lang.OutOfMemoryError: Direct buffer memory

at java.nio.Bits.reserveMemory(Bits.java:658) ~[?:1.8.0_73]

at java.nio.DirectByteBuffer.(DirectByteBuffer.java:123) ~[?:1.8.0_73]

at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) ~[?:1.8.0_73]

at io.druid.offheap.OffheapBufferPool$1.get(OffheapBufferPool.java:48) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.offheap.OffheapBufferPool$1.get(OffheapBufferPool.java:38) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.collections.StupidPool.take(StupidPool.java:64) ~[druid-common-0.9.1.1.jar:0.9.1.1]

at io.druid.query.groupby.GroupByQueryEngine.process(GroupByQueryEngine.java:104) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.groupby.GroupByQueryRunnerFactory$GroupByQueryRunner.run(GroupByQueryRunnerFactory.java:110) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.QueryRunnerHelper$2.run(QueryRunnerHelper.java:94) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.client.CachingQueryRunner.run(CachingQueryRunner.java:198) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:126) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:117) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_73]

at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) ~[guava-16.0.1.jar:?]

at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) ~[?:1.8.0_73]

at com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByMergedQueryRunner$1.apply(GroupByMergedQueryRunner.java:115) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1.apply(GroupByMergedQueryRunner.java:107) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at com.google.common.collect.Iterators$8.transform(Iterators.java:794) ~[guava-16.0.1.jar:?]

at com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Iterators.addAll(Iterators.java:357) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:147) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:129) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByMergedQueryRunner.run(GroupByMergedQueryRunner.java:103) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:87) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:171) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:41) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:162) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:80) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:127) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:117) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_73]

at io.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:271) [druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_73]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_73] at java.lang.Thread.run(Thread.java:745) [?:1.8.0_73]

2017-06-02 00:02:46,513 INFO c.m.h.c.p.ChannelResourceFactory [coordinator_handoff_scheduled_0] Generating: http://10.80.4.69:8082
2017-06-02 00:02:52,770 ERROR c.g.c.u.c.Futures$CombinedFuture [qtp1490466745-155[groupBy_buck_bidding_51b568fd-4748-11e7-a68a-000c29edcf08]] input future failed.

java.lang.OutOfMemoryError: Direct buffer memory

at java.nio.Bits.reserveMemory(Bits.java:658) ~[?:1.8.0_73]

at java.nio.DirectByteBuffer.(DirectByteBuffer.java:123) ~[?:1.8.0_73]

at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311) ~[?:1.8.0_73]

at io.druid.offheap.OffheapBufferPool$1.get(OffheapBufferPool.java:48) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.offheap.OffheapBufferPool$1.get(OffheapBufferPool.java:38) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.collections.StupidPool.take(StupidPool.java:64) ~[druid-common-0.9.1.1.jar:0.9.1.1]

at io.druid.query.groupby.GroupByQueryEngine.process(GroupByQueryEngine.java:104) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.groupby.GroupByQueryRunnerFactory$GroupByQueryRunner.run(GroupByQueryRunnerFactory.java:110) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.QueryRunnerHelper$2.run(QueryRunnerHelper.java:94) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.client.CachingQueryRunner.run(CachingQueryRunner.java:198) ~[druid-server-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:126) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:117) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_73]

at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297) ~[guava-16.0.1.jar:?]

at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134) ~[?:1.8.0_73]

at com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByMergedQueryRunner$1.apply(GroupByMergedQueryRunner.java:115) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1.apply(GroupByMergedQueryRunner.java:107) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at com.google.common.collect.Iterators$8.transform(Iterators.java:794) ~[guava-16.0.1.jar:?]

at com.google.common.collect.TransformedIterator.next(TransformedIterator.java:48) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Iterators.addAll(Iterators.java:357) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:147) ~[guava-16.0.1.jar:?]

at com.google.common.collect.Lists.newArrayList(Lists.java:129) ~[guava-16.0.1.jar:?]

at io.druid.query.GroupByMergedQueryRunner.run(GroupByMergedQueryRunner.java:103) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.MetricsEmittingQueryRunner$1.accumulate(MetricsEmittingQueryRunner.java:118) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2$1.call(SpecificSegmentQueryRunner.java:87) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner.doNamed(SpecificSegmentQueryRunner.java:171) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner.access$400(SpecificSegmentQueryRunner.java:41) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.doItNamed(SpecificSegmentQueryRunner.java:162) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.spec.SpecificSegmentQueryRunner$2.accumulate(SpecificSegmentQueryRunner.java:80) ~[druid-processing-0.9.1.1.jar:0.9.1.1]
at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:127) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at io.druid.query.GroupByMergedQueryRunner$1$1.call(GroupByMergedQueryRunner.java:117) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_73]

at io.druid.query.PrioritizedListenableFutureTask.run(PrioritizedExecutorService.java:271) ~[druid-processing-0.9.1.1.jar:0.9.1.1]

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[?:1.8.0_73]

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[?:1.8.0_73]

at java.lang.Thread.run(Thread.java:745) [?:1.8.0_73]

2017-06-02 00:02:52,770 ERROR c.s.j.s.c.ContainerResponse [qtp1490466745-155] The exception contained within MappableContainerException could not be mapped to a response, re-throwing to the HTTP container

``

Thanks,

Maurizio