Some of tasks are failing due to "Closing connections idle longer than 60000 MILLISECONDS" in task log

Hello folks:

I am having this issue intermittently. The memory usage is usually around 80% and sometimes it goes up to 95% on this standalone Druid 0.13 server. A few of my tasks fail from time to time during peak hour traffic. Druid is ingesting a Kafka topic with 4 partitions. Task duration is 10 minutes and the incoming traffic can reach up to 80K messages per second. All the components including zookeeper, kafka, druid are located on the same physical server.

Sometimes one, and sometimes more than one of the ingestion tasks for my 4 partitions fail on Druid. I cannot figure out what is going wrong. I don’t want to throw in extra memory without knowing the root cause. Something I noticed in the indexing log is the following two different sets of exceptions:

2020-03-11T22:03:27,604 INFO [main] org.apache.druid.guice.JsonConfigurator - Loaded class[class org.apache.druid.query.scan.ScanQueryConfig] from props[druid.query.scan.] as [ScanQueryConfig{legacy=false}]

2020-03-11T22:03:27,606 INFO [main] org.apache.druid.guice.JsonConfigurator - Loaded class[class org.apache.druid.query.select.SelectQueryConfig] from props[druid.query.select.] as [org.apache.druid.query.select.SelectQueryConfig@14a1769d]

2020-03-11T22:03:27,607 INFO [main] org.apache.druid.guice.JsonConfigurator - Loaded class[class org.apache.druid.query.topn.TopNQueryConfig] from props[druid.query.topN.] as [org.apache.druid.query.topn.TopNQueryConfig@658e7ead]

2020-03-11T22:03:27,610 INFO [main] org.apache.druid.guice.JsonConfigurator - Loaded class[interface org.apache.druid.server.log.RequestLoggerProvider] from props[druid.request.logging.] as [org.apache.druid.server.log.NoopRequestLoggerProvider@426710f0]

2020-03-11T22:03:27,610 DEBUG [main] org.apache.druid.server.log.NoopRequestLoggerProvider - Creating NoopRequestLogger at

java.lang.Exception: Stack trace

at org.apache.druid.server.log.NoopRequestLoggerProvider.get(NoopRequestLoggerProvider.java:33) [druid-server-0.13.0-incubating.jar:0.13.0-incubating]

at org.apache.druid.server.log.NoopRequestLoggerProvider.get(NoopRequestLoggerProvider.java:26) [druid-server-0.13.0-incubating.jar:0.13.0-incubating]

at com.google.inject.internal.ProviderInternalFactory.provision(ProviderInternalFactory.java:81) [guice-4.1.0.jar:?]

at com.google.inject.internal.BoundProviderFactory.provision(BoundProviderFactory.java:72) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderInternalFactory.circularGet(ProviderInternalFactory.java:61) [guice-4.1.0.jar:?]

at com.google.inject.internal.BoundProviderFactory.get(BoundProviderFactory.java:62) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) [guice-4.1.0.jar:?]

at org.apache.druid.guice.LifecycleScope$1.get(LifecycleScope.java:68) [druid-api-0.13.0-incubating.jar:0.13.0-incubating]

at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) [guice-4.1.0.jar:?]

at com.google.inject.internal.SingleParameterInjector.inject(SingleParameterInjector.java:38) [guice-4.1.0.jar:?]

at com.google.inject.internal.SingleParameterInjector.getAll(SingleParameterInjector.java:62) [guice-4.1.0.jar:?]

at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:110) [guice-4.1.0.jar:?]

at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:90) [guice-4.1.0.jar:?]

at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:268) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) [guice-4.1.0.jar:?]

at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:194) [guice-4.1.0.jar:?]

at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) [guice-4.1.0.jar:?]

at com.google.inject.internal.SingleParameterInjector.inject(SingleParameterInjector.java:38) [guice-4.1.0.jar:?]

at com.google.inject.internal.SingleParameterInjector.getAll(SingleParameterInjector.java:62) [guice-4.1.0.jar:?]

at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:110) [guice-4.1.0.jar:?]

at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:90) [guice-4.1.0.jar:?]

at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:268) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) [guice-4.1.0.jar:?]

at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) [guice-4.1.0.jar:?]

at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:194) [guice-4.1.0.jar:?]

at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl$2$1.call(InjectorImpl.java:1019) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1085) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl$2.get(InjectorImpl.java:1015) [guice-4.1.0.jar:?]

at com.google.inject.internal.InjectorImpl.getInstance(InjectorImpl.java:1050) [guice-4.1.0.jar:?]

at org.apache.druid.guice.LifecycleModule$2.start(LifecycleModule.java:132) [druid-api-0.13.0-incubating.jar:0.13.0-incubating]

at org.apache.druid.cli.GuiceRunnable.initLifecycle(GuiceRunnable.java:107) [druid-services-0.13.0-incubating.jar:0.13.0-incubating]

at org.apache.druid.cli.CliPeon.run(CliPeon.java:348) [druid-services-0.13.0-incubating.jar:0.13.0-incubating]

at org.apache.druid.cli.Main.main(Main.java:118) [druid-services-0.13.0-incubating.jar:0.13.0-incubating]

2020-03-11T22:03:27,618 DEBUG [main] com.google.inject.internal.BytecodeGen - Loading class org.apache.druid.segment.realtime.firehose.ChatHandlerResource FastClass with sun.misc.Launcher$AppClassLoader@7852e922

2020-03-11T22:03:27,621 INFO [main] org.apache.druid.curator.CuratorModule - Starting Curator

2020-03-11T22:03:27,621 INFO [main] org.apache.curator.framework.imps.CuratorFrameworkImpl - Starting

2020-03-11T22:03:27,622 DEBUG [main] org.apache.curator.CuratorZookeeperClient - Starting

2020-03-11T22:03:27,623 DEBUG [main] org.apache.curator.ConnectionState - Starting

2020-03-11T22:03:27,623 DEBUG [main] org.apache.curator.ConnectionState - reset

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:zookeeper.version=3.4.11-37e277162d567b55a07d1755f0b31c32e93c01a0, built on 11/01/2017 18:06 GMT

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:host.name=uszyc-nfa01

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.version=1.8.0_212

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.vendor=Oracle Corporation

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.home=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.212.b04-0.el7_6.x86_64/jre

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.class.path=conf/druid/_common:conf/druid/middleManager:lib/druid-services-0.13.0-incubating.jar:lib/druid-common-0.13.0-incubating.jar:lib/java-util-0.13.0-incubating.jar:lib/config-magic-0.9.jar:lib/guice-4.1.0.jar:lib/javax.inject-1.jar:lib/aopalliance-1.0.jar:lib/guava-16.0.1.jar:lib/jackson-annotations-2.6.7.jar:lib/jackson-core-2.6.7.jar:lib/jackson-databind-2.6.7.jar:lib/opencsv-4.2.jar:lib/commons-lang3-3.2.jar:lib/commons-text-1.3.jar:lib/commons-beanutils-1.9.3.jar:lib/commons-logging-1.1.1.jar:lib/commons-collections-3.2.2.jar:lib/commons-collections4-4.1.jar:lib/joda-time-2.9.9.jar:lib/rhino-1.7R5.jar:lib/commons-compress-1.16.jar:lib/objenesis-2.6.jar:lib/xz-1.8.jar:lib/zstd-jni-1.3.3-1.jar:lib/json-path-2.3.0.jar:lib/json-smart-2.3.jar:lib/accessors-smart-1.2.jar:lib/asm-5.2.jar:lib/jackson-jq-0.0.7.jar:lib/joni-2.1.11.jar:lib/jcodings-1.0.13.jar:lib/jsr305-2.0.1.jar:lib/jna-4.5.1.jar:lib/validation-api-1.1.0.Final.jar:lib/async-http-client-2.5.3.jar:lib/async-http-client-netty-utils-2.5.3.jar:lib/netty-buffer-4.1.29.Final.jar:lib/netty-common-4.1.29.Final.jar:lib/javax.activation-1.2.0.jar:lib/netty-codec-http-4.1.29.Final.jar:lib/netty-codec-4.1.29.Final.jar:lib/netty-transport-4.1.29.Final.jar:lib/netty-resolver-4.1.29.Final.jar:lib/netty-handler-4.1.29.Final.jar:lib/netty-codec-socks-4.1.29.Final.jar:lib/netty-handler-proxy-4.1.29.Final.jar:lib/netty-transport-native-epoll-4.1.29.Final-linux-x86_64.jar:lib/netty-transport-native-unix-common-4.1.29.Final.jar:lib/netty-resolver-dns-4.1.29.Final.jar:lib/netty-codec-dns-4.1.29.Final.jar:lib/reactive-streams-1.0.2.jar:lib/netty-reactive-streams-2.0.0.jar:lib/sigar-1.6.5.132.jar:lib/jvm-attach-api-1.2.jar:lib/netty-3.10.6.Final.jar:lib/druid-api-0.13.0-incubating.jar:lib/guice-multibindings-4.1.0.jar:lib/airline-0.7.jar:lib/jackson-dataformat-smile-2.6.7.jar:lib/hibernate-validator-5.1.3.Final.jar:lib/jboss-logging-3.1.3.GA.jar:lib/classmate-1.0.0.jar:lib/commons-io-2.5.jar:lib/fastutil-8.1.0.jar:lib/commons-codec-1.7.jar:lib/commons-lang-2.6.jar:lib/commons-dbcp2-2.0.1.jar:lib/commons-pool2-2.2.jar:lib/commons-pool-1.6.jar:lib/javax.el-api-3.0.0.jar:lib/jackson-datatype-guava-2.6.7.jar:lib/jackson-datatype-joda-2.6.7.jar:lib/jdbi-2.63.1.jar:lib/log4j-api-2.5.jar:lib/log4j-core-2.5.jar:lib/log4j-slf4j-impl-2.5.jar:lib/slf4j-api-1.6.4.jar:lib/log4j-jul-2.5.jar:lib/log4j-1.2-api-2.5.jar:lib/jcl-over-slf4j-1.7.12.jar:lib/metrics-core-4.0.0.jar:lib/disruptor-3.3.6.jar:lib/antlr4-runtime-4.5.1.jar:lib/druid-server-0.13.0-incubating.jar:lib/druid-processing-0.13.0-incubating.jar:lib/druid-hll-0.13.0-incubating.jar:lib/extendedset-0.13.0-incubating.jar:lib/RoaringBitmap-0.5.18.jar:lib/compress-lzf-1.0.4.jar:lib/error_prone_annotations-2.2.0.jar:lib/icu4j-54.1.1.jar:lib/lz4-java-1.4.0.jar:lib/mapdb-1.0.8.jar:lib/asm-commons-5.2.jar:lib/asm-tree-5.2.jar:lib/druid-aws-common-0.13.0-incubating.jar:lib/aws-java-sdk-ec2-1.11.199.jar:lib/aws-java-sdk-core-1.11.199.jar:lib/httpclient-4.5.3.jar:lib/httpcore-4.4.4.jar:lib/ion-java-1.0.2.jar:lib/jackson-dataformat-cbor-2.6.7.jar:lib/jmespath-java-1.11.199.jar:lib/aws-java-sdk-s3-1.11.199.jar:lib/aws-java-sdk-kms-1.11.199.jar:lib/druid-console-0.0.4.jar:lib/commons-cli-1.2.jar:lib/javax.el-3.0.0.jar:lib/zookeeper-3.4.11.jar:lib/jline-0.9.94.jar:lib/audience-annotations-0.5.0.jar:lib/curator-framework-4.0.0.jar:lib/curator-client-4.0.0.jar:lib/curator-x-discovery-4.0.0.jar:lib/curator-recipes-4.0.0.jar:lib/jackson-mapper-asl-1.9.13.jar:lib/jackson-core-asl-1.9.13.jar:lib/jackson-jaxrs-json-provider-2.6.7.jar:lib/jackson-jaxrs-base-2.6.7.jar:lib/jackson-module-jaxb-annotations-2.6.7.jar:lib/jackson-jaxrs-smile-provider-2.6.7.jar:lib/jersey-server-1.19.3.jar:lib/jersey-core-1.19.3.jar:lib/jsr311-api-1.1.1.jar:lib/guice-servlet-4.1.0.jar:lib/jersey-guice-1.19.3.jar:lib/jersey-servlet-1.19.3.jar:lib/jetty-server-9.4.10.v20180503.jar:lib/javax.servlet-api-3.1.0.jar:lib/jetty-http-9.4.10.v20180503.jar:lib/jetty-util-9.4.10.v20180503.jar:lib/jetty-io-9.4.10.v20180503.jar:lib/jetty-proxy-9.4.10.v20180503.jar:lib/jetty-client-9.4.10.v20180503.jar:lib/tesla-aether-0.0.5.jar:lib/aether-api-0.9.0.M2.jar:lib/aether-spi-0.9.0.M2.jar:lib/aether-util-0.9.0.M2.jar:lib/aether-impl-0.9.0.M2.jar:lib/aether-connector-file-0.9.0.M2.jar:lib/aether-connector-okhttp-0.0.9.jar:lib/okhttp-1.0.2.jar:lib/wagon-provider-api-2.4.jar:lib/plexus-utils-3.0.15.jar:lib/maven-aether-provider-3.1.1.jar:lib/maven-model-3.1.1.jar:lib/maven-model-builder-3.1.1.jar:lib/plexus-interpolation-1.19.jar:lib/maven-repository-metadata-3.1.1.jar:lib/maven-settings-builder-3.1.1.jar:lib/maven-settings-3.1.1.jar:lib/spymemcached-2.12.3.jar:lib/jetty-servlet-9.4.10.v20180503.jar:lib/jetty-security-9.4.10.v20180503.jar:lib/jetty-servlets-9.4.10.v20180503.jar:lib/jetty-continuation-9.4.10.v20180503.jar:lib/derby-10.11.1.1.jar:lib/derbynet-10.11.1.1.jar:lib/derbyclient-10.11.1.1.jar:lib/commons-math3-3.6.1.jar:lib/caffeine-2.5.5.jar:lib/druid-indexing-hadoop-0.13.0-incubating.jar:lib/druid-indexing-service-0.13.0-incubating.jar:lib/druid-sql-0.13.0-incubating.jar:lib/calcite-core-1.17.0.jar:lib/avatica-core-1.10.0.jar:lib/avatica-metrics-1.10.0.jar:lib/protobuf-java-3.1.0.jar:lib/calcite-linq4j-1.17.0.jar:lib/esri-geometry-api-2.0.0.jar:lib/aggdesigner-algorithm-6.0.jar:lib/janino-2.7.6.jar:lib/commons-compiler-2.7.6.jar:lib/avatica-server-1.10.0.jar:lib/netty-all-4.1.30.Final.jar:

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.io.tmpdir=/var/tmp

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:java.compiler=

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:os.name=Linux

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:os.arch=amd64

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:os.version=3.10.0-957.21.3.el7.x86_64

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:user.name=root

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:user.home=/root

2020-03-11T22:03:27,628 INFO [main] org.apache.zookeeper.ZooKeeper - Client environment:user.dir=/usr/local/share/druid