Unable to create new native thread

Hi,

During hadoop batch ingestion, some of the tasks are failing with the following exceptions:

[WARN ] 2017-03-06 12:32:10.216 [main] AbstractLifeCycle - FAILED qtp965138756{FAILED,66<=13<=66,i=12,q=0}: java.lang.OutOfMemoryError: unable to create new native thread

java.lang.OutOfMemoryError: unable to create new native thread

    at java.lang.Thread.start0(Native Method) ~[?:1.7.0_79]

    at java.lang.Thread.start(Thread.java:714) ~[?:1.7.0_79]

    at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:430) ~[jetty-util-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.util.thread.QueuedThreadPool.doStart(QueuedThreadPool.java:104) ~[jetty-util-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) [jetty-util-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132) [jetty-util-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.server.Server.start(Server.java:387) [jetty-server-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:114) [jetty-util-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61) [jetty-server-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.server.Server.doStart(Server.java:354) [jetty-server-9.2.5.v20141112.jar:9.2.5.v20141112]

    at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68) [jetty-util-9.2.5.v20141112.jar:9.2.5.v20141112]

    at io.druid.server.initialization.jetty.JettyServerModule$1.start(JettyServerModule.java:204) [druid-server-0.9.2.jar:0.9.2]

    at com.metamx.common.lifecycle.Lifecycle.start(Lifecycle.java:259) [java-util-0.27.10.jar:?]

    at io.druid.guice.LifecycleModule$2.start(LifecycleModule.java:155) [druid-api-0.9.2.jar:0.9.2]

    at io.druid.cli.GuiceRunnable.initLifecycle(GuiceRunnable.java:101) [druid-services-0.9.2.jar:0.9.2]

    at io.druid.cli.CliPeon.run(CliPeon.java:274) [druid-services-0.9.2.jar:0.9.2]

    at io.druid.cli.Main.main(Main.java:106) [druid-services-0.9.2.jar:0.9.2]

In some instance, the following logs are also being written:

[ERROR] 2017-03-06 02:00:49.813 [main] CliPeon - Error when starting up. Failing.

java.lang.OutOfMemoryError: unable to create new native thread

    at java.lang.Thread.start0(Native Method) ~[?:1.7.0_79]

    at java.lang.Thread.start(Thread.java:714) ~[?:1.7.0_79]

    at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:949) ~[?:1.7.0_79]

    at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1371) ~[?:1.7.0_79]

    at org.jboss.netty.util.internal.DeadLockProofWorker.start(DeadLockProofWorker.java:38) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.AbstractNioSelector.openSelector(AbstractNioSelector.java:368) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.AbstractNioSelector.<init>(AbstractNioSelector.java:100) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.AbstractNioWorker.<init>(AbstractNioWorker.java:52) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.NioWorker.<init>(NioWorker.java:45) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:44) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.NioWorkerPool.newWorker(NioWorkerPool.java:28) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.AbstractNioWorkerPool.init(AbstractNioWorkerPool.java:80) ~[netty-3.10.4.Final.jar:?]

    at org.jboss.netty.channel.socket.nio.NioWorkerPool.<init>(NioWorkerPool.java:39) ~[netty-3.10.4.Final.jar:?]

    at com.metamx.http.client.HttpClientInit.createBootstrap(HttpClientInit.java:133) ~[http-client-1.0.4.jar:?]

    at com.metamx.http.client.HttpClientInit.createClient(HttpClientInit.java:85) ~[http-client-1.0.4.jar:?]

    at io.druid.guice.http.HttpClientModule$HttpClientProvider.get(HttpClientModule.java:116) ~[druid-server-0.9.2.jar:0.9.2]

    at io.druid.guice.http.HttpClientModule$HttpClientProvider.get(HttpClientModule.java:86) ~[druid-server-0.9.2.jar:0.9.2]

    at com.google.inject.internal.ProviderInternalFactory.provision(ProviderInternalFactory.java:81) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InternalFactoryToInitializableAdapter.provision(InternalFactoryToInitializableAdapter.java:53) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ProviderInternalFactory.circularGet(ProviderInternalFactory.java:61) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InternalFactoryToInitializableAdapter.get(InternalFactoryToInitializableAdapter.java:45) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:194) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:41) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.SingleParameterInjector.inject(SingleParameterInjector.java:38) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.SingleParameterInjector.getAll(SingleParameterInjector.java:62) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:110) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:90) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:268) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InjectorImpl$2$1.call(InjectorImpl.java:1019) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InjectorImpl$2.get(InjectorImpl.java:1015) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InjectorImpl.getInstance(InjectorImpl.java:1050) ~[guice-4.1.0.jar:?]

    at io.druid.guice.PolyBind$ConfiggedProvider.get(PolyBind.java:178) ~[druid-api-0.9.2.jar:0.9.2]

    at com.google.inject.internal.ProviderInternalFactory.provision(ProviderInternalFactory.java:81) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InternalFactoryToInitializableAdapter.provision(InternalFactoryToInitializableAdapter.java:53) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ProviderInternalFactory.circularGet(ProviderInternalFactory.java:61) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InternalFactoryToInitializableAdapter.get(InternalFactoryToInitializableAdapter.java:45) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.SingleParameterInjector.inject(SingleParameterInjector.java:38) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.SingleParameterInjector.getAll(SingleParameterInjector.java:62) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:110) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:90) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:268) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.ProviderToInternalFactoryAdapter$1.call(ProviderToInternalFactoryAdapter.java:46) ~[guice-4.1.0.jar:?]

    at com.google.inject.internal.InjectorImpl.callInContext(InjectorImpl.java:1092) ~[guice-4.1.0.jar:?]

The common.runtime.properties is: