Druid Re-index exceptions

Hi,

I am trying to reindex druid datasource with transformation from one datasource “ABCD” to another datasource “XYZ” but getting exceptions as given below -

Druid Version: 0.19.0

2020-10-06T14:56:41,863 INFO [task-runner-0-priority-0] org.apache.druid.indexing.common.task.IndexTask - Found intervals and shardSpecs in 284,792ms
2020-10-06T14:56:41,864 INFO [task-runner-0-priority-0] org.apache.druid.indexing.common.task.AbstractBatchIndexTask - [forceTimeChunkLock] is set to true in task context. Use timeChunk lock
2020-10-06T14:56:42,274 INFO [task-runner-0-priority-0] org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver - New segment[XYZ_2020-09-08T00:00:00.000Z_2020-09-09T00:00:00.000Z_2020-10-06T14:56:42.547Z] for sequenceName[index_parallel_XYZ_ffpiimne_2020-10-06T14:51:51.533Z].
2020-10-06T14:56:42,329 WARN [task-runner-0-priority-0] org.apache.druid.indexing.input.DruidSegmentInputEntity - Could not clean temporary segment file: var/druid/task/index_parallel_XYZ_ffpiimne_2020-10-06T14:51:51.533Z/work/indexing-tmp/ABCD/2020-09-08T00:00:00.000Z_2020-09-09T00:00:00.000Z/2020-09-22T08:41:02.443Z/0
2020-10-06T14:56:42,331 ERROR [task-runner-0-priority-0] org.apache.druid.indexing.common.task.IndexTask - Encountered exception in BUILD_SEGMENTS.
java.lang.IllegalArgumentException: unsupported type STRING
at org.apache.druid.math.expr.BinaryEvalOpExprBase.evalString(Expr.java:1567) ~[druid-core-0.19.0.jar:0.19.0]
at org.apache.druid.math.expr.BinaryEvalOpExprBase.eval(Expr.java:1551) ~[druid-core-0.19.0.jar:0.19.0]
at org.apache.druid.segment.transform.ExpressionTransform$ExpressionRowFunction.eval(ExpressionTransform.java:86) ~[druid-processing-0.19.0.jar:0.19.0]
at org.apache.druid.segment.transform.Transformer$TransformedInputRow.getRaw(Transformer.java:193) ~[druid-processing-0.19.0.jar:0.19.0]
at org.apache.druid.segment.incremental.IncrementalIndex.toIncrementalIndexRow(IncrementalIndex.java:681) ~[druid-processing-0.19.0.jar:0.19.0]
at org.apache.druid.segment.incremental.IncrementalIndex.add(IncrementalIndex.java:616) ~[druid-processing-0.19.0.jar:0.19.0]
at org.apache.druid.segment.realtime.plumber.Sink.add(Sink.java:210) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.realtime.appenderator.AppenderatorImpl.add(AppenderatorImpl.java:259) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.append(BaseAppenderatorDriver.java:406) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.realtime.appenderator.BatchAppenderatorDriver.add(BatchAppenderatorDriver.java:114) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.InputSourceProcessor.process(InputSourceProcessor.java:143) ~[druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.IndexTask.generateAndPublishSegments(IndexTask.java:938) ~[druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.IndexTask.runTask(IndexTask.java:520) [druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.AbstractBatchIndexTask.run(AbstractBatchIndexTask.java:124) [druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexSupervisorTask.runSequential(ParallelIndexSupervisorTask.java:826) [druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexSupervisorTask.runTask(ParallelIndexSupervisorTask.java:454) [druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.common.task.AbstractBatchIndexTask.run(AbstractBatchIndexTask.java:124) [druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:421) [druid-indexing-service-0.19.0.jar:0.19.0]
at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:393) [druid-indexing-service-0.19.0.jar:0.19.0]
at java.util.concurrent.FutureTask.run(Unknown Source) [?:1.8.0_251]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:1.8.0_251]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:1.8.0_251]
at java.lang.Thread.run(Unknown Source) [?:1.8.0_251]
2020-10-06T14:56:42,353 WARN [task-runner-0-priority-0] org.apache.druid.segment.realtime.firehose.ServiceAnnouncingChatHandlerProvider - handler[index_parallel_XYZ_ffpiimne_2020-10-06T14:51:51.533Z] not currently registered, ignoring.
2020-10-06T14:56:42,359 INFO [task-runner-0-priority-0] org.apache.druid.indexing.worker.executor.ExecutorLifecycle - Task completed with status: {
“id” : “index_parallel_XYZ_ffpiimne_2020-10-06T14:51:51.533Z”,
“status” : “FAILED”,
“duration” : 285304,
“errorMsg” : “java.lang.IllegalArgumentException: unsupported type STRING\n\tat org.apache.druid.math.expr.BinaryEva…”,
“location” : {
“host” : null,
“port” : -1,
“tlsPort” : -1
}
}
2020-10-06T14:56:42,371 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [ANNOUNCEMENTS]
2020-10-06T14:56:42,372 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [SERVER]
2020-10-06T14:56:42,380 INFO [main] org.eclipse.jetty.server.AbstractConnector - Stopped ServerConnector@34070bd2{HTTP/1.1,[http/1.1]}{0.0.0.0:8100}
2020-10-06T14:56:42,380 INFO [main] org.eclipse.jetty.server.session - node0 Stopped scavenging
2020-10-06T14:56:42,383 INFO [main] org.eclipse.jetty.server.handler.ContextHandler - Stopped o.e.j.s.ServletContextHandler@6b4fd7d{/,null,UNAVAILABLE}
2020-10-06T14:56:42,390 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [NORMAL]
2020-10-06T14:56:42,391 INFO [main] org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner - Starting graceful shutdown of task[index_parallel_XYZ_ffpiimne_2020-10-06T14:51:51.533Z].
2020-10-06T14:56:42,398 INFO [main] org.apache.druid.security.basic.authentication.db.cache.CoordinatorPollingBasicAuthenticatorCacheManager - CoordinatorPollingBasicAuthenticatorCacheManager is stopping.
2020-10-06T14:56:42,398 INFO [main] org.apache.druid.security.basic.authentication.db.cache.CoordinatorPollingBasicAuthenticatorCacheManager - CoordinatorPollingBasicAuthenticatorCacheManager is stopped.
2020-10-06T14:56:42,398 INFO [main] org.apache.druid.security.basic.authorization.db.cache.CoordinatorPollingBasicAuthorizerCacheManager - CoordinatorPollingBasicAuthorizerCacheManager is stopping.
2020-10-06T14:56:42,398 INFO [main] org.apache.druid.security.basic.authorization.db.cache.CoordinatorPollingBasicAuthorizerCacheManager - CoordinatorPollingBasicAuthorizerCacheManager is stopped.
2020-10-06T14:56:42,401 INFO [LookupExtractorFactoryContainerProvider-MainThread] org.apache.druid.query.lookup.LookupReferencesManager - Lookup Management loop exited. Lookup notices are not handled anymore.
2020-10-06T14:56:42,445 INFO [Curator-Framework-0] org.apache.curator.framework.imps.CuratorFrameworkImpl - backgroundOperationsLoop exiting
2020-10-06T14:56:42,449 INFO [main] org.apache.zookeeper.ZooKeeper - Session: 0x1007bcda49302cb closed
2020-10-06T14:56:42,450 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle$CloseableHandler - Closing object[org.asynchttpclient.DefaultAsyncHttpClient@3d7b3b18]
2020-10-06T14:56:42,452 INFO [main-EventThread] org.apache.zookeeper.ClientCnxn - EventThread shut down for session: 0x1007bcda49302cb
2020-10-06T14:56:42,460 INFO [main] org.apache.druid.java.util.common.lifecycle.Lifecycle - Stopping lifecycle [module] stage [INIT]
Finished peon task

Can you please help me out of this issue.

Thanks
Amit Srivastava

Looks like it is trying to do a transform on type that is not supported? Can you send over the spec?

Thanks!

Rachel

Hi Rachel,

I am trying to reindex with following type of spec -

{
“type”: “index_parallel”,
“spec”: {
“ioConfig”: {
“type”: “index_parallel”,
“inputSource”: {
“type”: “druid”,
“dataSource”: “ABCD”,
“interval”: “2020-09-08T00:00:00/2020-09-09T00:00:00”
}
},
“tuningConfig”: {
“type”: “index_parallel”,
“partitionsSpec”: {
“type”: “dynamic”
}
},
“dataSchema”: {
“dataSource”: “ABCD_Reindex”,
“granularitySpec”: {
“type”: “uniform”,
“queryGranularity”: “DAY”,
“rollup”: true,
“segmentGranularity”: “DAY”
},
“timestampSpec”: {
“column”: “__time”,
“format”: “iso”
},
“dimensionsSpec”: {
“dimensions”: [
{
“name”: “Bureau_Articles_Read_LifeTime”,
“type”: “string”
},
{
“name”: “bureau_articles_read_lifetime”,
“type”: “string”
},
{
“name”: “Days_Active_Life_Time”,
“type”: “string”
},
{
“name”: “days_active_life_time”,
“type”: “string”
},

{
“name”: “author”,
“type”: “string”
},
{
“name”: “author_name”,
“type”: “string”
},
{
“name”: “author_id”,
“type”: “string”
},
{
“name”: “category”,
“type”: “string”
},
{
“name”: “city”,
“type”: “string”
},

{
“name”: “createdAt”,
“type”: “string”
},
{
“name”: “created_at”,
“type”: “string”
},
{
“name”: “current_subscriber_status”,
“type”: “string”
},
{
“name”: “deviceCategory”,
“type”: “string”
},
{
“name”: “device_category”,
“type”: “string”
},
{
“name”: “deviceManufacturer”,
“type”: “string”
},
{
“name”: “device_manufacturer”,
“type”: “string”
},
{
“name”: “deviceModel”,
“type”: “string”
},
{
“name”: “device_model”,
“type”: “string”
},
{
“name”: “email”,
“type”: “string”
},
{
“name”: “event_action”,
“type”: “string”
},
{
“name”: “event_category”,
“type”: “string”
},
{
“name”: “event_label”,
“type”: “string”
},

{
“name”: “first_name”,
“type”: “string”
},
{
“name”: “free_to_read”,
“type”: “string”
},
{
“name”: “gender”,
“type”: “string”
},
{
“name”: “gid”,
“type”: “string”
},
{
“name”: “gidint”,
“type”: “string”
},
{
“name”: “gid_int”,
“type”: “string”
},
{
“name”: “ip_addr”,
“type”: “string”
},
{
“name”: “last_name”,
“type”: “string”
},
{
“name”: “last_visited_date”,
“type”: “string”
},
{
“name”: “localeLang”,
“type”: “string”
},
{
“name”: “locale_lang”,
“type”: “string”
},
{
“name”: “logged_in”,
“type”: “string”
},
{
“name”: “name”,
“type”: “string”
},
{
“name”: “event_name”,
“type”: “string”
},
{
“name”: “nature_of_content”,
“type”: “string”
},
{
“name”: “network”,
“type”: “string”
},
{
“name”: “new_version”,
“type”: “string”
},
{
“name”: “osAPILevel”,
“type”: “string”
},
{
“name”: “os_api_level”,
“type”: “string”
},
{
“name”: “osVersion”,
“type”: “string”
},
{
“name”: “os_version”,
“type”: “string”
},
{
“name”: “page”,
“type”: “string”
},
{
“name”: “page_url”,
“type”: “string”
},
{
“name”: “page_template”,
“type”: “string”
},
{
“name”: “past_subscription”,
“type”: “string”
},
{
“name”: “personalized_content”,
“type”: “string”
},
{
“name”: “phone”,
“type”: “string”
},
{
“name”: “planId”,
“type”: “string”
},
{
“name”: “planid”,
“type”: “string”
},
{
“name”: “platform”,
“type”: “string”
},
{
“name”: “prime_article_read_before_syft”,
“type”: “string”
},

{
“name”: “prime_hp_ui_content_b_color”,
“type”: “string”
},
{
“name”: “prime_hp_ui_content_size”,
“type”: “string”
},
{
“name”: “prime_hp_ui_template”,
“type”: “string”
},
{
“name”: “prime_user_acquisition_type”,
“type”: “string”
},
{
“name”: “product”,
“type”: “string”
},
{
“name”: “projectCode”,
“type”: “string”
},
{
“name”: “project_code”,
“type”: “string”
},
{
“name”: “publish_date”,
“type”: “string”
},
{
“name”: “referer”,
“type”: “string”
},
{
“name”: “screen_title”,
“type”: “string”
},
{
“name”: “sdkBuild”,
“type”: “string”
},
{
“name”: “sdk_build”,
“type”: “string”
},
{
“name”: “sdkVersion”,
“type”: “string”
},
{
“name”: “sdk_version”,
“type”: “string”
},
{
“name”: “section_id”,
“type”: “string”
},
{
“name”: “sentAt”,
“type”: “string”
},
{
“name”: “sent_at”,
“type”: “string”
},
{
“name”: “sign_in_initiation_position”,
“type”: “string”
},
{
“name”: “signin_initiate_page”,
“type”: “string”
},
{
“name”: “site_section”,
“type”: “string”
},
{
“name”: “site_sub_section”,
“type”: “string”
},
{
“name”: “socialaction”,
“type”: “string”
},
{
“name”: “social_action”,
“type”: “string”
},
{
“name”: “socialnetwork”,
“type”: “string”
},
{
“name”: “social_network”,
“type”: “string”
},
{
“name”: “socialtarget”,
“type”: “string”
},
{
“name”: “social_target”,
“type”: “string”
},
{
“name”: “ssoId”,
“type”: “string”
},
{
“name”: “sso_id”,
“type”: “string”
},
{
“name”: “status”,
“type”: “string”
},
{
“name”: “sub_section_name”,
“type”: “string”
},
{
“name”: “subscription_cancellation_date”,
“type”: “string”
},
{
“name”: “subscription_end_date”,
“type”: “string”
},
{
“name”: “subscription_method”,
“type”: “string”
},
{
“name”: “subscription_start_date”,
“type”: “string”
},
{
“name”: “syft_initiate_page”,
“type”: “string”
},
{
“name”: “tags_meta_keyword”,
“type”: “string”
},
{
“name”: “timestamp”,
“type”: “string”
},
{
“name”: “timezone”,
“type”: “string”
},
{
“name”: “title”,
“type”: “string”
},
{
“name”: “trial_end_date”,
“type”: “string”
},
{
“name”: “type”,
“type”: “string”
},
{
“name”: “event_type”,
“type”: “string”
},
{
“name”: “url”,
“type”: “string”
},
{
“name”: “user-agent”,
“type”: “string”
},
{
“name”: “user_agent”,
“type”: “string”
},
{
“name”: “userId”,
“type”: “string”
},
{
“name”: “user_id”,
“type”: “string”
},
{
“name”: “user_login_status”,
“type”: “string”
},
{
“name”: “user_login_status_hit”,
“type”: “string”
},
{
“name”: “uuid”,
“type”: “string”
},
{
“name”: “video”,
“type”: “string”
},
{
“name”: “video_embed”,
“type”: “string”
}
]
},
“metricsSpec”: [
{
“type”: “longSum”,
“name”: “count”,
“fieldName”: “count”,
“expression”: null
}
],
“transformSpec”: {
“transforms”: [
{
“type”: “expression”,
“name”: “bureau_articles_read_lifetime”,
“expression”: “Bureau_Articles_Read_LifeTime”
},
{
“type”: “expression”,
“name”: “days_active_life_time”,
“expression”: “Days_Active_Life_Time”
},

{
“type”: “expression”,
“name”: “author_name”,
“expression”: “author”
},
{
“type”: “expression”,
“name”: “msid”,
“expression”: “content_msid”
},
{
“type”: “expression”,
“name”: “created_at”,
“expression”: “createdAt”
},
{
“type”: “expression”,
“name”: “device_category”,
“expression”: “deviceCategory”
},
{
“type”: “expression”,
“name”: “device_manufacturer”,
“expression”: “deviceManufacturer”
},
{
“type”: “expression”,
“name”: “device_model”,
“expression”: “deviceModel”
},
{
“type”: “expression”,
“name”: “gid_int”,
“expression”: “gidint”
},
{
“type”: “expression”,
“name”: “ip_addr”,
“expression”: “ip”
},
{
“type”: “expression”,
“name”: “locale_lang”,
“expression”: “localeLang”
},
{
“type”: “expression”,
“name”: “event_name”,
“expression”: “name”
},
{
“type”: “expression”,
“name”: “os_api_level”,
“expression”: “osAPILevel”
},
{
“type”: “expression”,
“name”: “os_version”,
“expression”: “osVersion”
},
{
“type”: “expression”,
“name”: “page_url”,
“expression”: “page”
},
{
“type”: “expression”,
“name”: “planid”,
“expression”: “planId”
},
{
“type”: “expression”,
“name”: “project_code”,
“expression”: “projectCode”
},
{
“type”: “expression”,
“name”: “sdk_build”,
“expression”: “sdkBuild”
},
{
“type”: “expression”,
“name”: “sdk_version”,
“expression”: “sdkVersion”
},
{
“type”: “expression”,
“name”: “sent_at”,
“expression”: “sentAt”
},
{
“type”: “expression”,
“name”: “social_action”,
“expression”: “socialaction”
},
{
“type”: “expression”,
“name”: “social_network”,
“expression”: “socialnetwork”
},
{
“type”: “expression”,
“name”: “social_target”,
“expression”: “socialtarget”
},
{
“type”: “expression”,
“name”: “sso_id”,
“expression”: “ssoId”
},
{
“type”: “expression”,
“name”: “event_type”,
“expression”: “type”
},
{
“type”: “expression”,
“name”: “user_agent”,
“expression”: “user-agent”
},
{
“type”: “expression”,
“name”: “user_id”,
“expression”: “userId”
}
]
}
}
}
}

Regards
Amit