Flatten Json spec structure issue

Hi All,

I am facing issue when try to use “Flatten Json Spec”. (I am trying to fetch data from Twitter , using Tranquility server)

In my spec single level data ingested but second level of data was not ingesting, its show NULL.

I am getting data in “text” but in “username” generate null values.(for code referee Full spec)

In my spec I used below code for Flatten

"parser" : {

"type" : “string”,

"parseSpec" : {

"format" : “json”,

"flattenSpec": {

"useFieldDiscovery": true,

"fields": [

{

"type": “path”,

"name": “username”,

"expr": "$.user.name"

}

]

},

Full Spec
{

“dataSources” : {

“twitter_8” : {

“spec” : {

“dataSchema” : {

“dataSource” : “twitter_8”,

“parser” : {

“type” : “string”,

“parseSpec” : {

“format” : “json”,

“flattenSpec”: {

“useFieldDiscovery”: true,

“fields”: [

{

“type”: “path”,

“name”: “username”,

“expr”: “$.user.name”

}

]

},

“dimensionsSpec” : {

“dimensions” : [

“username”,

“text”

],

“dimensionsExclusions”: [“ignore_me”]

},

“timestampSpec”: {

“column”: “created_at”,

“format”: “EEE MMM dd HH:mm:ss Z yyyy”

}

}

},

“metricsSpec” : [

{

“name” : “added”,

“type” : “longSum”,

“fieldName” : “source.retweet_count”

}

],

“granularitySpec” : {

“type” : “uniform”,

“segmentGranularity” : “hour”,

“queryGranularity” : “none”,

“intervals” : [“2015-09-12/2019-09-25”],

“rollup” : false

}

},

“ioConfig” : {

“type” : “realtime”

},

“tuningConfig” : {

“type” : “realtime”,

“intermediatePersistPeriod” : “PT10M”,

“windowPeriod” : “PT15M”

}

},

“properties” : {

“task.partitions” : “1”,

“task.replicants” : “1”

}

}

},

“properties” : {

“zookeeper.connect” : “localhost”,

“druid.discovery.curator.path” : “/druid/discovery”,

“druid.selectors.indexing.serviceName” : “druid/overlord”,

“http.port” : “8200”,

“http.threads” : “9”

}

}

Data screenshot

Please let me know, how could I access 2ndlevel of field.
Eg : from below source , I want to fetch location.

{

“created_at”: “Thu Jun 27 13:23:44 +0000 2019”,

“id”: 1144234859767840800,

“id_str”: “1144234859767840774”,

“text”: “RT @MsPackyetti: Seeing @Dell, @BoozAllen & @usbank among the most well-recognized companies who are contracted with ICE. \n\nKeep your #Prid…”,

“truncated”: false,

“entities”: {

“hashtags”: [

],

“symbols”: [

],

“user_mentions”: [

{

“screen_name”: “MsPackyetti”,

“name”: “Brittany Packnett”,

“id”: 239509917,

“id_str”: “239509917”,

“indices”: [

3,

15

]

},

{

“screen_name”: “Dell”,

“name”: “Dell”,

“id”: 58561993,

“id_str”: “58561993”,

“indices”: [

24,

29

]

},

{

“screen_name”: “BoozAllen”,

“name”: “Booz Allen Hamilton”,

“id”: 17375116,

“id_str”: “17375116”,

“indices”: [

31,

41

]

},

{

“screen_name”: “usbank”,

“name”: “U.S. Bank”,

“id”: 15577098,

“id_str”: “15577098”,

“indices”: [

48,

55

]

}

],

“urls”: [

]

},

“metadata”: {

“iso_language_code”: “en”,

“result_type”: “recent”

},

“source”: “<a href=”[http://twitter.com/#!/download/ipad](http://twitter.com/#!/download/ipad%5C)" rel=“nofollow”>Twitter for iPad",

“in_reply_to_status_id”: null,

“in_reply_to_status_id_str”: null,

“in_reply_to_user_id”: null,

“in_reply_to_user_id_str”: null,

“in_reply_to_screen_name”: null,

“user”: {

“id”: 16006207,

“id_str”: “16006207”,

“name”: “rage against the patriarchy”,

“screen_name”: “Daphnesmama”,

“location”: “California”,

“description”: “Mid-life crisis in full swing. Stop and smell the rosacea. #Bernie2020”,

“url”: null,

“entities”: {

“description”: {

“urls”: [

]

}

},

“protected”: false,

“followers_count”: 209,

“friends_count”: 1425,

“listed_count”: 3,

“created_at”: “Wed Aug 27 01:54:04 +0000 2008”,

“favourites_count”: 46301,

“utc_offset”: null,

“time_zone”: null,

“geo_enabled”: false,

“verified”: false,

“statuses_count”: 12346,

“lang”: “en”,

“contributors_enabled”: false,

“is_translator”: false,

“is_translation_enabled”: false,

“profile_background_color”: “C0DEED”,

“profile_background_image_url”: “http://abs.twimg.com/images/themes/theme1/bg.png”,

“profile_background_image_url_https”: “https://abs.twimg.com/images/themes/theme1/bg.png”,

“profile_background_tile”: false,

“profile_image_url”: “http://pbs.twimg.com/profile_images/1048450656774680576/uvLreyvl_normal.jpg”,

“profile_image_url_https”: “https://pbs.twimg.com/profile_images/1048450656774680576/uvLreyvl_normal.jpg”,

“profile_banner_url”: “https://pbs.twimg.com/profile_banners/16006207/1544660056”,

“profile_link_color”: “1DA1F2”,

“profile_sidebar_border_color”: “C0DEED”,

“profile_sidebar_fill_color”: “DDEEF6”,

“profile_text_color”: “333333”,

“profile_use_background_image”: true,

“has_extended_profile”: false,

“default_profile”: true,

“default_profile_image”: false,

“following”: null,

“follow_request_sent”: null,

“notifications”: null,

“translator_type”: “none”

},

“geo”: null,

“coordinates”: null,

“place”: null,

“contributors”: null,

“retweeted_status”: {

“created_at”: “Wed Jun 26 12:23:13 +0000 2019”,

“id”: 1143857245496709100,

“id_str”: “1143857245496709120”,

“text”: “Seeing @Dell, @BoozAllen & @usbank among the most well-recognized companies who are contracted with ICE. \n\nKeep you… https://t.co/um7bF2t4TJ”,

“truncated”: true,

“entities”: {

“hashtags”: [

],

“symbols”: [

],

“user_mentions”: [

{

“screen_name”: “Dell”,

“name”: “Dell”,

“id”: 58561993,

“id_str”: “58561993”,

“indices”: [

7,

12

]

},

{

“screen_name”: “BoozAllen”,

“name”: “Booz Allen Hamilton”,

“id”: 17375116,

“id_str”: “17375116”,

“indices”: [

14,

24

]

},

{

“screen_name”: “usbank”,

“name”: “U.S. Bank”,

“id”: 15577098,

“id_str”: “15577098”,

“indices”: [

31,

38

]

}

],

“urls”: [

{

“url”: “https://t.co/um7bF2t4TJ”,

“expanded_url”: “https://twitter.com/i/web/status/1143857245496709120”,

“display_url”: “twitter.com/i/web/status/1…”,

“indices”: [

121,

144

]

}

]

},

“metadata”: {

“iso_language_code”: “en”,

“result_type”: “recent”

},

“source”: “<a href=”[http://twitter.com/download/iphone](http://twitter.com/download/iphone/)" rel=“nofollow”>Twitter for iPhone",

“in_reply_to_status_id”: null,

“in_reply_to_status_id_str”: null,

“in_reply_to_user_id”: null,

“in_reply_to_user_id_str”: null,

“in_reply_to_screen_name”: null,

“user”: {

“id”: 239509917,

“id_str”: “239509917”,

“name”: “Brittany Packnett”,

“screen_name”: “MsPackyetti”,

“location”: “brittany@brittanypacknett.com”,

“description”: “learning in public | teaching truth to move people to action | @buildlovepower fndr | fmr @HarvardIOP fellow | Ferguson | @kidnoble’s +1 for life | she/her/hers”,

“url”: “https://t.co/7Q9CYUv0eq”,

“entities”: {

“url”: {

“urls”: [

{

“url”: “https://t.co/7Q9CYUv0eq”,

“expanded_url”: “http://brittanypacknett.com”,

“display_url”: “brittanypacknett.com”,

“indices”: [

0,

23

]

}

]

},

“description”: {

“urls”: [

]

}

},

“protected”: false,

“followers_count”: 203553,

“friends_count”: 3402,

“listed_count”: 2320,

“created_at”: “Mon Jan 17 19:58:52 +0000 2011”,

“favourites_count”: 45948,

“utc_offset”: null,

“time_zone”: null,

“geo_enabled”: true,

“verified”: true,

“statuses_count”: 91171,

“lang”: “en”,

“contributors_enabled”: false,

“is_translator”: false,

“is_translation_enabled”: false,

“profile_background_color”: “B2DFDA”,

“profile_background_image_url”: “http://abs.twimg.com/images/themes/theme1/bg.png”,

“profile_background_image_url_https”: “https://abs.twimg.com/images/themes/theme1/bg.png”,

“profile_background_tile”: true,

“profile_image_url”: “http://pbs.twimg.com/profile_images/1099192062434525185/blDam2Ac_normal.jpg”,

“profile_image_url_https”: “https://pbs.twimg.com/profile_images/1099192062434525185/blDam2Ac_normal.jpg”,

“profile_banner_url”: “https://pbs.twimg.com/profile_banners/239509917/1552431921”,

“profile_link_color”: “E81C4F”,

“profile_sidebar_border_color”: “FFFFFF”,

“profile_sidebar_fill_color”: “E6E640”,

“profile_text_color”: “6D2CB8”,

“profile_use_background_image”: true,

“has_extended_profile”: true,

“default_profile”: false,

“default_profile_image”: false,

“following”: null,

“follow_request_sent”: null,

“notifications”: null,

“translator_type”: “none”

},

“geo”: null,

“coordinates”: null,

“place”: null,

“contributors”: null,

“is_quote_status”: true,

“quoted_status_id”: 1143653733437444100,

“quoted_status_id_str”: “1143653733437444096”,

“quoted_status”: {

“created_at”: “Tue Jun 25 22:54:32 +0000 2019”,

“id”: 1143653733437444100,

“id_str”: “1143653733437444096”,

“text”: “An incomplete list of companies working with ICE (thread):”,

“truncated”: false,

“entities”: {

“hashtags”: [

],

“symbols”: [

],

“user_mentions”: [

],

“urls”: [

]

},

“metadata”: {

“iso_language_code”: “en”,

“result_type”: “recent”

},

“source”: “<a href=”[http://twitter.com](http://twitter.com/)" rel=“nofollow”>Twitter Web Client",

“in_reply_to_status_id”: null,

“in_reply_to_status_id_str”: null,

“in_reply_to_user_id”: null,

“in_reply_to_user_id_str”: null,

“in_reply_to_screen_name”: null,

“user”: {

“id”: 27991531,

“id_str”: “27991531”,

“name”: “Babadook 2: Back in the Dook”,

“screen_name”: “babadookspinoza”,

"location": “Occupied Wampanoag Territory”,

“description”: “@AmericanU and @Wheaton College alum. Philosophy and leftist politics. Actual communist. ☭ Please don’t follow me. (he/him)”,

“url”: “https://t.co/9GhXAIRLcY”,

“entities”: {

“url”: {

“urls”: [

{

“url”: “https://t.co/9GhXAIRLcY”,

“expanded_url”: “https://curiouscat.me/babadook_spinoza”,

“display_url”: “curiouscat.me/babadook_spino…”,

“indices”: [

0,

23

]

}

]

},

“description”: {

“urls”: [

]

}

},

“protected”: false,

“followers_count”: 15676,

“friends_count”: 2987,

“listed_count”: 208,

“created_at”: “Tue Mar 31 23:18:32 +0000 2009”,

“favourites_count”: 123819,

“utc_offset”: null,

“time_zone”: null,

“geo_enabled”: true,

“verified”: false,

“statuses_count”: 33834,

“lang”: “en”,

“contributors_enabled”: false,

“is_translator”: false,

“is_translation_enabled”: false,

“profile_background_color”: “B2DFDA”,

“profile_background_image_url”: “http://abs.twimg.com/images/themes/theme13/bg.gif”,

“profile_background_image_url_https”: “https://abs.twimg.com/images/themes/theme13/bg.gif”,

“profile_background_tile”: false,

“profile_image_url”: “http://pbs.twimg.com/profile_images/1116918074043764736/nLsweU3h_normal.jpg”,

“profile_image_url_https”: “https://pbs.twimg.com/profile_images/1116918074043764736/nLsweU3h_normal.jpg”,

“profile_banner_url”: “https://pbs.twimg.com/profile_banners/27991531/1514749069”,

“profile_link_color”: “4A913C”,

“profile_sidebar_border_color”: “FFFFFF”,

“profile_sidebar_fill_color”: “FFFFFF”,

“profile_text_color”: “333333”,

“profile_use_background_image”: true,

“has_extended_profile”: true,

“default_profile”: false,

“default_profile_image”: false,

“following”: null,

“follow_request_sent”: null,

“notifications”: null,

“translator_type”: “none”

},

“geo”: null,

“coordinates”: null,

“place”: null,

“contributors”: null,

“is_quote_status”: false,

“retweet_count”: 14049,

“favorite_count”: 15352,

“favorited”: false,

“retweeted”: false,

“lang”: “en”

},

“retweet_count”: 270,

“favorite_count”: 506,

“favorited”: false,

“retweeted”: false,

“possibly_sensitive”: false,

“lang”: “en”

},

“is_quote_status”: true,

“quoted_status_id”: 1143653733437444100,

“quoted_status_id_str”: “1143653733437444096”,

“retweet_count”: 270,

“favorite_count”: 0,

“favorited”: false,

“retweeted”: false,

“lang”: “en”

}

regrads,

Sumeet Lalvani