Hi All,
I am facing issue when try to use “Flatten Json Spec”. (I am trying to fetch data from Twitter , using Tranquility server)
In my spec single level data ingested but second level of data was not ingesting, its show NULL.
I am getting data in “text” but in “username” generate null values.(for code referee Full spec)
In my spec I used below code for Flatten
"parser" : {
"type" : “string”,
"parseSpec" : {
"format" : “json”,
"flattenSpec": {
"useFieldDiscovery": true,
"fields": [
{
"type": “path”,
"name": “username”,
"expr": "$.user.name"
}
]
},
Full Spec
{
“dataSources” : {
“twitter_8” : {
“spec” : {
“dataSchema” : {
“dataSource” : “twitter_8”,
“parser” : {
“type” : “string”,
“parseSpec” : {
“format” : “json”,
“flattenSpec”: {
“useFieldDiscovery”: true,
“fields”: [
{
“type”: “path”,
“name”: “username”,
“expr”: “$.user.name”
}
]
},
“dimensionsSpec” : {
“dimensions” : [
“username”,
“text”
],
“dimensionsExclusions”: [“ignore_me”]
},
“timestampSpec”: {
“column”: “created_at”,
“format”: “EEE MMM dd HH:mm:ss Z yyyy”
}
}
},
“metricsSpec” : [
{
“name” : “added”,
“type” : “longSum”,
“fieldName” : “source.retweet_count”
}
],
“granularitySpec” : {
“type” : “uniform”,
“segmentGranularity” : “hour”,
“queryGranularity” : “none”,
“intervals” : [“2015-09-12/2019-09-25”],
“rollup” : false
}
},
“ioConfig” : {
“type” : “realtime”
},
“tuningConfig” : {
“type” : “realtime”,
“intermediatePersistPeriod” : “PT10M”,
“windowPeriod” : “PT15M”
}
},
“properties” : {
“task.partitions” : “1”,
“task.replicants” : “1”
}
}
},
“properties” : {
“zookeeper.connect” : “localhost”,
“druid.discovery.curator.path” : “/druid/discovery”,
“druid.selectors.indexing.serviceName” : “druid/overlord”,
“http.port” : “8200”,
“http.threads” : “9”
}
}
Data screenshot
Please let me know, how could I access 2ndlevel of field.
Eg : from below source , I want to fetch location.
{
“created_at”: “Thu Jun 27 13:23:44 +0000 2019”,
“id”: 1144234859767840800,
“id_str”: “1144234859767840774”,
“text”: “RT @MsPackyetti: Seeing @Dell, @BoozAllen & @usbank among the most well-recognized companies who are contracted with ICE. \n\nKeep your #Prid…”,
“truncated”: false,
“entities”: {
“hashtags”: [
],
“symbols”: [
],
“user_mentions”: [
{
“screen_name”: “MsPackyetti”,
“name”: “Brittany Packnett”,
“id”: 239509917,
“id_str”: “239509917”,
“indices”: [
3,
15
]
},
{
“screen_name”: “Dell”,
“name”: “Dell”,
“id”: 58561993,
“id_str”: “58561993”,
“indices”: [
24,
29
]
},
{
“screen_name”: “BoozAllen”,
“name”: “Booz Allen Hamilton”,
“id”: 17375116,
“id_str”: “17375116”,
“indices”: [
31,
41
]
},
{
“screen_name”: “usbank”,
“name”: “U.S. Bank”,
“id”: 15577098,
“id_str”: “15577098”,
“indices”: [
48,
55
]
}
],
“urls”: [
]
},
“metadata”: {
“iso_language_code”: “en”,
“result_type”: “recent”
},
“source”: “<a href=”[http://twitter.com/#!/download/ipad](http://twitter.com/#!/download/ipad%5C)" rel=“nofollow”>Twitter for iPad",
“in_reply_to_status_id”: null,
“in_reply_to_status_id_str”: null,
“in_reply_to_user_id”: null,
“in_reply_to_user_id_str”: null,
“in_reply_to_screen_name”: null,
“user”: {
“id”: 16006207,
“id_str”: “16006207”,
“name”: “rage against the patriarchy”,
“screen_name”: “Daphnesmama”,
“location”: “California”,
“description”: “Mid-life crisis in full swing. Stop and smell the rosacea. #Bernie2020”,
“url”: null,
“entities”: {
“description”: {
“urls”: [
]
}
},
“protected”: false,
“followers_count”: 209,
“friends_count”: 1425,
“listed_count”: 3,
“created_at”: “Wed Aug 27 01:54:04 +0000 2008”,
“favourites_count”: 46301,
“utc_offset”: null,
“time_zone”: null,
“geo_enabled”: false,
“verified”: false,
“statuses_count”: 12346,
“lang”: “en”,
“contributors_enabled”: false,
“is_translator”: false,
“is_translation_enabled”: false,
“profile_background_color”: “C0DEED”,
“profile_background_image_url”: “http://abs.twimg.com/images/themes/theme1/bg.png”,
“profile_background_image_url_https”: “https://abs.twimg.com/images/themes/theme1/bg.png”,
“profile_background_tile”: false,
“profile_image_url”: “http://pbs.twimg.com/profile_images/1048450656774680576/uvLreyvl_normal.jpg”,
“profile_image_url_https”: “https://pbs.twimg.com/profile_images/1048450656774680576/uvLreyvl_normal.jpg”,
“profile_banner_url”: “https://pbs.twimg.com/profile_banners/16006207/1544660056”,
“profile_link_color”: “1DA1F2”,
“profile_sidebar_border_color”: “C0DEED”,
“profile_sidebar_fill_color”: “DDEEF6”,
“profile_text_color”: “333333”,
“profile_use_background_image”: true,
“has_extended_profile”: false,
“default_profile”: true,
“default_profile_image”: false,
“following”: null,
“follow_request_sent”: null,
“notifications”: null,
“translator_type”: “none”
},
“geo”: null,
“coordinates”: null,
“place”: null,
“contributors”: null,
“retweeted_status”: {
“created_at”: “Wed Jun 26 12:23:13 +0000 2019”,
“id”: 1143857245496709100,
“id_str”: “1143857245496709120”,
“text”: “Seeing @Dell, @BoozAllen & @usbank among the most well-recognized companies who are contracted with ICE. \n\nKeep you… https://t.co/um7bF2t4TJ”,
“truncated”: true,
“entities”: {
“hashtags”: [
],
“symbols”: [
],
“user_mentions”: [
{
“screen_name”: “Dell”,
“name”: “Dell”,
“id”: 58561993,
“id_str”: “58561993”,
“indices”: [
7,
12
]
},
{
“screen_name”: “BoozAllen”,
“name”: “Booz Allen Hamilton”,
“id”: 17375116,
“id_str”: “17375116”,
“indices”: [
14,
24
]
},
{
“screen_name”: “usbank”,
“name”: “U.S. Bank”,
“id”: 15577098,
“id_str”: “15577098”,
“indices”: [
31,
38
]
}
],
“urls”: [
{
“url”: “https://t.co/um7bF2t4TJ”,
“expanded_url”: “https://twitter.com/i/web/status/1143857245496709120”,
“display_url”: “twitter.com/i/web/status/1…”,
“indices”: [
121,
144
]
}
]
},
“metadata”: {
“iso_language_code”: “en”,
“result_type”: “recent”
},
“source”: “<a href=”[http://twitter.com/download/iphone](http://twitter.com/download/iphone/)" rel=“nofollow”>Twitter for iPhone",
“in_reply_to_status_id”: null,
“in_reply_to_status_id_str”: null,
“in_reply_to_user_id”: null,
“in_reply_to_user_id_str”: null,
“in_reply_to_screen_name”: null,
“user”: {
“id”: 239509917,
“id_str”: “239509917”,
“name”: “Brittany Packnett”,
“screen_name”: “MsPackyetti”,
“location”: “brittany@brittanypacknett.com”,
“description”: “learning in public | teaching truth to move people to action | @buildlovepower fndr | fmr @HarvardIOP fellow | Ferguson | @kidnoble’s +1 for life | she/her/hers”,
“url”: “https://t.co/7Q9CYUv0eq”,
“entities”: {
“url”: {
“urls”: [
{
“url”: “https://t.co/7Q9CYUv0eq”,
“expanded_url”: “http://brittanypacknett.com”,
“display_url”: “brittanypacknett.com”,
“indices”: [
0,
23
]
}
]
},
“description”: {
“urls”: [
]
}
},
“protected”: false,
“followers_count”: 203553,
“friends_count”: 3402,
“listed_count”: 2320,
“created_at”: “Mon Jan 17 19:58:52 +0000 2011”,
“favourites_count”: 45948,
“utc_offset”: null,
“time_zone”: null,
“geo_enabled”: true,
“verified”: true,
“statuses_count”: 91171,
“lang”: “en”,
“contributors_enabled”: false,
“is_translator”: false,
“is_translation_enabled”: false,
“profile_background_color”: “B2DFDA”,
“profile_background_image_url”: “http://abs.twimg.com/images/themes/theme1/bg.png”,
“profile_background_image_url_https”: “https://abs.twimg.com/images/themes/theme1/bg.png”,
“profile_background_tile”: true,
“profile_image_url”: “http://pbs.twimg.com/profile_images/1099192062434525185/blDam2Ac_normal.jpg”,
“profile_image_url_https”: “https://pbs.twimg.com/profile_images/1099192062434525185/blDam2Ac_normal.jpg”,
“profile_banner_url”: “https://pbs.twimg.com/profile_banners/239509917/1552431921”,
“profile_link_color”: “E81C4F”,
“profile_sidebar_border_color”: “FFFFFF”,
“profile_sidebar_fill_color”: “E6E640”,
“profile_text_color”: “6D2CB8”,
“profile_use_background_image”: true,
“has_extended_profile”: true,
“default_profile”: false,
“default_profile_image”: false,
“following”: null,
“follow_request_sent”: null,
“notifications”: null,
“translator_type”: “none”
},
“geo”: null,
“coordinates”: null,
“place”: null,
“contributors”: null,
“is_quote_status”: true,
“quoted_status_id”: 1143653733437444100,
“quoted_status_id_str”: “1143653733437444096”,
“quoted_status”: {
“created_at”: “Tue Jun 25 22:54:32 +0000 2019”,
“id”: 1143653733437444100,
“id_str”: “1143653733437444096”,
“text”: “An incomplete list of companies working with ICE (thread):”,
“truncated”: false,
“entities”: {
“hashtags”: [
],
“symbols”: [
],
“user_mentions”: [
],
“urls”: [
]
},
“metadata”: {
“iso_language_code”: “en”,
“result_type”: “recent”
},
“source”: “<a href=”[http://twitter.com](http://twitter.com/)" rel=“nofollow”>Twitter Web Client",
“in_reply_to_status_id”: null,
“in_reply_to_status_id_str”: null,
“in_reply_to_user_id”: null,
“in_reply_to_user_id_str”: null,
“in_reply_to_screen_name”: null,
“user”: {
“id”: 27991531,
“id_str”: “27991531”,
“name”: “Babadook 2: Back in the Dook”,
“screen_name”: “babadookspinoza”,
"location": “Occupied Wampanoag Territory”,
“description”: “@AmericanU and @Wheaton College alum. Philosophy and leftist politics. Actual communist. ☭ Please don’t follow me. (he/him)”,
“url”: “https://t.co/9GhXAIRLcY”,
“entities”: {
“url”: {
“urls”: [
{
“url”: “https://t.co/9GhXAIRLcY”,
“expanded_url”: “https://curiouscat.me/babadook_spinoza”,
“display_url”: “curiouscat.me/babadook_spino…”,
“indices”: [
0,
23
]
}
]
},
“description”: {
“urls”: [
]
}
},
“protected”: false,
“followers_count”: 15676,
“friends_count”: 2987,
“listed_count”: 208,
“created_at”: “Tue Mar 31 23:18:32 +0000 2009”,
“favourites_count”: 123819,
“utc_offset”: null,
“time_zone”: null,
“geo_enabled”: true,
“verified”: false,
“statuses_count”: 33834,
“lang”: “en”,
“contributors_enabled”: false,
“is_translator”: false,
“is_translation_enabled”: false,
“profile_background_color”: “B2DFDA”,
“profile_background_image_url”: “http://abs.twimg.com/images/themes/theme13/bg.gif”,
“profile_background_image_url_https”: “https://abs.twimg.com/images/themes/theme13/bg.gif”,
“profile_background_tile”: false,
“profile_image_url”: “http://pbs.twimg.com/profile_images/1116918074043764736/nLsweU3h_normal.jpg”,
“profile_image_url_https”: “https://pbs.twimg.com/profile_images/1116918074043764736/nLsweU3h_normal.jpg”,
“profile_banner_url”: “https://pbs.twimg.com/profile_banners/27991531/1514749069”,
“profile_link_color”: “4A913C”,
“profile_sidebar_border_color”: “FFFFFF”,
“profile_sidebar_fill_color”: “FFFFFF”,
“profile_text_color”: “333333”,
“profile_use_background_image”: true,
“has_extended_profile”: true,
“default_profile”: false,
“default_profile_image”: false,
“following”: null,
“follow_request_sent”: null,
“notifications”: null,
“translator_type”: “none”
},
“geo”: null,
“coordinates”: null,
“place”: null,
“contributors”: null,
“is_quote_status”: false,
“retweet_count”: 14049,
“favorite_count”: 15352,
“favorited”: false,
“retweeted”: false,
“lang”: “en”
},
“retweet_count”: 270,
“favorite_count”: 506,
“favorited”: false,
“retweeted”: false,
“possibly_sensitive”: false,
“lang”: “en”
},
“is_quote_status”: true,
“quoted_status_id”: 1143653733437444100,
“quoted_status_id_str”: “1143653733437444096”,
“retweet_count”: 270,
“favorite_count”: 0,
“favorited”: false,
“retweeted”: false,
“lang”: “en”
}
regrads,
Sumeet Lalvani