Tranquility flattenSpec not working as documented

I am using tranquility HTTP server with Druid 0.13. Its generally working except flattenSpec doesn’t do what it should.

As you can see below, I have created a dataSource tsrc. I then write something with a nested json, and a flattenSpec to un-nest. I end up with a value for ‘host’ (‘me’), but no value for ‘action’. I don’t see any errors.

This is tranquility 0.8.3

Any suggestions on what am i doing wrong?

import datetime
import requests

msg = {
    "date": datetime.datetime.utcnow().isoformat() + 'Z',
    "host": "me",
    "values": {
     "action": "allow",
     "dummy": "foobar"
    }
}

auth = requests.auth.HTTPBasicAuth('USER','PASS')
r = requests.post('https://tranquility.DOMAIN/v1/post/tsrc', auth=auth, json=msg)
print(r.status_code)

    "dataSources": [
    {
      "spec" : {
        "dataSchema" : {
          "dataSource" : "tsrc",
          "parser" : {
            "type" : "string",
            "parseSpec" : {
              "format" : "json",
              "flattenSpec": {
                "useFieldDiscovery": true,
                "fields": [
                  { "type": "root", "name": "host", "expr": "dummy" },
                  { "type": "path", "name": "action", "expr": "$.values.action" }
                ]
              },
              "dimensionsSpec" : {
                "dimensions" : ["host","action"],
                "dimensionsExclusions": []
              },
              "timestampSpec": {
                "column": "date",
                "format": "auto"
              }
            }
          },
          "metricsSpec" : [],
          "granularitySpec" : {
            "type" : "uniform",
            "segmentGranularity" : "day",
            "queryGranularity" : "none",
            "rollup" : false
          }
        },
        "tuningConfig": {
          "type": "realtime",
          "intermediatePersistPeriod": "PT15M",
          "windowPeriod": "PT15M"
        }
      },
      "properties" : {
        "task.partitions" : "1",
        "task.replicants" : "1"
      }
    }
]

Looking on ‘middleManager’ in the log, the ‘flattenSpec’ section is missing. Is this expected? E.g. the original spec to tranqulity was logged out from middle manager as :

{

“type” : “index_realtime”,

“id” : “index_realtime_tsrc_2019-02-03T00:00:00.000Z_0_0”,

“resource” : {

"availabilityGroup" : "tsrc-2019-02-03T00:00:00.000Z-0000",                                                                                                          

"requiredCapacity" : 1                                                                                                                                               

},

“spec” : {

"dataSchema" : {                                                                                                                                                     

  "dataSource" : "tsrc",                                                                                                                                             

  "parser" : {                                                                                                                                                       

    "type" : "map",                                                                                                                                                  

    "parseSpec" : {                                                                                                                                                  

      "format" : "json",                                                                                                                                             

      "timestampSpec" : {                                                                                                                                            

        "column" : "date",                                                                                                                                           

        "format" : "millis",                                                                                                                                         

        "missingValue" : null                                                                                                                                        

      },                                                                                                                                                             

      "dimensionsSpec" : {                                                                                                                                           

        "dimensions" : [ "host", "action" ],                                                                                                                         

        "spatialDimensions" : [ ]                                                                                                                                    

      }                                                                                                                                                              

    }                                                                                                                                                                

  },                                                                                                                                                                 

  "metricsSpec" : [ ],                                                                                                                                               

  "granularitySpec" : {                                                                                                                                              

    "type" : "uniform",                                                                                                                                              

    "segmentGranularity" : "DAY",                                                                                                                                    

    "queryGranularity" : {                                                                                                                                           

      "type" : "none"                                                                                                                                                

    },                                                                                                                                                               

    "rollup" : false,                                                                                                                                                

    "intervals" : null                                                                                                                                               

  },                                                                                                                                                                 

  "transformSpec" : {                                                                                                                                                

    "filter" : null,                                                                                                                                                 

    "transforms" : [ ]                                                                                                                                               

  }                                                                                                                                                                  

},                                                                                                                                                                   

"ioConfig" : {                                                                                                                                                       

  "type" : "realtime",                                                                                                                                               

  "firehose" : {                                                                                                                                                     

    "type" : "clipped",                                                                                                                                              

    "delegate" : {                                                                                                                                                   

      "type" : "timed",                                                                                                                                              

      "delegate" : {                                                                                                                                                 

        "type" : "receiver",                                                                                                                                         

        "serviceName" : "firehose:druid:overlord:tsrc-003-0000-0000",                                                                                                

        "bufferSize" : 100000,                                                                                                                                       

        "maxIdleTime" : 9223372036854775807                                                                                                                          

      },                                                                                                                                                             

      "shutoffTime" : "2019-02-04T00:20:00.000Z"                                                                                                                     

    },                                                                                                                                                               

    "interval" : "2019-02-03T00:00:00.000Z/2019-02-04T00:00:00.000Z"                                                                                                 

  },                                                                                                                                                                 

  "firehoseV2" : null                                                                                                                                                

},                                                                                                                                                                   

"tuningConfig" : {                                                                                                                                                   

  "type" : "realtime",                                                                                                                                               

  "maxRowsInMemory" : 75000,                                                                                                                                         

  "intermediatePersistPeriod" : "PT15M",                                                                                                                             

  "windowPeriod" : "PT15M",                                                                                                                                          

  "basePersistDirectory" : "/opt/apache-druid-0.13.0-incubating-SNAPSHOT/var/tmp/1549229845092-0",                                                                   

  "versioningPolicy" : {                                                                                                                                             

    "type" : "intervalStart"                                                                                                                                         

  },                                                                                                                                                                 

  "rejectionPolicy" : {                                                                                                                                              

    "type" : "none" 

  },                                                                                                                                                                 

  "maxPendingPersists" : 0,                                                                                                                                          

  "shardSpec" : {                                                                                                                                                    

    "type" : "linear",                                                                                                                                               

    "partitionNum" : 0                                                                                                                                               

  },                                                                                                                                                                 

  "indexSpec" : {                                                                                                                                                    

    "bitmap" : {                                                                                                                                                     

      "type" : "concise"                                                                                                                                             

    },                                                                                                                                                               

    "dimensionCompression" : "lz4",                                                                                                                                  

    "metricCompression" : "lz4",                                                                                                                                     

    "longEncoding" : "longs"                                                                                                                                         

  },                                                                                                                                                                 

  "buildV9Directly" : true,                                                                                                                                          

  "persistThreadPriority" : 0,                                                                                                                                       

  "mergeThreadPriority" : 0,                                                                                                                                         

  "reportParseExceptions" : false,                                                                                                                                   

  "handoffConditionTimeout" : 0,                                                                                                                                     

  "alertTimeout" : 0,                                                                                                                                                

  "segmentWriteOutMediumFactory" : null,                                                                                                                             

  "dedupColumn" : null                                                                                                                                               

}                                                                                                                                                                    

},

“context” : { },

“groupId” : “index_realtime_tsrc”,

“dataSource” : “tsrc”

}