Error While Load the data from Postgresql to Apache Druid through Spec

Hi Team,

I am trying to load the data from PostgreSQL to Druid through Spec,but the task is getting failed with the below error as :

SQL firehose error with org.apache.druid.segment.realtime.firehose.SqlFirehoseFactory cannot be cast to org.apache.druid.data.input.FiniteFirehoseFactory.

Request you to please check for the above error/Issue.

Below the sample Spec using to load data from Postgresql to Druid:

{

“type”: “index”,

“spec”: {

"dataSchema": {

  "dataSource": "Emp_Inc_load",

  "parser": {

    "type": "map",

    "parseSpec": {

     "format": "timeAndDims",

      "dimensionsSpec": {

      "dimensionExclusions": [],

      "dimensionsSpec": [

“emp_name”,

“adress”,

“emp_id”,

“sal”,

“phone”,

“doj”,

“last_update_date”,

“flag”,

“cdc_date”

]

      },



      "timestampSpec": {

        "format": "auto",

        "column": "last_update_date" 

      }

    }

  },

  "metricsSpec": [],

“granularitySpec”: {

    "type": "uniform",

    "segmentGranularity": "all",

    "queryGranularity": {

      "type": "none"

    },

    "rollup": false,

    "intervals": null

  },

  "transformSpec": {

    "filter": null,

    "transforms": []

  }

},

"ioConfig": {

  "type": "index",

  "firehose": {

    "type": "sql",

    "database": {

      "type": "postgresql",

      "connectorConfig": {

       "connectURI": "jdbc:postgresql://<Hostname>:<Port No.>/<Database name>",

“user”: “”,

        "password": "<DB Password>"

         }

    },

    "sqls": [

      "select * from Table;"

    ]

  }

},

“intervals”: {

  "type": "intervals",         

  "intervals": null

   },            

“virtualColumns”: ,

"resultFormat": "compactedList",            

"batchSize": 20480,         

"limit": 50, 

"context": {        

"timeout": 36000000           

},

"tuningconfig": {

  "type": "index"

}

}

}

Could you please help us to find the resolution for the above error .

Hi Moushmi,

It seems SQL firehose doesn’t actually work well in general. I would recommend exporting tables data to CSV /JSON and ingest into druid using general indexing methods.

Thanks and Regards,

Vaibhav