I am reading data from mongodb using pyspark my job is failing with bson error.
spark version - 2.4.3
jars - mongo-java-driver-3.12.11, mongo-spark-connector_2.12-2.4.2
action - reading data from mongodb and writing into s3 bucket , simple read and write operation.
Error:
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.bson.BsonInvalidOperationException: Invalid state INITIAL
at org.bson.json.StrictCharacterStreamJsonWriter.checkState(StrictCharacterStreamJsonWriter.java:395)
at org.bson.json.StrictCharacterStreamJsonWriter.writeNull(StrictCharacterStreamJsonWriter.java:192)
at org.bson.json.JsonNullConverter.convert(JsonNullConverter.java:24)
at org.bson.json.JsonNullConverter.convert(JsonNullConverter.java:21)
at org.bson.json.JsonWriter.doWriteNull(JsonWriter.java:206)
at org.bson.AbstractBsonWriter.writeNull(AbstractBsonWriter.java:557)
at org.bson.codecs.BsonNullCodec.encode(BsonNullCodec.java:38)
at org.bson.codecs.BsonNullCodec.encode(BsonNullCodec.java:28)
at org.bson.codecs.EncoderContext.encodeWithChildContext(EncoderContext.java:91)
at org.bson.codecs.BsonValueCodec.encode(BsonValueCodec.java:62)
at com.mongodb.spark.sql.BsonValueToJson$.apply(BsonValueToJson.scala:29)
at com.mongodb.spark.sql.MapFunctions$.bsonValueToString(MapFunctions.scala:103)
at com.mongodb.spark.sql.MapFunctions$.com$mongodb$spark$sql$MapFunctions$$convertToDataType(MapFunctions.scala:78)
at com.mongodb.spark.sql.MapFunctions$$anonfun$3.apply(MapFunctions.scala:39)
at com.mongodb.spark.sql.MapFunctions$$anonfun$3.apply(MapFunctions.scala:37)