I have data Like this (shown below).
When I run the AWS Glue crawler on this data, I get the error mentioned in the topic name
{
"_id":{
"$binary":{
"base64":"EyYO+A68T2WJm/p+ny+buw==",
"subType":"04"
}
},
"marketIntel":{
"general":{
"sentiment":{
"totalPositiveness":{
"$numberDouble":"0.501"
},
"sentimentBySource":[
{
"countries":[
{
"monthly":[
{
"date":"2018-10",
"negative_count":{
"$numberInt":"0"
},
"positive_count":{
"$numberInt":"1"
}
}
],
"country":"DE",
"positive_polarity":{
"$numberDouble":"0.9865"
}
},
{
"monthly":[
{
"date":"2018-07",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2018-08",
"negative_count":{
"$numberInt":"0"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2018-09",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"0"
}
},
{
"date":"2018-10",
"negative_count":{
"$numberInt":"0"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2018-11",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2018-12",
"negative_count":{
"$numberInt":"0"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2019-01",
"negative_count":{
"$numberInt":"0"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2019-02",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2019-03",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"1"
}
},
{
"date":"2019-04",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"0"
}
},
{
"date":"2019-05",
"negative_count":{
"$numberInt":"1"
},
"positive_count":{
"$numberInt":"1"
}
}
],
"country":"N/A",
"positive_polarity":{
"$numberDouble":"0.479"
}
}
],
"source":"All"
}
],
"lastModified":"2020-06-08 20:10:33.627029",
"totalNegativeCount":{
"$numberInt":"10"
},
"totalPositiveCount":{
"$numberInt":"13"
}
},
"vindowCorporateScore":false,
"vindowScore":{
"$numberDouble":"0.2388"
},
"reviewsCount":{
"$numberInt":"15"
}
},
"sentiment":{
"totalPositiveness":{
"$numberDouble":"0.6771"
},
"sentimentBySource":[
{
"countries":[
{
"country":"N/A",
"monthly":[
{
"date":"2020-02",
"positive_count":{
"$numberInt":"7"
},
"negative_count":{
"$numberInt":"3"
}
},
{
"date":"2020-03",
"positive_count":{
"$numberInt":"1"
},
"negative_count":{
"$numberInt":"1"
}
},
{
"date":"2020-04",
"positive_count":{
"$numberInt":"1"
},
"negative_count":{
"$numberInt":"0"
}
},
{
"date":"2020-09",
"positive_count":{
"$numberInt":"1"
},
"negative_count":{
"$numberInt":"0"
}
},
{
"date":"2020-10",
"positive_count":{
"$numberInt":"1"
},
"negative_count":{
"$numberInt":"0"
}
}
],
"positive_polarity":{
"$numberDouble":"0.6771"
}
}
],
"source":"All"
}
],
"totalNegativeCount":{
"$numberInt":"4"
},
"totalPositiveCount":{
"$numberInt":"11"
},
"lastModified":"2021-02-09 16:10:36.835867"
},
"reviewsBySourceCount":{
"All":{
"$numberInt":"15"
},
"Google":{
"$numberInt":"15"
},
"Booking":{
"$numberInt":"8"
},
"TripAdvisor":{
"$numberInt":"2"
},
"Booking_com":{
"$numberInt":"3"
}
},
"scores":[
{
"source":"All",
"score":{
},
"lastModified":"2021-03-02T20:22:32.181Z",
"vindowScore":{
"$numberDouble":"0.2388"
}
}
],
"recommendation":{
"reducedFeaturesVector":[
{
"$numberDouble":"2.2315"
},
{
"$numberDouble":"14.7779"
}
],
"clusterId":{
"$numberInt":"29"
}
},
"corporateScores":[
{
"source":"All",
"score":{
},
"lastModified":"2021-02-09 16:10:37.204984",
"vindowCorporateScore":false
}
],
"topicAnalysis":[
{
"source":"All",
"negative":[
{
"word":"bed",
"relevance":{
"$numberInt":"1"
},
"phrases":[
"And the bed I had in my room was just... bad."
]
},
{
"word":"room",
"relevance":{
"$numberInt":"1"
},
"phrases":[
"And the bed I had in my room was just... bad."
]
}
],
"positive":[
{
"word":"room",
"relevance":{
"$numberInt":"4"
},
"phrases":[
"The rooms are large.",
"Loved the wine and food.",
"I'm sure that the bed thing can be fixed easily and probably just my room.",
"Great rooms.",
"The view from the window of my room 15 is gorgeous."
]
},
{
"word":"bed",
"relevance":{
"$numberInt":"3"
},
"phrases":[
"It had straight up a hole in the centre of the bed spring and every night felt like sleeping in a hill.",
"I'm sure that the bed thing can be fixed easily and probably just my room.",
"If the beds are fixed in the future I'd be glad to come back !"
]
},
{
"word":"place",
"relevance":{
"$numberInt":"2"
},
"phrases":[
"Perfect place.",
"Perhaps one of the best places on Rubinstein."
]
},
{
"word":"hotel",
"relevance":{
"$numberInt":"2"
},
"phrases":[
"A very good hotel with a very friendly staff.",
"In the end : a very good hotel if you're not on an unlimited budget and want to save."
]
},
{
"word":"staff",
"relevance":{
"$numberInt":"2"
},
"phrases":[
"A very good hotel with a very friendly staff.",
"Location is good staff are nice."
]
},
{
"word":"location",
"relevance":{
"$numberInt":"2"
},
"phrases":[
"Location is good staff are nice.",
"Great location in the city center."
]
},
{
"word":"wine",
"relevance":{
"$numberInt":"1"
},
"phrases":[
"The rooms are large.",
"Loved the wine and food."
]
},
{
"word":"food",
"relevance":{
"$numberInt":"1"
},
"phrases":[
"The rooms are large.",
"Loved the wine and food."
]
},
{
"word":"cleanliness",
"relevance":{
"$numberInt":"1"
},
"phrases":[
"The cleanliness is perfect."
]
},
{
"word":"hole",
"relevance":{
"$numberInt":"1"
},
"phrases":[
"It had straight up a hole in the centre of the bed spring and every night felt like sleeping in a hill."
]
}
],
"lastModified":"2021-02-09 16:10:37.009042"
}
]
},
"distanceToAirports":{
"32d5f078-aa9b-4200-8d2b-128bf4c5dcb3":{
"distanceInMiles":{
"$numberDouble":"12.052283170140424"
},
"drivingTimeInMinutes":{
"$numberDouble":"40.25"
},
"drivingDistanceInMiles":{
"$numberDouble":"21.452833775"
},
"airportId":"32d5f078-aa9b-4200-8d2b-128bf4c5dcb3"
},
"d429db4d-0b1e-41a3-8350-dd5c64cb8c41":{
"distanceInMiles":{
"$numberDouble":"16.88938793551284"
},
"drivingTimeInMinutes":{
"$numberDouble":"42.21666666666667"
},
"drivingDistanceInMiles":{
"$numberDouble":"21.501300713000003"
},
"airportId":"d429db4d-0b1e-41a3-8350-dd5c64cb8c41"
},
"89c8b15f-87d5-41de-89dc-caeab5bab3c9":{
"distanceInMiles":{
"$numberDouble":"9.361319890017072"
},
"drivingTimeInMinutes":{
"$numberDouble":"31.083333333333332"
},
"drivingDistanceInMiles":{
"$numberDouble":"13.494935378000001"
},
"airportId":"89c8b15f-87d5-41de-89dc-caeab5bab3c9"
},
"88c86070-05f3-4c83-8e0d-8fa5437c4018":{
"distanceInMiles":{
"$numberDouble":"41.00256449186227"
},
"drivingTimeInMinutes":{
"$numberDouble":"76.55"
},
"drivingDistanceInMiles":{
"$numberDouble":"50.31862358"
},
"airportId":"88c86070-05f3-4c83-8e0d-8fa5437c4018"
}
},
"hotelId":"13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb",
"foods":{
"restaurants":[
],
"breakfastIds":[
"080d7030-591e-4844-8077-238cd02778ed",
"080d7030-591e-4844-8077-238cd02778ed"
]
},
"active":true,
"expediaId":{
"$numberInt":"18698320"
},
"ntmCode":"EXP-18698320",
"hotelAmenityIds":[
"49f6b262-1b2a-4b7d-85d6-af4d08fb55fc",
"909470da-7cae-490c-849b-47965aef2f44",
"39a61cb3-79f1-4a6e-8627-e951aafb0fcc",
"ad0949dd-c696-4695-8502-19f4ceebb4e3",
"930c10e5-8f52-4154-835c-87319f4113ff",
"c518cc99-911a-47c1-8054-765855c4ba62",
"944bbf7e-8c8d-4afd-8f2f-8dcee8ae972b",
"ae74d71f-38bc-4720-8715-be4a142a1f96",
"cda480a3-407d-41f2-8341-2c6dae7a1bb0",
"9badeede-2a67-4261-85c2-56defcce81ad"
],
"updatedAt":{
"$date":{
"$numberLong":"1678807851107"
}
},
"rooms":{
"amenityIds":[
"3e39107f-398f-4247-88ef-899f2780d20e",
"3aeb7e4e-6127-4105-8364-b6d72d05a60a",
"f6f0c79b-a5a7-4b50-858e-bff7777cc759",
"973bf297-f87d-4781-856d-c922e0ce64bb",
"9871e1d3-34bc-4900-8279-aa89a540fc66",
"40e3709e-fd0a-41ae-8d74-d1cbbfbd7d70",
"12b046db-dca7-4b83-81fe-207594d1da41",
"28641076-8eb6-4fd6-8b01-ca644889447e",
"560f5c2c-2bda-4422-86be-bf3ab3f8fc9b",
"2429f72e-9227-4ce0-8820-fa287ec19f2d",
"f4a81c00-8dfd-497f-8dcc-e680a8c86c65",
"c24f3bc4-df77-424d-8b44-364be4a44a46"
],
"roomTypes":[
]
},
"crawling":{
"googlePlaces":{
"cid":"15923351505077507338",
"lastDateCrawledBySource":{
"tripadvisor":"2022-09-20T03:31:36.780Z",
"priceline":"2022-09-20T03:30:40.754Z",
"agoda":"2022-09-20T03:29:39.112Z",
"booking":"2022-09-20T03:31:36.734Z",
"hotels":"2022-09-20T03:30:33.983Z",
"orbitz":"2022-09-20T03:30:37.895Z",
"marriot":"2022-09-20T03:29:40.412Z",
"travelocity":"2022-09-20T03:30:40.988Z",
"google":"2022-09-20T03:29:39.227Z",
"expedia":"2022-09-20T03:31:36.786Z"
},
"lastReviewIdBySource":{
"google":"105258523960198986909"
},
"reviewsCountBySource":{
"tripadvisor":{
"$numberInt":"0"
},
"priceline":{
"$numberInt":"0"
},
"agoda":{
"$numberInt":"0"
},
"booking":{
"$numberInt":"0"
},
"hotels":{
"$numberInt":"0"
},
"orbitz":{
"$numberInt":"0"
},
"marriot":{
"$numberInt":"0"
},
"travelocity":{
"$numberInt":"0"
},
"google":{
"$numberInt":"1"
},
"expedia":{
"$numberInt":"0"
}
}
},
"tripAdvisor":{
"url":"https://www.tripadvisor.com/Hotel_Review-g298507-d12643830-Reviews-LARGO_Hotel-St_Petersburg_Northwestern_District.html",
"lastDateCrawled":"2021-02-09T14:37:51.199Z",
"lastReviewId":{
"$numberInt":"624661546"
},
"reviewsCount":{
"$numberInt":"2"
}
}
},
"images":{
"41d05eee-4dfe-4a11-8e66-5b6aea5a0589":{
"title":"Coffee and/or Coffee Maker",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/41d05eee-4dfe-4a11-8e66-5b6aea5a0589.jpg",
"imageId":"41d05eee-4dfe-4a11-8e66-5b6aea5a0589"
},
"abd4fd88-bfcf-43d6-8c84-fa0849299e91":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/abd4fd88-bfcf-43d6-8c84-fa0849299e91.jpg",
"imageId":"abd4fd88-bfcf-43d6-8c84-fa0849299e91"
},
"a026b9fb-a2f2-4f5f-832c-c5dc5394f5a6":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/a026b9fb-a2f2-4f5f-832c-c5dc5394f5a6.jpg",
"imageId":"a026b9fb-a2f2-4f5f-832c-c5dc5394f5a6"
},
"9f8c941d-5d4e-4547-8ee9-354ef4644137":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/9f8c941d-5d4e-4547-8ee9-354ef4644137.jpg",
"imageId":"9f8c941d-5d4e-4547-8ee9-354ef4644137"
},
"55c81c75-6999-4ad8-8c47-f2ee91e93c3c":{
"title":"Breakfast Area",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/55c81c75-6999-4ad8-8c47-f2ee91e93c3c.jpg",
"imageId":"55c81c75-6999-4ad8-8c47-f2ee91e93c3c"
},
"4d1f2ad1-6428-4f35-85ef-375cbbd5e5cc":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/4d1f2ad1-6428-4f35-85ef-375cbbd5e5cc.jpg",
"imageId":"4d1f2ad1-6428-4f35-85ef-375cbbd5e5cc"
},
"b0352a6b-ccd8-43c8-83a8-9593c247123d":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/b0352a6b-ccd8-43c8-83a8-9593c247123d.jpg",
"imageId":"b0352a6b-ccd8-43c8-83a8-9593c247123d"
},
"c36ae604-bcf4-4a6a-8ac0-4d664e31c510":{
"title":"View from Property",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/c36ae604-bcf4-4a6a-8ac0-4d664e31c510.jpg",
"imageId":"c36ae604-bcf4-4a6a-8ac0-4d664e31c510"
},
"db3ce258-5f4b-4e22-8a77-4c0d932e1853":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/db3ce258-5f4b-4e22-8a77-4c0d932e1853.jpg",
"imageId":"db3ce258-5f4b-4e22-8a77-4c0d932e1853"
},
"fafc74d5-f413-4b42-8f2c-ce82ea6ee2f8":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/fafc74d5-f413-4b42-8f2c-ce82ea6ee2f8.jpg",
"imageId":"fafc74d5-f413-4b42-8f2c-ce82ea6ee2f8"
},
"e059dda4-ef7b-4132-8b80-6f6f9a38b116":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/e059dda4-ef7b-4132-8b80-6f6f9a38b116.jpg",
"imageId":"e059dda4-ef7b-4132-8b80-6f6f9a38b116"
},
"575b6ce4-e4d5-4875-8a95-b6170b411804":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/575b6ce4-e4d5-4875-8a95-b6170b411804.jpg",
"imageId":"575b6ce4-e4d5-4875-8a95-b6170b411804"
},
"14811ac0-a6ce-43e0-8dd0-dff22b8ed18b":{
"title":"Bathroom",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/14811ac0-a6ce-43e0-8dd0-dff22b8ed18b.jpg",
"imageId":"14811ac0-a6ce-43e0-8dd0-dff22b8ed18b"
},
"d292bf33-36ca-460c-8624-bc5c906aa23a":{
"title":"Front of Property",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/d292bf33-36ca-460c-8624-bc5c906aa23a.jpg",
"imageId":"d292bf33-36ca-460c-8624-bc5c906aa23a"
},
"3de7facb-cfa7-4dbf-801e-d7524841c082":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/3de7facb-cfa7-4dbf-801e-d7524841c082.jpg",
"imageId":"3de7facb-cfa7-4dbf-801e-d7524841c082"
},
"9b92c046-2702-4eae-8e01-8d2e2ba7c236":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/9b92c046-2702-4eae-8e01-8d2e2ba7c236.jpg",
"imageId":"9b92c046-2702-4eae-8e01-8d2e2ba7c236"
},
"99264784-0c4a-4400-8783-9236e661ac69":{
"title":"Featured Image",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/99264784-0c4a-4400-8783-9236e661ac69.jpg",
"imageId":"99264784-0c4a-4400-8783-9236e661ac69"
},
"cb0269c7-c406-4d4f-898b-173e0e3577b1":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/cb0269c7-c406-4d4f-898b-173e0e3577b1.jpg",
"imageId":"cb0269c7-c406-4d4f-898b-173e0e3577b1"
},
"e4a4d44a-fcc8-4a62-87c4-c863dbdd1b8e":{
"title":"Property Entrance",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/e4a4d44a-fcc8-4a62-87c4-c863dbdd1b8e.jpg",
"imageId":"e4a4d44a-fcc8-4a62-87c4-c863dbdd1b8e"
},
"6ff14e0f-b130-46a6-813b-2d040c323c22":{
"title":"Property Entrance",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/6ff14e0f-b130-46a6-813b-2d040c323c22.jpg",
"imageId":"6ff14e0f-b130-46a6-813b-2d040c323c22"
},
"9609cc92-dc9b-485f-8831-6edd6b4ef268":{
"title":"Room",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/9609cc92-dc9b-485f-8831-6edd6b4ef268.jpg",
"imageId":"9609cc92-dc9b-485f-8831-6edd6b4ef268"
},
"637e78f4-c7d2-4ce4-8dd8-7ac4696f6326":{
"title":"Breakfast Area",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/637e78f4-c7d2-4ce4-8dd8-7ac4696f6326.jpg",
"imageId":"637e78f4-c7d2-4ce4-8dd8-7ac4696f6326"
},
"c5f16d1b-f750-4705-8a03-0b809df4ef4a":{
"title":"Lobby Lounge",
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/c5f16d1b-f750-4705-8a03-0b809df4ef4a.jpg",
"imageId":"c5f16d1b-f750-4705-8a03-0b809df4ef4a"
}
},
"general":{
"crewFriendly":{
"crewInHouse":false,
"hadCrewRecently":false
},
"thumbnail":{
"fileName":"/hotels/13260ef8-0ebc-4f65-899b-fa7e9f2f9bbb/images/thumbnail.jpg",
"imageId":"thumbnail"
},
"address":{
"address":"Ulitsa Rubinshteina 26",
"cityId":"russia__saintpetersburg__saintpetersburg",
"countryId":"russia",
"stateId":"russia__saintpetersburg"
},
"totalNumberOfRooms":{
"$numberInt":"15"
},
"phone":"+7 812 438-03-31",
"rating":{
"$numberInt":"3"
},
"location":{
"lat":{
"$numberDouble":"59.929426"
},
"lon":{
"$numberDouble":"30.344232"
}
},
"hotelName":"Largo hotel",
"standardCheckOutTime":"1200",
"hotelDescription":"The hotel offers a coffee shop/café. A complimentary breakfast is offered each morning. Wireless Internet access is complimentary. For a surcharge, an airport shuttle (available 24 hours) is offered to guests. This bu...",
"standardCheckInTime":"1400",
"brandId":"822c07db-57d9-4f8c-9588-e2c03795eb75"
},
"vindow15Id":{
"$numberInt":"379583"
},
"updatedBy":"SCRIPT-V20T-8310",
"hotelOpenStatus":"Open",
"vervotechId":{
"$numberInt":"32363897"
},
"vervotechModifiedAt":{
"$date":{
"$numberLong":"1650514753000"
}
},
"vervotechIdUpdatedAt":{
"$date":{
"$numberLong":"1671625756523"
}
},
"primero":{
},
"externalIds":{
"seqId":{
"$numberInt":"1"
}
}
}
The error I get:
PlanExecutor error during aggregation:: caused by:: Sort exceeded memory limit of 104857600 bytes, but did not opt into external sorting. Aborting operation. Pass allowDiskUse: true to opt-in.
Since I am running an AWS Crawler using AWS’s inbuilt MongoDB connector, I am not sure where I will add allowDiskUse: true. It would be very helpful if I can get some input on how to fix this issue.
Looking forward to your response. Thank you in Advance!
Best,
Prasanna
