I have attempted to write a library to convert JSON Patch operations into Mongo Queries. At face value this appears to be very straight forward but I have learned there are a lot of easy JSON Patches that become quite complex on the Mongo side.
I am curious if anyone has had better luck or if there are any libraries that have solved this.
What would be nice is that you first share what you
Sharing what you
could be nice too. Knowing what you find complex could help us. It looks like there is a good resource directory on jsonpatch.com.
It is indeed pretty straight-forward! I have written some TypeScript code to convert JSON Patch to MongoDB operations that can be written in bulk: fathomas/json-patch-to-mongodb-ops
The JSON Patch operations that manipulate arrays start to get real tricky. I also had the desire to have all JSON Patches be performed in a single mongo query, didnt want to deal with transactions slowing things down.
here is the converter class I had working…
class JsonPatchOperationsConverterUtilities {
static pathToDotNotation(path) {
if (!path) {
return '';
}
const dotNotation =
path.replace(/^\//, '')
.replace(/\//g, '.')
.replace(/~1/g, '/')
.replace(/~0/g, '~');
return dotNotation;
}
static convertToMongoUpdate(jsonPatchOperations) {
function createMongoArrayFunctionQuery(targetLocation, statements) {
if (!Object.prototype.toString.call(statements), `[object Array]`) {
statements = [statements];
}
statements.push('return array;');
const mongoFn = new Function('array', statements.join('\n'));
return {
$function: {
body: mongoFn,
args: [targetLocation],
lang: "js"
}
}
}
function convertMongoPushQueryToArrayFunctionQuery(targetLocation, mongoQuery) {
mqlUpdate.$set = mqlUpdate.$set || {};
const startingIndex = mongoQuery.$push[targetLocation].$position;
if (startingIndex === undefined) {
//TODO wont work if $set is already set
mongoQuery.$set[targetLocation] = createMongoArrayFunctionQuery(targetLocation, [
`array.push(...${JSON.stringify(mongoQuery.$push[targetLocation].$each)});`
]);
}
else {
const eachAsArgs = JSON.stringify(mongoQuery.$push[targetLocation].$each).slice(1, -1);
//TODO wont work if $set is already set
mongoQuery.$set[targetLocation] = createMongoArrayFunctionQuery(targetLocation, `array.splice(${startingIndex},0,${eachAsArgs});`);
}
delete mongoQuery.$push[targetLocation];
if (Object.keys(mongoQuery.$push).length === 0) {
delete mongoQuery.$push;
}
}
function appendStatementToMongoArrayFunctionQuery(mongoFnQuery, statement) {
const mongoFn = mongoFnQuery.body;
let currentMongoFnStr = mongoFn.toString()
.replace('function anonymous(array\n) {\n', '')
.replace('return array;\n}', '');
currentMongoFnStr += statement;
currentMongoFnStr += `\nreturn array;`;
mongoFnQuery.body = new Function('array', currentMongoFnStr);
return mongoFnQuery;
}
let mqlUpdate = {};
for (const operation of jsonPatchOperations) {
const path = JsonPatchOperationsConverterUtilities.pathToDotNotation(operation.path);
const pathParts = path.split('.');
const targetLocation = pathParts.slice(0, -1).join('.');
const lastPathPart = pathParts[pathParts.length - 1];
const position = Number.parseInt(lastPathPart);
switch (operation.op) {
//Add Operation
//See https://datatracker.ietf.org/doc/html/rfc6902#section-4.1
case 'add':
//Append to array
//You can provide an '-' this indicates the targetLocation is an array and that you want to "append" to that array
if (lastPathPart === '-') {
//if we have found non contiguous adds and are now using $function...
if (mqlUpdate?.$set?.[targetLocation]?.hasOwnProperty('$function')) {
appendStatementToMongoArrayFunctionQuery(mqlUpdate.$set[targetLocation].$function, `array.push(${JSON.stringify(operation.value)});`);
}
else {
mqlUpdate.$push = mqlUpdate.$push || {};
if (mqlUpdate.$push[targetLocation]?.$position !== undefined) {
convertMongoPushQueryToArrayFunctionQuery(targetLocation, mqlUpdate);
appendStatementToMongoArrayFunctionQuery(mqlUpdate.$set[targetLocation].$function, `array.push(${JSON.stringify(operation.value)});`);
}
if (!mqlUpdate.$push?.hasOwnProperty(targetLocation)) {
mqlUpdate.$push = mqlUpdate.$push || {};
mqlUpdate.$push[targetLocation] = {
$each: [operation.value]
};
}
else {
mqlUpdate.$push[targetLocation].$each = [...mqlUpdate.$push[targetLocation].$each, operation.value];
}
}
}
// Insert in array
//If the last path part is a number we will assume you are inserting into an array
//If you are trying to add to property that is a number then this is not going to work
else if (!Number.isNaN(position)) {
if (mqlUpdate?.$set?.[targetLocation]?.hasOwnProperty('$function')) {
appendStatementToMongoArrayFunctionQuery(mqlUpdate.$set[targetLocation].$function, `array.splice(${position},0,${JSON.stringify(operation.value)});`);
}
else {
//If $push with position
if (mqlUpdate?.$push?.[targetLocation]?.hasOwnProperty('$position')) {
const posDiff = position - mqlUpdate.$push[targetLocation].$position;
//If this add forces us to become not contiguous, switch over to using $function
if (posDiff > mqlUpdate.$push[targetLocation]?.$each?.length) {
convertMongoPushQueryToArrayFunctionQuery(targetLocation, mqlUpdate);
}
else {
mqlUpdate.$push[targetLocation].$each.splice(posDiff, 0, operation.value);
mqlUpdate.$push[targetLocation].$position = Math.min(position, mqlUpdate.$push[targetLocation].$position);
}
}
//If $push without position
else if (mqlUpdate?.$push?.hasOwnProperty(targetLocation)) {
convertMongoPushQueryToArrayFunctionQuery(targetLocation, mqlUpdate);
}
//No $push yet exists
else {
mqlUpdate.$push = mqlUpdate.$push || {};
if (!mqlUpdate.$push[targetLocation]) {
mqlUpdate.$push[targetLocation] = {
$each: [operation.value],
$position: position
};
}
}
//if we have found non contiguous adds and are now using $function...
if (mqlUpdate?.$set?.[targetLocation]?.hasOwnProperty('$function')) {
appendStatementToMongoArrayFunctionQuery(mqlUpdate.$set[targetLocation].$function, `array.splice(${position},0,${JSON.stringify(operation.value)});`);
}
}
}
//Add with no position or '-' is basically a replace
else {
//TODO wont work if $set is already set
mqlUpdate.$set = mqlUpdate.$set || {};
mqlUpdate.$set[path] = operation.value;
}
break;
case 'remove':
// Remove element from an array
//If the last path part is a number we will assume you are removing an element from an array
if (!Number.isNaN(position)) {
mqlUpdate.$set = {};
mqlUpdate.$set[targetLocation] = createMongoArrayFunctionQuery(targetLocation, [
`array.splice(${position}, 1);`
]);
}
else {
mqlUpdate.$unset = mqlUpdate.$unset || {};
mqlUpdate.$unset[path] = 1;
}
break;
case 'replace':
mqlUpdate.$set = mqlUpdate.$set || {};
mqlUpdate.$set[path] = operation.value;
break;
case 'move':
//Note: will overwrite any previous renames
mqlUpdate.$rename = mqlUpdate.$rename || {};
const moveFrom = JsonPatchOperationsConverterUtilities.pathToDotNotation(operation.from);
const moveTo = path;
mqlUpdate.$rename[moveFrom] = moveTo;
break;
case 'copy':
const copyFrom = JsonPatchOperationsConverterUtilities.pathToDotNotation(operation.from);
const copyTo = path;
mqlUpdate.$set = mqlUpdate.$set || {};
mqlUpdate.$set[copyTo] = {
$function:
{
body: function (fromField) {
return fromField;
},
args: [copyFrom],
lang: "js"
}
}
break;
case 'test':
//test operations have no translation to a mongo query
break;
default:
throw new Error('Unsupported Operation! op = ' + operation.op);
}
}
const queryContainsFunction = JSON.stringify(mqlUpdate).match(/\$function/);
if (queryContainsFunction !== null) {
mqlUpdate = [mqlUpdate];
}
return mqlUpdate;
}
}
export { JsonPatchOperationsConverterUtilities };
export default JsonPatchOperationsConverterUtilities;
`
And here is the test suite:
import { strict as assert } from "node:assert";
import { JsonPatchOperationsConverterUtilities } from "../index.mjs";
describe("JsonPatchOperationsConverter Utilities", async function () {
it('should handle single add operations', function () {
var patches = [{
op: 'add',
path: '/name/-',
value: 'jake'
}];
var expected = {
$push: {
name: {
$each: ['jake']
}
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle multiple add operations', function () {
var patches = [
{
op: 'add',
path: '/name/-',
value: 'jake'
},
{
op: 'add',
path: '/name/-',
value: 'johnny'
},
{
op: 'add',
path: '/name/-',
value: 'jeff'
}
];
var expected = {
$push: {
name: {
$each: ['jake', 'johnny', 'jeff']
}
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle escaped characters operations', function () {
var patches = [{
op: 'replace',
path: '/foo~1bar~0',
value: 'jake'
}];
var expected = {
$set: {
"foo/bar~": 'jake'
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle array add with position operations', function () {
var patches = [{
op: 'add',
path: '/name/1',
value: 'jake'
}];
var expected = {
$push: {
name: {
$each: [
'jake'
],
$position: 1
}
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle multiple add with position operations', function () {
var patches = [{
op: 'add',
path: '/name/1',
value: 'jake'
}, {
op: 'add',
path: '/name/2',
value: 'bob'
}, {
op: 'add',
path: '/name/2',
value: 'john'
}];
var expected = {
$push: {
name: {
$each: [
'jake',
'john',
'bob'
],
$position: 1
}
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle multiple adds in reverse position operations', function () {
var patches = [{
op: 'add',
path: '/name/1',
value: 'jake'
}, {
op: 'add',
path: '/name/1',
value: 'bob'
}, {
op: 'add',
path: '/name/1',
value: 'john'
}];
var expected = {
$push: {
name: { $each: ['john', 'bob', 'jake'], $position: 1 }
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle multiple add - operations', function () {
var patches = [{
op: 'add',
path: '/name/-',
value: 'jake'
}, {
op: 'add',
path: '/name/-',
value: 'bob'
}, {
op: 'add',
path: '/name/-',
value: 'john'
}];
var expected = {
$push: {
name: { $each: ['jake', 'bob', 'john'] }
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle multiple adds with some null at the end operations', function () {
var patches = [{
op: 'add',
path: '/name/-',
value: null
}, {
op: 'add',
path: '/name/-',
value: 'bob'
}, {
op: 'add',
path: '/name/-',
value: null
}];
var expected = {
$push: {
name: { $each: [null, 'bob', null] }
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle multiple adds with some null and position operations', function () {
var patches = [{
op: 'add',
path: '/name/1',
value: null
}, {
op: 'add',
path: '/name/1',
value: 'bob'
}, {
op: 'add',
path: '/name/1',
value: null
}];
var expected = {
$push: {
name: { $each: [null, 'bob', null], $position: 1 }
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle remove operations', function () {
var patches = [{
op: 'remove',
path: '/name',
value: 'jake'
}];
var expected = {
$unset: {
name: 1
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle replace operations', function () {
var patches = [{
op: 'replace',
path: '/name',
value: 'jake'
}];
var expected = {
$set: {
name: 'jake'
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should handle test operations', function () {
var patches = [{
op: 'test',
path: '/name',
value: 'jake'
}];
var expected = {};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('more than 2 adds with non contiguous positions will be converted to a $function', function () {
var patches = [
{
op: 'add',
path: '/name/0',
value: 'bob0'
},
{
op: 'add',
path: '/name/1',
value: 'bob1'
}, {
op: 'add',
path: '/name/3',
value: 'bob3'
}];
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches)
const testFunction = result[0].$set.name.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction([]), ['bob0', 'bob1', 'bob3']);
assert.deepEqual(testFunction([1, 2, 3, 4]), ['bob0', 'bob1', 1, 'bob3', 2, 3, 4]);
});
it('should support 2 adds with non contiguous positions and then an append', function () {
var patches = [
{
op: 'add',
path: '/name/0',
value: 'bob0'
},
{
op: 'add',
path: '/name/3',
value: 'bob1'
},
{
op: 'add',
path: '/name/-',
value: 'bob2'
}];
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches)
const testFunction = result[0].$set.name.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction([]), ['bob0', 'bob1', 'bob2']);
assert.deepEqual(testFunction([1, 2, 3, 4]), ['bob0', 1, 2, 'bob1', 3, 4, 'bob2']);
});
it('should support 3 adds with non contiguous positions', function () {
var patches = [
{
op: 'add',
path: '/name/0',
value: 'bob0'
},
{
op: 'add',
path: '/name/2',
value: 'bob1'
},
{
op: 'add',
path: '/name/4',
value: 'bob2'
}];
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches)
const testFunction = result[0].$set.name.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction([]), ['bob0', 'bob1', 'bob2']);
assert.deepEqual(testFunction([1, 2, 3, 4]), ['bob0', 1, 'bob1', 2, 'bob2', 3, 4]);
});
it('adds with positions and adds with appends are supported', function () {
var patches = [{
op: 'add',
path: '/name/0',
value: 'bob'
}, {
op: 'add',
path: '/name/-',
value: 'john'
}];
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches)
const testFunction = result[0].$set.name.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction([]), ['bob', 'john']);
assert.deepEqual(testFunction([1, 2, 3, 4]), ['bob', 1, 2, 3, 4, 'john']);
});
it('adds with first appends and then position based', function () {
var patches = [{
op: 'add',
path: '/name/-',
value: 'bob'
}, {
op: 'add',
path: '/name/0',
value: 'john'
}];
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
const testFunction = result[0].$set.name.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction([]), ['john', 'bob']);
assert.deepEqual(testFunction([1, 2, 3, 4]), ['john', 1, 2, 3, 4, 'bob']);
});
it('should blow up on add without position', function () {
var patches = [{
op: 'add',
path: '/name',
value: 'jake'
}];
var expected = {
$set: {
name: 'jake'
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should support move operation', function () {
var patches = [{
op: 'move',
path: '/name',
from: '/old_name'
}];
var expected = {
$rename: {
old_name: 'name'
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should support nested move operation', function () {
var patches = [{
op: 'move',
path: '/nested/name',
from: '/old/name'
}];
var expected = {
$rename: {
"old.name": 'nested.name'
}
};
const mql = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
assert.deepEqual(mql, expected);
});
it('should support the copy operation', function () {
var patches = [{
op: 'copy',
path: '/name',
from: '/old_name'
}];
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
const testFunction = result[0].$set.name.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction(42), 42);
assert.deepEqual(result[0].$set.name.$function.args[0], 'old_name');
});
it('should blow up on unknown operation', function () {
var patches = [{
op: 'foogasha'
}];
assert.throws(
() => { JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches) },
{ name: "Error", message: "Unsupported Operation! op = foogasha" }
);
});
it('should handle removing element of an array', function () {
var patches = [{ op: 'remove', path: '/onBlock/0' }];
var expected = {
$set: {
"onBlock": {
$function: {
body: function (array) { array.splice(0, 1); return array; },
args: ["onBlock"],
lang: "js"
}
}
}
};
const result = JsonPatchOperationsConverterUtilities.convertToMongoUpdate(patches);
const testFunction = result[0].$set.onBlock.$function.body;
assert.deepEqual(Object.prototype.toString.call(testFunction), `[object Function]`);
assert.deepEqual(testFunction([0, 1, 2]), [1, 2]);
});
});