Create Map Object From Elements JavaScript Add Or Remove Values

I have objects

let obj1 = { categoryId:1, category:"Fruits", name:"Orange"}
let obj2 = { categoryId:1, category:"Fruits",name:"Apple"}
let obj3 = { categoryId:2, category:"Vegetables", name:"Onion"} 
let obj4 = { categoryId:2, category:"Vegetables", name:"Ginger"}....etc

I want to create a map from this array expected:

{
  "Fruits": [{
    "categoryId": 1,
    "category": "Fruits",
    "name": "Orange"
  }, {
    "categoryId": 1,
    "category": "Fruits",
    "name": "Apple"
  }],
  "Vegetables": [{
    "categoryId": 2,
    "category": "Vegetables",
    "name": "Onion"
  }, {
    "categoryId": 2,
    "category": "Vegetables",
    "name": "Ginger"
  }]
}

I am looking for a function to add to the map one by one

addtoMap( obj1);
addtoMap( obj2); 
addtoMap( obj3); 
addtoMap( obj4);....etc 

Also remove from the map

removeFromMap( obj1);

I have tried

addtoMap(xs, key) {
  return xs.reduce(function(rv, x) {
    (rv[x[key]] = rv[x[key]] || []).push(x);
    return rv;
  }, {});
};

You could check if the object exists and add the object in the wanted category. The same goes for deleting an object, until no more objects are in the group. Then the group gets removed as well.

function addToCollection(collection, object) {
    if (!collection[object.category]) {
        collection[object.category] = [object];
        return;
    }
    if (collection[object.category].includes(object)) return;
    collection[object.category].push(object);
}

function removeFromCollection(collection, object) {
    if (!collection[object.category]) return;
    var index = collection[object.category].indexOf(object);
    if (index === -1) return;
    collection[object.category].splice(index, 1);
    if (!collection[object.category].length) delete collection[object.category];
}

var obj1 = { categoryId: 1, category: "Fruits", name:"Orange" },
    obj2 = { categoryId: 1, category: "Fruits", name:"Apple" },
    obj3 = { categoryId: 2, category: "Vegetables", name:"Onion" },
    obj4 = { categoryId: 2, category: "Vegetables", name:"Ginger" },
    collection = {};


addToCollection(collection, obj1);
addToCollection(collection, obj2);
addToCollection(collection, obj3);
addToCollection(collection, obj4);
console.log(collection);
removeFromCollection(collection, obj2);
console.log(collection);
removeFromCollection(collection, obj1);
console.log(collection);