How To Improve Or Avoid Find / Fetch Cycle In Meteor's Publication?

TL;DR: Chat is one collection. ChatMess another one that has messages refering to a Chat's _id. How do I get the last messages from a list of chats with the less computation possib

Solution 1:

Have you considered using the reywood/publishComposite package? With this package you can publish related data in the same method without having to do a bunch of logic to get the correct data published.

The below code should get you started:

Meteor.publishComposite("publishNewChat", function() {
return [{
    find:function(){
        returnUsers.find({ _id: this.userId },{fields:{"profile.chat":1}});
    },
    children:[{
        find:function(user){ //this function is passed each user returned from the cursor above.returnUserChats.find({userId:user._id},{fields:{blah:1,blah:1}}); //find the user chats using whatever query 
        },
        children:[
            //if there are any children of user chats that you need to publish, do so here...
            {
                find:function(userchat){
                    returnChats.find({_id:userchat.chatId})
                },
                children:[
                    {
                        find:function(chat){
                            returnChatMess.find({chatId:chat._id},{ sort: { date: -1 } });
                        },
                        children:[
                            {
                                find:function(chatMess){
                                    var uids = _.without(chatMess.participants, this.userId);
                                    returnUsers.find({_id:{$in:uids}});
                                }
                            }
                        ]
                    }
                ]
            }
        ]
    },
    ]
}]

This will publish the cursors for all of the documents related to each of the parent documents. It is pretty fast, I use this package on a production platform high traffic and large datasets with no problems. On the client you could then query the documents as normal to get the ones you need to display.

Something like:

Users.findOne({_id:Meteor.userId()});
UserChats.find({userId:Meteor.userId()});
etc...

Solution 2:

Here is a solution I developped :

Meteor.publish("publishNewChat", function() {
this.unblock();

let user = Modules.both.queryGet({
                type        : 'users',
                method      : 'findOne',
                query       : { _id: this.userId },
                projection  : { fields: { "profile.chat": true } }
            });

let thisUserschats = tryReach(user, "profile", "chat").value;

if (!thisUserschats)
    return ;

thisUserschats = thisUserschats.map(function(e) { return (e.chatId); });

let chats = Modules.both.queryGet({
                type        : 'chat',
                method      : 'find',
                query       : { _id: { $in: thisUserschats } },
                projection  : { sort    : { _id: 1 },
                                limit   : 1000
                              }
            });

let chatArray = chats.fetch(),
    uids = cmid = [];

let messages_id_list = [],
    i = chatArray.length;

let_parallelQuery = index => {
    Meteor.setTimeout(function () {
        let tmp = Modules.both.queryGet({
                      type      : 'chatMess',
                      method    : 'find',
                      query     : { chatId: chatArray[index]._id },
                      projection: { limit: 1, sort: { date: -1 } }
                  });

        tmp.forEach(doc => {
            messages_id_list.push((doc && doc._id) ? doc._id : null);
        });
    }, 1);
}

while (--i >= 0)
    _parallelQuery(i);

let cursors = {
    chats           : chats,
    chatMessages    : null
}

let interval = Meteor.setInterval(function () {
    if (messages_id_list.length === chatArray.length)
    {
        Meteor.clearInterval(interval);

        cursors.chatMessages = Modules.both.queryGet({
                                    type        : 'chatMess',
                                    method      : 'find',
                                    query       : { _id: { $in: messages_id_list } },
                                    projection  : { sort: { date: -1 }, limit: 1000 }
                               });

        cursors.chats.observeChanges({
            // ...
        });

        cursors.chatMessages.observeChanges({
            // ...
        });

        self.ready();

      self.onStop(() => subHandle.stop(); );
    }
}, 10);

});

I used async function with Meteor.setTimeout to parallelize the queries and save an index refering to a chat _id to look for. Then, when a query is finished, I add the last message to an array. With a Meteor.setInterval, I check the array length to know when all the queries are done. Then, as I can't return cursors anymore, I use the Meteor publication low level API to handle the publishing of the documents.

FYI : in a first attempt, I was using 'findOne' in my _parallelQueries, which divided my computation time by 2/3. But then, thanks to a friend, I tried the cursor.foreach() function, which allowed me to divide the computation time by 2 again !

In production, the benchmarks allowed me to go from a 7/8 second response time to an average response time of 1.6 second :)

Hope this will be usefull to you people ! :)