populate() fails to reliably populate virtual fields in an embedded subdocument array, depending on the number of matches

[sharang-lyric]

Prerequisites

• I have written a descriptive issue title
• I have searched existing issues to ensure the bug has not already been reported

Mongoose version

6.7.3

Node.js version

16.14

MongoDB server version

5.0

Typescript version (if applicable)

No response

Description

I have a model Store with an array property employees where each element is a nested schema EmployeeSchema. I have another model Shift which contains details of the shifts for every employee. The schema EmployeeSchema has a virtual called mostRecentShift with Shift as a reference.

When calling store.populate('employees.mostRecentShift'), even though matching documents always exist for the mostRecentShift, depending on the number of matches, it may or may not be populated.

Steps to Reproduce

The following code shows the behavior of populate in two cases:

1. When one employee has 1 shift and the other employee has 2 shifts
2. When both employees have 1 shift each

import mongoose from 'mongoose';

const uri = process.env.MONGO_URI;

const shiftSchema = new mongoose.Schema({
  employeeId: mongoose.Types.ObjectId,
  startedAt: Date,
  endedAt: Date
});

const Shift = mongoose.model('Shift', shiftSchema);

const employeeSchema = new mongoose.Schema({
  name: String,
});

employeeSchema.virtual('mostRecentShift', {
  ref: Shift,
  localField: '_id',
  foreignField: 'employeeId',
  options: {
    sort: { startedAt: -1 },
  },
  justOne: true,
});

const storeSchema = new mongoose.Schema({
  location: String,
  employees: [employeeSchema],
});

const Store = mongoose.model('Store', storeSchema)

mongoose.connect(uri);

const store = await Store.create({
  location: 'Tashbaan',
  employees: [
    { name: 'Aravis' },
    { name: 'Shasta' },
  ],
});

const employeeAravis = store.employees.find(({ name }) => name === 'Aravis');
const employeeShasta = store.employees.find(({ name }) => name === 'Shasta');

Shift.insertMany([
  { employeeId: employeeAravis._id, startedAt: new Date(Date.now() - 57600000), endedAt: new Date(Date.now() - 43200000) },
  { employeeId: employeeAravis._id, startedAt: new Date(Date.now() - 28800000), endedAt: new Date(Date.now() - 14400000) },
  { employeeId: employeeShasta._id, startedAt: new Date(Date.now() - 14400000), endedAt: new Date() },
])

const storeWithMostRecentShifts = await Store.findOne({ location: 'Tashbaan' })
  .populate('employees.mostRecentShift')
  .select('-__v')
  .exec();

// Even though shifts exist for both Aravis and Shasta, Shasta's mostRecentShift is NOT populated
console.log('When Aravis has 2 shifts and Shasta has 1 shift')
console.log(JSON.stringify(storeWithMostRecentShifts.toJSON({ virtuals: true }).employees, null, 4));

// Now I remove Aravis's oldest shift, so that her mostRecentShift does not change
await Shift.findOne({ employeeId: employeeAravis._id }).sort({ startedAt: 1 }).then((s) => s.remove());

// Re-populating mostRecentShift 
const storeWithMostRecentShiftsNew = await Store.findOne({ location: 'Tashbaan' })
  .populate('employees.mostRecentShift')
  .select('-__v')
  .exec();

// Now, mostRecentShift is populated for both employees
console.log('When both employees have 1 shift each')
console.log(JSON.stringify(storeWithMostRecentShiftsNew.toJSON({ virtuals: true }).employees, null, 4));

Output:

When Aravis has 2 shifts and Shasta has 1 shift
[
    {
        "name": "Aravis",
        "_id": "63839d301838d361657e885d",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8863",
            "employeeId": "63839d301838d361657e885e",
            "startedAt": "2022-11-27T13:24:09.074Z",
            "endedAt": "2022-11-27T17:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8863"
        },
        "id": "63839d301838d361657e885d"
    },
    {
        "name": "Shasta",
        "_id": "63839d301838d361657e885e",
        "id": "63839d301838d361657e885e"
    }
]
When both employees have 1 shift each
[
    {
        "name": "Aravis",
        "_id": "63839d301838d361657e885d",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8863",
            "employeeId": "63839d301838d361657e885e",
            "startedAt": "2022-11-27T13:24:09.074Z",
            "endedAt": "2022-11-27T17:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8863"
        },
        "id": "63839d301838d361657e885d"
    },
    {
        "name": "Shasta",
        "_id": "63839d301838d361657e885e",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8862",
            "employeeId": "63839d301838d361657e885d",
            "startedAt": "2022-11-27T09:24:09.074Z",
            "endedAt": "2022-11-27T13:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8862"
        },
        "id": "63839d301838d361657e885e"
    }
]

Expected Behavior

mostRecentShift to have been populated for both employees in both cases.
Expected output:

When Aravis has 2 shifts and Shasta has 1 shift
[
    {
        "name": "Aravis",
        "_id": "63839d301838d361657e885d",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8863",
            "employeeId": "63839d301838d361657e885e",
            "startedAt": "2022-11-27T13:24:09.074Z",
            "endedAt": "2022-11-27T17:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8863"
        },
        "id": "63839d301838d361657e885d"
    },
    {
        "name": "Shasta",
        "_id": "63839d301838d361657e885e",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8862",
            "employeeId": "63839d301838d361657e885d",
            "startedAt": "2022-11-27T09:24:09.074Z",
            "endedAt": "2022-11-27T13:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8862"
        },
        "id": "63839d301838d361657e885e"
    }
]
When both employees have 1 shift each
[
    {
        "name": "Aravis",
        "_id": "63839d301838d361657e885d",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8863",
            "employeeId": "63839d301838d361657e885e",
            "startedAt": "2022-11-27T13:24:09.074Z",
            "endedAt": "2022-11-27T17:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8863"
        },
        "id": "63839d301838d361657e885d"
    },
    {
        "name": "Shasta",
        "_id": "63839d301838d361657e885e",
        "mostRecentShift": {
            "_id": "63839d391838d361657e8862",
            "employeeId": "63839d301838d361657e885d",
            "startedAt": "2022-11-27T09:24:09.074Z",
            "endedAt": "2022-11-27T13:24:09.074Z",
            "__v": 0,
            "id": "63839d391838d361657e8862"
        },
        "id": "63839d301838d361657e885e"
    }
]

[IslandRhythms]

One problem I just noticed is that you're calling findOne() when you should be calling find()

[IslandRhythms]

const mongoose = require('mongoose');

const shiftSchema = new mongoose.Schema({
  employeeId: mongoose.Types.ObjectId,
  startedAt: Date,
  endedAt: Date
});

const Shift = mongoose.model('Shift', shiftSchema);

const employeeSchema = new mongoose.Schema({
  name: String,
}, {toJSON: {virtuals: true}, toObject: {virtuals: true}});

employeeSchema.virtual('mostRecentShift', {
  ref: Shift,
  localField: '_id',
  foreignField: 'employeeId',
  options: {
    sort: { startedAt: -1 },
  },
  justOne: true,
});

const storeSchema = new mongoose.Schema({
  location: String,
  employees: [employeeSchema],
});

const Store = mongoose.model('Store', storeSchema)

async function run() {

  await mongoose.connect('mongodb://localhost:27017');
  await mongoose.connection.dropDatabase();

  const store = await Store.create({
    location: 'Tashbaan',
    employees: [
      { name: 'Aravis' },
      { name: 'Shasta' },
    ],
  });
  
  const employeeAravis = store.employees.find(({ name }) => name === 'Aravis');
  const employeeShasta = store.employees.find(({ name }) => name === 'Shasta');
  console.log('is aravis aravis', employeeAravis)
  console.log('is shasta shasta', employeeShasta)
  
  await Shift.insertMany([
    { employeeId: employeeAravis._id, startedAt: new Date(Date.now() - 57600000), endedAt: new Date(Date.now() - 43200000) },
    { employeeId: employeeAravis._id, startedAt: new Date(Date.now() - 28800000), endedAt: new Date(Date.now() - 14400000) },
    { employeeId: employeeShasta._id, startedAt: new Date(Date.now() - 14400000), endedAt: new Date() },
  ])
  
  const storeWithMostRecentShifts = await Store.findOne({ location: 'Tashbaan' })
    .populate('employees.mostRecentShift')
    .select('-__v');

  // Even though shifts exist for both Aravis and Shasta, Shasta's mostRecentShift is NOT populated
  console.log('When Aravis has 2 shifts and Shasta has 1 shift')
  console.log(JSON.stringify(storeWithMostRecentShifts.toJSON({ virtuals: true }).employees, null, 4));
  
  // Now I remove Aravis's oldest shift, so that her mostRecentShift does not change
  console.log('original shift', await Shift.find({ employeeId: employeeAravis._id }))
  const Shifts = await Shift.find({ employeeId: employeeAravis._id }).sort({ startedAt: 1 });
  const removedShift = await Shift.findByIdAndDelete({ _id: Shifts[0]._id });
  console.log('removedShift', removedShift);
  console.log('remaining shifts', await Shift.find());
  
  // Re-populating mostRecentShift 
  const storeWithMostRecentShiftsNew = await Store.findOne({ location: 'Tashbaan' })
    .populate('employees.mostRecentShift')
    .select('-__v');
  
  // Now, mostRecentShift is populated for both employees
  console.log('When both employees have 1 shift each')
  console.log(JSON.stringify(storeWithMostRecentShiftsNew.toJSON({ virtuals: true }).employees, null, 4));
}

run();