Memory leak - rss keep increasing #4497
Labels
needs more info
issues that need more info from the author
Comments
|
Can you provide a minimal reproduction without any dependencies?
If not, this might be an issue with Express? |
|
Hey @RedYetiDev I've removed express but I still using So it might be from the bug still persist: package.json "dependencies": {
"debug": "~2.6.9",
"firebase-admin": "^12.7.0",
"http-errors": "~1.6.3"
}app.js const { initializeApp, cert } = require("firebase-admin/app");
const { getFirestore } = require("firebase-admin/firestore");
const { PROJECT_DIR } = require("./settings");
const FirebaseServiceAccount = require(`${PROJECT_DIR}/firebaseCredentials.dev.json`);
const adminConfig = {
credential: cert(FirebaseServiceAccount),
};
initializeApp(adminConfig);
const firestore = getFirestore();
const formatMemoryUsage = (data) =>
`${Math.round((data / 1024 / 1024) * 100) / 100} MB`;
const formatProces = () => {
const formatedObj = {};
const memory = process.memoryUsage();
Object.keys(memory).forEach(
(key) => (formatedObj[key] = formatMemoryUsage(memory[key]))
);
return formatedObj;
};
module.exports = async (req, res) => {
try {
const aData = [];
const aSnap = await firestore.collection("test").limit(20).get();
aSnap.forEach((doc) => {
aData.push(doc.data());
});
console.log(new Date(), "after finish Total", formatProces());
res.writeHead(200, { 'Content-Type': 'application/json' });
res.write(JSON.stringify(aData));
res.end();
} catch (err) {
console.error(err);
res.writeHead(400);
res.end(err.message);
}
};#!/usr/bin/env node
/**
* Module dependencies.
*/
var app = require("../app");
var debug = require("debug")("memory:server");
var http = require("http");
/**
* Get port from environment and store in Express.
*/
var port = normalizePort(process.env.PORT || "8080");
let requestNumber = 0;
/**
* Create HTTP server.
*/
var server = http.createServer(app);
/**
* Listen on provided port, on all network interfaces.
*/
const formatMemoryUsage = (data) =>
`${Math.round((data / 1024 / 1024) * 100) / 100} MB`;
const formatProces = () => {
const formatedObj = {};
const memory = process.memoryUsage();
Object.keys(memory).forEach(
(key) => (formatedObj[key] = formatMemoryUsage(memory[key]))
);
return formatedObj;
};
server.listen(port);
server.on("error", onError);
server.on("listening", onListening);
server.on("request", () => {
console.log(
"---------------------------------------------------------------------------------------------"
);
console.info(new Date(), `Request ${++requestNumber} start `, formatProces());
});
server.on("close", () =>
console.info(new Date(), `On request close memory `, formatProces())
);
/**
* Normalize a port into a number, string, or false.
*/
function normalizePort(val) {
var port = parseInt(val, 10);
if (isNaN(port)) {
// named pipe
return val;
}
if (port >= 0) {
// port number
return port;
}
return false;
}
/**
* Event listener for HTTP server "error" event.
*/
function onError(error) {
if (error.syscall !== "listen") {
throw error;
}
var bind = typeof port === "string" ? "Pipe " + port : "Port " + port;
// handle specific listen errors with friendly messages
switch (error.code) {
case "EACCES":
console.error(bind + " requires elevated privileges");
process.exit(1);
break;
case "EADDRINUSE":
console.error(bind + " is already in use");
process.exit(1);
break;
default:
throw error;
}
}
/**
* Event listener for HTTP server "listening" event.
*/
function onListening() {
var addr = server.address();
var bind = typeof addr === "string" ? "pipe " + addr : "port " + addr.port;
debug("Listening on " + bind);
}After running some requests constantly: Logs: You will notice that after request 23 `rss` got released a bit, but than it kept growing, till request 130 `rss` reached `191 MB`, then I waited 5 minutes and make another request, at the 131 request start rss was still at `191 MB ` but it got released a bit after request finish. |
Preferably, there should be no external dependencies |
|
@RedYetiDev Now testing on public api package.json "dependencies": {
"debug": "~2.6.9",
"http-errors": "~1.6.3"
}app.js module.exports = async (req, res) => {
try {
// const aData = [];
// const aSnap = await firestore.collection("autentique").limit(20).get();
// aSnap.forEach((doc) => {
// aData.push(doc.data());
// });
const result = {};
const keys = ["books", "characters", "houses", "spells"];
await Promise.all(
keys.map(async (key) => {
const response = await fetch(
"https://potterapi-fedeperin.vercel.app/en/" + key
);
const aData = await response.json();
result[key] = aData;
})
);
console.log(new Date(), "after finish Total", formatProces());
res.writeHead(200, { "Content-Type": "application/json" });
res.write(JSON.stringify(result));
res.end();
} catch (err) {
console.error(err);
res.writeHead(400);
res.end(err.message);
}
};Logs: Logs: making requests 1 by 1 constantly you will notice very tiny grow in `rss`, after request 214 I waited 10 minutes then make request number 215 which on start it has already `rss` huge release., So why the rss got freed only after stop making requests ( cooldown between requests ), On real prod server requests are made each seconds. !Logs from docker container: You will see making requests over 2 minutes constantly have grow `rss` without any release, in request 232 I waited 5 minutes before the making another request and `rss` still same, but got small decreases in request 239
|
|
@MrLibya - can you check the installed memory and the load characteristics of your Mac m1 and the container? (top command may help). rss do not come down automatically when the app frees up allocations, instead, only in response to memory demand from elsewhere in the system. |
|
I've re-run the app start with as for docker container you can check the image:
Also in my machine I run this line as you requested: File is attached |
|
@MrLibya - thanks. it is little tricky to compare the container and the local box data, as you used two different metrics. however, the screenshot is very verbose: 2 GB of memory and ~200MB of rss. it is very much possible that the rss is not coming down as there is no |
|
@gireeshpunathil Doing this wont get me the correct result im looking for, Because our production is google cloud run service that run container of nodeJs app only, so there's no other demand of memory from other apps. So I re-run with 256MB memory so I can get better result, Running requests made
|
|
@MrLibya - thanks.
|
|
@gireeshpunathil I created script to allocate buffer ( outside of the http server ), so its different process when run it the container memory stay same at So memory got freed from the http server due to another process requesting memory on the container, So that mean no memory leak correct ? but the main problem is in our prod server we have 4gb of memory that keep go out of limit due to big numbers of requests, the rss will keep increasing in each request same as in the test I wrote above. What I don't understand is that for this script that have 1 endpoint no dependencies and it make call to outside api to fetch same why the rss keep increasing by each request I make ? const test = () => {
try {
for(let i =0; i< 1000000; i ++){
console.log("allocate buffer");
Buffer.allocUnsafe(10 * 1024 * 1024);
}
} catch (err) {
console.error(err);
}
};
test() |
|
@MrLibya - first, thanks for doing all the tests on request!
yes, you are right! there is no visible leak in your program. a complete explanation of this behaviour can be found in the investigation we did here: nodejs/node#31641 in summary, when there is a demand (by other processes), one or more of such pages (which are not currently in use) will be taken away from the process, leading to rss reduction. so as long as rss comes down in a sufficiently loaded system, you don't need to worry about leaks. however, in cloud deployments if they charge you based on the rss, then you should work with the vendor and explain this to them why they shouldn't use rss for metering memory usage. (unfortunately I don't see a standard evolving around this) hope this helps! |
|
@gireeshpunathil Thanks for the explanation, Our issue actually is not about cloud charge us for the |
|
@MrLibya - container failure was never mentioned in your issue description or title! either way, an ideal way out would be:
hope this helps! |
|
@gireeshpunathil Ah my bad I though I've mentioned it. the heapUsed & healTotal is perfectly normal on the production server, I though of the reason for memory limit reached is also the number of requests we having per container but I didn't understand before how rss actually work so though its memory leak cuz is never go down ! Thank you very much for the help, I'll close this. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Node.js Version
18.20.1
NPM Version
10.5.0
Operating System
Linux 51d05a3105c7 6.10.4-linuxkit #1 SMP Wed Oct 2 16:38:00 UTC 2024 x86_64 GNU/Linux
Subsystem
fs, https, process, v8, Other
Description
Hello, I've ExpreesJS application and I'm facing memory limit reached on prod server, Which is
rssreaching memory limitWhen I run the application locally on
Mac M1machine I've normalrss, heap usagetherssstart at z90MBgo up to150MBon load and in few seconds it will be freed to be back at100MB.But on my docker container the
rssstart at400MBand goes up to500MBon some load and it doesn't free at all, maybe in few minutes it free 1-5MB but that's it.heap seems to be normal, and since when I run the node on my local machine the
rssis normal too I can say is not related to any native libraries I use. So can it be from docker image ?Docker file:
I also tried to use
libjemallocbut same resultAny tips on debugging this ?
Minimal Reproduction
No response
Output
No response
Before You Submit
The text was updated successfully, but these errors were encountered: