問題描述

使用mongoose從數據庫中一次性find出數據，提示：

但是我查了default the memory limit of Node.js is 512 mb，我的數據集合大小只有127MB，并沒有超過這個大小，也不需要設置--max_old_space_size吧，求解

代碼如下：

//model.jsvar LagouInfo = require(’./schema.js’)var DesData = require(’./des.js’);var async = require(’async’);LagouInfo.find({}, function(res){ res.forEach(function(item){if(item.content.trim()){ var sumSalary = 0; item.salary.forEach(function(sitem){var num;if(sitem.indexOf(’-’) == -1){ num = sitem.replace(/D+/, ’’);}else{ num = parseInt(sitem.trim().split(’-’)[1]);}// console.log(sitem, num);sumSalary += num; }) var tags = item.tag.trim().split(’,’); tags.forEach(function(tag){DesData.update({’tag’: tag}, {$push: {’content’: item.content}});DesData.update({’tag’: tag}, {$inc: {’total’: item.total}});DesData.update({’tag’: tag}, {$inc:{’salary’: sumSalary}}); })} })})

//des.jsvar mongoose = require(’./db.js’), Schema = mongoose.Schema;var InfoSchema = new Schema({ tag: {type: String}, content: {type: Array}, total: {type: Number}, salary: {type: Number}, });var Data = mongoose.model(’desdata’, InfoSchema, ’desdata’);function insert(obj, callback){ var data = new Data(obj); data.save(function(err, res){if(err) console.log(’Error:’ + err);else callback(null, res); })}function update(conditions, updateStr){ Data.update(conditions, updateStr, function(err, res){if(err) console.log(’Error:’ + err);else console.log(’Update Success’);// else callback(null, res); })}function find(conditions, callback){ Data.find(conditions, function(err, res){if(err) console.log(’Error:’ + err);else callback(res); }) // return Data.find(conditions).exec();}module.exports = { insert: insert, update: update, find: find}

//schema.jsvar mongoose = require(’./db.js’), Schema = mongoose.Schema;var LagouSchema = new Schema({ name: {type: String}, cid: {type: Number}, process: {type: String}, content: {type: String}, url: {type: String}, tag: {type: String}, total: {type: Number}, salary: {type: Array}});var Lagou = mongoose.model(’lagou’, LagouSchema, ’lagou’);function update(conditions, update){ Lagou.update(conditions, update, function(err, res){if(err) console.log(’Error:’ + err);else console.log(’Res:’ + res); })}function del(conditions){ Lagou.remove(conditions, function(err, res){if(err) console.log(’Error:’ + err);else console.log(’Res:’ + res); })}function find(conditions, callback){ Lagou.find(conditions, function(err, res){if(err) console.log(’Error:’ + err);else callback(res); })}module.exports = { find: find, del: del, update: update}

//db.jsvar mongoose = require(’mongoose’), DB_URL = ’mongodb://localhost:27017/result’;mongoose.Promise = global.Promise;mongoose.connect(DB_URL);//連接成功mongoose.connection.on(’connected’, function(){ console.log(’Mongoose connection open to ’ + DB_URL);})//連接異常mongoose.connection.on(’error’, function(err){ console.log(’Mongoose connection error: ’ + err);})//連接斷開mongoose.connection.on(’disconnected’, function(){ console.log(’Mongoose connection disconnected’);})module.exports = mongoose;

============更新2017-01-12===================

嘗試設置了--max_old_space_size還是報錯

報錯內容：

$ node --max_old_space_size=2000 models/test.jsMongoose connection open to mongodb://localhost:27017/result<--- Last few GCs ---> 251717 ms: Mark-sweep 1989.2 (2067.0) -> 1989.3 (2067.0) MB, 1564.0 / 0 ms [allocation failure] [scavenge might not succeed]. 253271 ms: Mark-sweep 1989.3 (2067.0) -> 1989.2 (2067.0) MB, 1554.2 / 0 ms [allocation failure] [scavenge might not succeed]. 254868 ms: Mark-sweep 1989.2 (2067.0) -> 1989.3 (2067.0) MB, 1597.0 / 0 ms [last resort gc]. 256434 ms: Mark-sweep 1989.3 (2067.0) -> 1989.2 (2067.0) MB, 1566.2 / 0 ms [last resort gc].<--- JS stacktrace --->==== JS stack trace =========================================Security context: 00000394003B4639 <JS Object> 2: /* anonymous */(aka /* anonymous */) [D:softwareself_learnPaperpreprocessnode_modulesmongodb-corelibconnectionpool.js:~1096] [pc=000002BA3DA6C022] (this=00000394003041B9 <undefined>) 3: waitForAuth(aka waitForAuth) [D:softwareself_learnPaperpreprocessnode_modulesmongodb-corelibconnectionpool.js:1088] [pc=000002BA41DE1FBC] (this=00000394003041B9 <undefined>,cb=0000...FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

問題解答

感覺這個報錯和mongodb沒有關系了 是node程序的內存問題吧

試試這樣子可以嗎

node --max_old_space_size=2000 server.js

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

首先，數據大小在數據庫里面看到的和程序讀到內存里的體積是不同的。建議加些條件，縮小數據量，看是否有問題？然后加多點數據，看是否有問題？看看有問題的時候記錄條數是多少。第二，我怎么覺得你這函數寫的有點問題？DesData.update是異步方法，你用的是同步寫法，然后我看你的tags.foreach方法應該是循環多次的，多少次我不清楚，但應該挺大的，導致你oom問題的很可能出現在這里----堆積太多異步方法等待執行了。而且你這樣寫異步真的好嗎。建議用async包來改一下你的代碼，最好用forEachLimit，手機打字就不直接幫你改代碼了，如果還不行，我上電腦的時候幫你改。

能否貼出您的部分代碼，看看代碼是否有環節存在潛在Issue。