mongodb 跑15W数据量的数据老挂，内存爆满，CPU占用很高，VPS 是KVM 512的，有没有解决办法啊？

V2EX = way to explore

V2EX 是一个关于分享和探索的地方

现在注册

已注册用户请登录

这是一个创建于 3812 天前的主题，其中的信息可能已经有所发展或是发生改变。

这是我之前发的帖子

http://www.v2ex.com/t/88732

网站是 http://www.peinimai.com

数据量

老挂

MongoDB

17 条回复 • 1970-01-01 08:00:00 +08:00

humiaozuzu

2013-11-10 23:07:48 +08:00

贴一下 log？还有 mongoshell 的 db.serverStatus()

zhfsxtx

2013-11-10 23:20:43 +08:00

@humiaozuzu

> db.serverStatus()
{
"host" : "hhVPS",
"version" : "2.4.8",
"process" : "mongod",
"pid" : 30181,
"uptime" : 744,
"uptimeMillis" : NumberLong(744564),
"uptimeEstimate" : 736,
"localTime" : ISODate("2013-11-10T15:19:11.837Z"),
"asserts" : {
"regular" : 0,
"warning" : 0,
"msg" : 0,
"user" : 0,
"rollovers" : 0
},
"backgroundFlushing" : {
"flushes" : 12,
"total_ms" : 76,
"average_ms" : 6.333333333333333,
"last_ms" : 1,
"last_finished" : ISODate("2013-11-10T15:18:47.290Z")
},
"connections" : {
"current" : 11,
"available" : 808,
"totalCreated" : NumberLong(11)
},
"cursors" : {
"totalOpen" : 0,
"clientCursors_size" : 0,
"timedOut" : 0
},
"dur" : {
"commits" : 30,
"journaledMB" : 0,
"writeToDataFilesMB" : 0,
"compression" : 0,
"commitsInWriteLock" : 0,
"earlyCommits" : 0,
"timeMs" : {
"dt" : 3070,
"prepLogBuffer" : 0,
"writeToJournal" : 0,
"writeToDataFiles" : 0,
"remapPrivateView" : 0
}
},
"extra_info" : {
"note" : "fields vary by platform",
"heap_usage_bytes" : 122573112,
"page_faults" : 245
},
"globalLock" : {
"totalTime" : NumberLong(744564000),
"lockTime" : NumberLong(14523),
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 0,
"readers" : 0,
"writers" : 0
}
},
"indexCounters" : {
"accesses" : 0,
"hits" : 0,
"misses" : 0,
"resets" : 0,
"missRatio" : 0
},
"locks" : {
"." : {
"timeLockedMicros" : {
"R" : NumberLong(19785),
"W" : NumberLong(14523)
},
"timeAcquiringMicros" : {
"R" : NumberLong(12923),
"W" : NumberLong(159266)
}
},
"admin" : {
"timeLockedMicros" : {
"r" : NumberLong(1190),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(1554),
"w" : NumberLong(0)
}
},
"local" : {
"timeLockedMicros" : {
"r" : NumberLong(1287),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(139),
"w" : NumberLong(0)
}
},
"double11" : {
"timeLockedMicros" : {
"r" : NumberLong(28627609),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(10399787),
"w" : NumberLong(0)
}
}
},
"network" : {
"bytesIn" : 9707,
"bytesOut" : 2832815,
"numRequests" : 131
},
"opcounters" : {
"insert" : 1,
"query" : 118,
"update" : 0,
"delete" : 0,
"getmore" : 22,
"command" : 29
},
"opcountersRepl" : {
"insert" : 0,
"query" : 0,
"update" : 0,
"delete" : 0,
"getmore" : 0,
"command" : 0
},
"recordStats" : {
"accessesNotInMemory" : 43,
"pageFaultExceptionsThrown" : 2,
"admin" : {
"accessesNotInMemory" : 0,
"pageFaultExceptionsThrown" : 0
},
"double11" : {
"accessesNotInMemory" : 43,
"pageFaultExceptionsThrown" : 2
},
"local" : {
"accessesNotInMemory" : 0,
"pageFaultExceptionsThrown" : 0
}
},
"writeBacksQueued" : false,
"mem" : {
"bits" : 64,
"resident" : 291,
"virtual" : 1964,
"supported" : true,
"mapped" : 288,
"mappedWithJournal" : 576
},
"metrics" : {
"document" : {
"deleted" : NumberLong(0),
"inserted" : NumberLong(1),
"returned" : NumberLong(4846),
"updated" : NumberLong(0)
},
"getLastError" : {
"wtime" : {
"num" : 0,
"totalMillis" : 0
},
"wtimeouts" : NumberLong(0)
},
"operation" : {
"fastmod" : NumberLong(0),
"idhack" : NumberLong(0),
"scanAndOrder" : NumberLong(0)
},
"queryExecutor" : {
"scanned" : NumberLong(4804690)
},
"record" : {
"moves" : NumberLong(0)
},
"repl" : {
"apply" : {
"batches" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0)
},
"buffer" : {
"count" : NumberLong(0),
"maxSizeBytes" : 268435456,
"sizeBytes" : NumberLong(0)
},
"network" : {
"bytes" : NumberLong(0),
"getmores" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0),
"readersCreated" : NumberLong(0)
},
"oplog" : {
"insert" : {
"num" : 0,
"totalMillis" : 0
},
"insertBytes" : NumberLong(0)
},
"preload" : {
"docs" : {
"num" : 0,
"totalMillis" : 0
},
"indexes" : {
"num" : 0,
"totalMillis" : 0
}
}
},
"ttl" : {
"deletedDocuments" : NumberLong(0),
"passes" : NumberLong(12)
}
},
"ok" : 1
}

bengtuo

2013-11-10 23:20:46 +08:00

512的内存跑mongodb 有点小

zhfsxtx

2013-11-10 23:21:09 +08:00

@humiaozuzu log 太多不知道怎么贴

moname

2013-11-10 23:22:41 +08:00 via iPhone

15万数据，搞个大点的vps吧。

webflier

2013-11-10 23:43:30 +08:00

1.要么用大点内存。
2.要么用多台512M的机器做sharding。
我能偷偷告诉你我有一个10多台512M的机器做的mongodbb sharding cluster嘛......物美价廉，跑得非常的欢畅
另外你有没有在数据库前面做适当的caching挡一下？这样数据库的压力会小很多？

so898

2013-11-10 23:59:02 +08:00

512MB内存也敢上mongodb，楼主你太小瞧这玩意了吧……

zhfsxtx

2013-11-11 00:15:11 +08:00

@webflier
什么caching啊，我没有，我都是自己瞎折腾，门外汉
10多台512M ，你真是高富帅啊

likuku

2013-11-11 00:18:36 +08:00

挂的时候啥报错啊？

mikale

2013-11-11 00:32:26 +08:00

mongodb有small file模式，你用用，这适合你这种环境，不然就这点小内存，根本不适合

zhfsxtx

2013-11-11 00:39:49 +08:00

@mikale 怎么开户smalll file 模式啊，是不是我要先把数据导出来，再用--smallfiles 重新导入数据啊

zhfsxtx

2013-11-11 00:42:28 +08:00

@likuku

Sun Nov 10 15:05:45.852 [initandlisten] git version: a350fc38922fbda2cec8d5dd842237b904eafc14
Sun Nov 10 15:05:45.852 [initandlisten] build info: Linux ip-10-2-29-40 2.6.21.7-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49
Sun Nov 10 15:05:45.852 [initandlisten] allocator: tcmalloc
Sun Nov 10 15:05:45.852 [initandlisten] options: { dbpath: "/data/db", fork: true, logappend: true, logpath: "/root/mongodb/work.log", port: 27017, quiet: true }
Sun Nov 10 15:05:46.485 [initandlisten] journal dir=/data/db/journal
Sun Nov 10 15:05:46.485 [initandlisten] recover begin
Sun Nov 10 15:05:46.485 [initandlisten] info no lsn file in journal/ directory
Sun Nov 10 15:05:46.485 [initandlisten] recover lsn: 0
Sun Nov 10 15:05:46.485 [initandlisten] recover /data/db/journal/j._0
Sun Nov 10 15:05:46.616 [initandlisten] recover cleaning up
Sun Nov 10 15:05:46.616 [initandlisten] removeJournalFiles
Sun Nov 10 15:05:46.753 [initandlisten] recover done
Sun Nov 10 15:05:46.754 [initandlisten] preallocating a journal file /data/db/journal/prealloc.0
Sun Nov 10 15:05:49.055 [initandlisten] File Preallocator Progress: 167772160/1073741824 15%
Sun Nov 10 15:05:52.981 [initandlisten] File Preallocator Progress: 272629760/1073741824 25%
Sun Nov 10 15:05:55.019 [initandlisten] File Preallocator Progress: 408944640/1073741824 38%
Sun Nov 10 15:05:58.063 [initandlisten] File Preallocator Progress: 555745280/1073741824 51%
Sun Nov 10 15:06:01.267 [initandlisten] File Preallocator Progress: 681574400/1073741824 63%
Sun Nov 10 15:06:04.246 [initandlisten] File Preallocator Progress: 796917760/1073741824 74%
Sun Nov 10 15:06:07.414 [initandlisten] File Preallocator Progress: 880803840/1073741824 82%
Sun Nov 10 15:06:10.098 [initandlisten] File Preallocator Progress: 891289600/1073741824 83%
Sun Nov 10 15:06:13.134 [initandlisten] File Preallocator Progress: 975175680/1073741824 90%
Sun Nov 10 15:06:20.004 [initandlisten] command local.$cmd command: { create: "startup_log", size: 10485760, capped: true } ntoreturn:1 keyUpdates:0 reslen:75 212ms
Sun Nov 10 15:06:20.028 [websvr] admin web console waiting for connections on port 28017
Sun Nov 10 15:06:20.449 [initandlisten] waiting for connections on port 27017
Sun Nov 10 15:06:31.422 [signalProcessingThread] got signal 2 (Interrupt), will terminate after current cmd ends
Sun Nov 10 15:06:31.422 [signalProcessingThread] now exiting
Sun Nov 10 15:06:31.422 dbexit:
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: going to close listening sockets...
Sun Nov 10 15:06:31.422 [signalProcessingThread] closing listening socket: 9
Sun Nov 10 15:06:31.422 [signalProcessingThread] closing listening socket: 10
Sun Nov 10 15:06:31.422 [signalProcessingThread] closing listening socket: 11
Sun Nov 10 15:06:31.422 [signalProcessingThread] removing socket file: /tmp/mongodb-27017.sock
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: going to flush diaglog...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: going to close sockets...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: waiting for fs preallocator...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: lock for final commit...
Sun Nov 10 15:06:31.422 [signalProcessingThread] shutdown: final commit...
Sun Nov 10 15:06:31.478 [signalProcessingThread] shutdown: closing all files...
Sun Nov 10 15:06:31.478 [signalProcessingThread] closeAllFiles() finished
Sun Nov 10 15:06:31.478 [signalProcessingThread] journalCleanup...
Sun Nov 10 15:06:31.478 [signalProcessingThread] removeJournalFiles
Sun Nov 10 15:06:31.607 [signalProcessingThread] shutdown: removing fs lock...
Sun Nov 10 15:06:31.620 dbexit: really exiting now

webflier

2013-11-11 00:46:10 +08:00

看着信息，你是不是硬盘没空间了啊

zhfsxtx

2013-11-11 00:58:59 +08:00

@webflier

Filesystem Size Used Avail Use% Mounted on
/dev/sda1 9.5G 5.5G 3.6G 61% /
udev 242M 4.0K 242M 1% /dev
tmpfs 99M 208K 99M 1% /run
none 5.0M 0 5.0M 0% /run/lock
none 246M 0 246M 0% /run/shm

硬盘有空间的，而且这个网站只会查询数据，不会写数据库的

zhfsxtx

2013-11-11 01:01:34 +08:00

fdisk -l 看是这样的

Disk /dev/sda: 10.7 GB, 10737418240 bytes
255 heads, 63 sectors/track, 1305 cylinders, total 20971520 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00032c86

Device Boot Start End Blocks Id System
/dev/sda1 2048 19924991 9961472 83 Linux
/dev/sda2 19924992 20971519 523264 82 Linux swap / Solaris

但是为什么free的时候看，swap 是0呢

total used free shared buffers cached
Mem: 503528 434740 68788 0 8584 174816
-/+ buffers/cache: 251340 252188
Swap: 0 0 0

thai9quohs6jae1C

2013-11-11 01:02:49 +08:00

如果换作是mariadb毫无压力啊哈哈

zhfsxtx

2013-11-11 12:38:19 +08:00

今天加挂了512M SWAP，不知道会不会好点