8000 turn off warnings about `overcommit_memory` settings (#7237) · arangodb/arangodb@96b7a60 · GitHub
[go: up one dir, main page]
Skip to content

Commit 96b7a60

Browse files
jsteemannjsteemann
authored
turn off warnings about overcommit_memory settings (#7237)
1 parent 0ed466c commit 96b7a60

File tree

3 files changed

+85
-75
lines changed

3 files changed

+85
-75
lines changed

‎CHANGELOG

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,28 @@
1+
v3.2.18 (XXXX-XX-XX)
2+
--------------------
3+
4+
* disable startup warning for Linux kernel variable `vm.overcommit_memory` settings
5+
values of 0 or 1.
6+
Effectively `overcommit_memory` settings value of 0 or 1 fix two memory-allocation
7+
related issues with the default memory allocator used in ArangoDB release builds on
8+
64bit Linux.
9+
The issues will remain when running with an `overcommit_memory` settings value of 2,
10+
so this is now discouraged.
11+
Setting `overcommit_memory` to 0 or 1 (0 is the Linux kernel's default) fixes issues
12+
with increasing numbers of memory mappings for the arangod process (which may lead
13+
to an out-of-memory situation if the kernel's maximum number of mappings threshold
14+
is hit) and an increasing amount of memory that the kernel counts as "committed".
15+
With an `overcommit_memory` setting of 0 or 1, an arangod process may either be
16+
killed by the kernel's OOM killer or will die with a segfault when accessing memory
17+
it has allocated before but the kernel could not provide later on. This is still
18+
more acceptable than the kernel not providing any more memory to the process when
19+
there is still physical memory left, which may have occurred with an `overcommit_memory`
20+
setting of 2 after the arangod process had done lots of allocations.
21+
22+
In summary, the recommendation for the `overcommit_memory` setting is now to set it
23+
to 0 or 1 (0 is kernel default) and not use 2.
24+
25+
126
v3.2.17 (2018-10-24)
227
--------------------
328

‎lib/ApplicationFeatures/EnvironmentFeature.cpp

Lines changed: 60 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,6 @@
2626
#include "Basics/FileUtils.h"
2727
#include "Basics/StringUtils.h"
2828
#include "Logger/Logger.h"
29-
#include "RocksDBEngine/RocksDBEngine.h"
30-
#include "StorageEngine/EngineSelectorFeature.h"
3129

3230
#ifdef __linux__
3331
#include <sys/sysinfo.h>
@@ -56,6 +54,66 @@ void EnvironmentFeature::prepare() {
5654
}
5755

5856
#ifdef __linux__
57+
// check overcommit_memory & overcommit_ratio
58+
try {
59+
std::string value =
60+
basics::FileUtils::slurp("/proc/sys/vm/overcommit_memory");
61+
uint64_t v = basics::StringUtils::uint64(value);
62+
63+
if (v == 2) {
64+
// from https://www.kernel.org/doc/Documentation/sysctl/vm.txt:
65+
//
66+
// When this flag is 0, the kernel attempts to estimate the amount
67+
// of free memory left when userspace requests more memory.
68+
// When this flag is 1, the kernel pretends there is always enough
69+
// memory until it actually runs out.
70+
// When this flag is 2, the kernel uses a "never overcommit"
71+
// policy that attempts to prevent any overcommit of memory.
72+
std::string ratio =
73+
basics::FileUtils::slurp("/proc/sys/vm/overcommit_ratio");
74+
uint64_t r = basics::StringUtils::uint64(ratio);
75+
// from https://www.kernel.org/doc/Documentation/sysctl/vm.txt:
76+
//
77+
// When overcommit_memory is set to 2, the committed address
78+
// space is not permitted to exceed swap plus this percentage
79+
// of physical RAM.
80+
81+
struct sysinfo info;
82+
int res = sysinfo(&info);
83+
if (res == 0) {
84+
double swapSpace = static_cast<double>(info.totalswap);
85+
double ram = static_cast<double>(TRI_PhysicalMemory);
86+
double rr = (ram >= swapSpace)
87+
? 100.0 * ((ram - swapSpace) / ram)
88+
: 0.0;
89+
if (static_cast<double>(r) < 0.99 * rr) {
90+
LOG_TOPIC(WARN, Logger::MEMORY)
91+
<< "/proc/sys/vm/overcommit_ratio is set to '" << r
92+
<< "'. It is recommended to set it to at least '" << std::llround(rr)
93+
<< "' (100 * (max(0, (RAM - Swap Space)) / RAM)) to utilize all "
94+
<< "available RAM. Setting it to this value will minimize swap "
95+
<< "usage, but may result in more out-of-memory errors, while "
96+
<< "setting it to 100 will allow the system to use both all "
97+
<< "available RAM and swap space.";
98+
LOG_TOPIC(WARN, Logger::MEMORY) << "execute 'sudo bash -c \"echo "
99+
<< std::llround(rr) << " > "
100+
<< "/proc/sys/vm/overcommit_ratio\"'";
101+
}
102+
}
103+
}
104+
} catch (...) {
105+
// file not found or value not convertible into integer
106+
}
107+
108+
// test local ipv6 support
109+
try {
110+
if (!basics::FileUtils::exists("/proc/net/if_inet6")) {
111+
LOG_TOPIC(INFO, arangodb::Logger::COMMUNICATION) << "IPv6 support seems to be disabled";
112+
}
113+
} catch (...) {
114+
// file not found
115+
}
116+
59117
// test local ipv4 port range
60118
try {
61119
std::string value =
@@ -209,75 +267,3 @@ void EnvironmentFeature::prepare() {
209267

210268
#endif
211269
}
212-
213-
void EnvironmentFeature::start() {
214-
#ifdef __linux__
215-
bool usingRocksDB =
216-
(EngineSelectorFeature::engineName() == RocksDBEngine::EngineName);
217-
try {
218-
std::string value =
219-
basics::FileUtils::slurp("/proc/sys/vm/overcommit_memory");
220-
uint64_t v = basics::StringUtils::uint64(value);
221-
// from https://www.kernel.org/doc/Documentation/sysctl/vm.txt:
222-
//
223-
// When this flag is 0, the kernel attempts to estimate the amount
224-
// of free memory left when userspace requests more memory.
225-
// When this flag is 1, the kernel pretends there is always enough
226-
// memory until it actually runs out.
227-
// When this flag is 2, the kernel uses a "never overcommit"
228-
// policy that attempts to prevent any overcommit of memory.
229-
std::string ratio =
230-
basics::FileUtils::slurp("/proc/sys/vm/overcommit_ratio");
231-
uint64_t r = basics::StringUtils::uint64(ratio);
232-
// from https://www.kernel.org/doc/Documentation/sysctl/vm.txt:
233-
//
234-
// When overcommit_memory is set to 2, the committed address
235-
// space is not permitted to exceed swap plus this percentage
236-
// of physical RAM.
237-
238-
if (usingRocksDB) {
239-
if (v != 2) {
240-
LOG_TOPIC(WARN, Logger::MEMORY)
241-
<< "/proc/sys/vm/overcommit_memory is set to '" << v
242-
<< "'. It is recommended to set it to a value of 2";
243-
LOG_TOPIC(WARN, Logger::MEMORY) << "execute 'sudo bash -c \"echo 2 > "
244-
<< "/proc/sys/vm/overcommit_memory\"'";
245-
}
246-
} else {
247-
if (v == 1) {
248-
LOG_TOPIC(WARN, Logger::MEMORY)
249-
<< "/proc/sys/vm/overcommit_memory is set to '" << v
250-
<< "'. It is recommended to set it to a value of 0 or 2";
251-
LOG_TOPIC(WARN, Logger::MEMORY) << "execute 'sudo bash -c \"echo 2 > "
252-
<< "/proc/sys/vm/overcommit_memory\"'";
253-
}
254-
}
255-
if (v == 2) {
256-
struct sysinfo info;
257-
int res = sysinfo(&info);
258-
if (res == 0) {
259-
double swapSpace = static_cast<double>(info.totalswap);
260-
double ram = static_cast<double>(TRI_PhysicalMemory);
261-
double rr = (ram >= swapSpace)
262-
? 100.0 * ((ram - swapSpace) / ram)
263-
: 0.0;
264-
if (static_cast<double>(r) < 0.99 * rr) {
265-
LOG_TOPIC(WARN, Logger::MEMORY)
266-
<< "/proc/sys/vm/overcommit_ratio is set to '" << r
267-
<< "'. It is recommended to set it to at least '" << std::llround(rr)
268-
<< "' (100 * (max(0, (RAM - Swap Space)) / RAM)) to utilize all "
269-
<< "available RAM. Setting it to this value will minimize swap "
270-
<< "usage, but may result in more out-of-memory errors, while "
271-
<< "setting it to 100 will allow the system to use both all "
272-
<< "available RAM and swap space.";
273-
LOG_TOPIC(WARN, Logger::MEMORY) << "execute 'sudo bash -c \"echo "
274-
<< std::llround(rr) << " > "
275-
<< "/proc/sys/vm/overcommit_ratio\"'";
276-
}
277-
}
278-
}
279-
} catch (...) {
280-
// file not found or value not convertible into integer
281-
}
282-
#endif
283-
}

‎lib/ApplicationFeatures/EnvironmentFeature.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ class EnvironmentFeature final : public application_features::ApplicationFeature
3232

3333
public:
3434
void prepare() override final;
35-
void start() override final;
3635
};
3736
}
3837

0 commit comments

Comments
 (0)
0