共计 12015 个字符，预计需要花费 31 分钟才能阅读完成。
序
本文主要研究一下 netty 的 maxDirectMemory
PlatformDependent
netty-common-4.1.33.Final-sources.jar!/io/netty/util/internal/PlatformDependent.java
public final class PlatformDependent {
private static final InternalLogger logger = InternalLoggerFactory.getInstance(PlatformDependent.class);
private static final Pattern MAX_DIRECT_MEMORY_SIZE_ARG_PATTERN = Pattern.compile(
“\\s*-XX:MaxDirectMemorySize\\s*=\\s*([0-9]+)\\s*([kKmMgG]?)\\s*$”);
private static final boolean IS_WINDOWS = isWindows0();
private static final boolean IS_OSX = isOsx0();
private static final boolean MAYBE_SUPER_USER;
private static final boolean CAN_ENABLE_TCP_NODELAY_BY_DEFAULT = !isAndroid();
private static final Throwable UNSAFE_UNAVAILABILITY_CAUSE = unsafeUnavailabilityCause0();
private static final boolean DIRECT_BUFFER_PREFERRED;
private static final long MAX_DIRECT_MEMORY = maxDirectMemory0();
//……
static {
if (javaVersion() >= 7) {
RANDOM_PROVIDER = new ThreadLocalRandomProvider() {
@Override
public Random current() {
return java.util.concurrent.ThreadLocalRandom.current();
}
};
} else {
RANDOM_PROVIDER = new ThreadLocalRandomProvider() {
@Override
public Random current() {
return ThreadLocalRandom.current();
}
};
}
// Here is how the system property is used:
//
// * < 0 – Don’t use cleaner, and inherit max direct memory from java. In this case the
// “practical max direct memory” would be 2 * max memory as defined by the JDK.
// * == 0 – Use cleaner, Netty will not enforce max memory, and instead will defer to JDK.
// * > 0 – Don’t use cleaner. This will limit Netty’s total direct memory
// (note: that JDK’s direct memory limit is independent of this).
long maxDirectMemory = SystemPropertyUtil.getLong(“io.netty.maxDirectMemory”, -1);
if (maxDirectMemory == 0 || !hasUnsafe() || !PlatformDependent0.hasDirectBufferNoCleanerConstructor()) {
USE_DIRECT_BUFFER_NO_CLEANER = false;
DIRECT_MEMORY_COUNTER = null;
} else {
USE_DIRECT_BUFFER_NO_CLEANER = true;
if (maxDirectMemory < 0) {
maxDirectMemory = MAX_DIRECT_MEMORY;
if (maxDirectMemory <= 0) {
DIRECT_MEMORY_COUNTER = null;
} else {
DIRECT_MEMORY_COUNTER = new AtomicLong();
}
} else {
DIRECT_MEMORY_COUNTER = new AtomicLong();
}
}
logger.debug(“-Dio.netty.maxDirectMemory: {} bytes”, maxDirectMemory);
DIRECT_MEMORY_LIMIT = maxDirectMemory >= 1 ? maxDirectMemory : MAX_DIRECT_MEMORY;
int tryAllocateUninitializedArray =
SystemPropertyUtil.getInt(“io.netty.uninitializedArrayAllocationThreshold”, 1024);
UNINITIALIZED_ARRAY_ALLOCATION_THRESHOLD = javaVersion() >= 9 && PlatformDependent0.hasAllocateArrayMethod() ?
tryAllocateUninitializedArray : -1;
logger.debug(“-Dio.netty.uninitializedArrayAllocationThreshold: {}”, UNINITIALIZED_ARRAY_ALLOCATION_THRESHOLD);
MAYBE_SUPER_USER = maybeSuperUser0();
if (!isAndroid()) {
// only direct to method if we are not running on android.
// See https://github.com/netty/netty/issues/2604
if (javaVersion() >= 9) {
CLEANER = CleanerJava9.isSupported() ? new CleanerJava9() : NOOP;
} else {
CLEANER = CleanerJava6.isSupported() ? new CleanerJava6() : NOOP;
}
} else {
CLEANER = NOOP;
}
// We should always prefer direct buffers by default if we can use a Cleaner to release direct buffers.
DIRECT_BUFFER_PREFERRED = CLEANER != NOOP
&& !SystemPropertyUtil.getBoolean(“io.netty.noPreferDirect”, false);
if (logger.isDebugEnabled()) {
logger.debug(“-Dio.netty.noPreferDirect: {}”, !DIRECT_BUFFER_PREFERRED);
}
/*
* We do not want to log this message if unsafe is explicitly disabled. Do not remove the explicit no unsafe
* guard.
*/
if (CLEANER == NOOP && !PlatformDependent0.isExplicitNoUnsafe()) {
logger.info(
“Your platform does not provide complete low-level API for accessing direct buffers reliably. ” +
“Unless explicitly requested, heap buffer will always be preferred to avoid potential system ” +
“instability.”);
}
}
private static long maxDirectMemory0() {
long maxDirectMemory = 0;
ClassLoader systemClassLoader = null;
try {
systemClassLoader = getSystemClassLoader();
// When using IBM J9 / Eclipse OpenJ9 we should not use VM.maxDirectMemory() as it not reflects the
// correct value.
// See:
// – https://github.com/netty/netty/issues/7654
String vmName = SystemPropertyUtil.get(“java.vm.name”, “”).toLowerCase();
if (!vmName.startsWith(“ibm j9”) &&
// https://github.com/eclipse/openj9/blob/openj9-0.8.0/runtime/include/vendor_version.h#L53
!vmName.startsWith(“eclipse openj9”)) {
// Try to get from sun.misc.VM.maxDirectMemory() which should be most accurate.
Class<?> vmClass = Class.forName(“sun.misc.VM”, true, systemClassLoader);
Method m = vmClass.getDeclaredMethod(“maxDirectMemory”);
maxDirectMemory = ((Number) m.invoke(null)).longValue();
}
} catch (Throwable ignored) {
// Ignore
}
if (maxDirectMemory > 0) {
return maxDirectMemory;
}
try {
// Now try to get the JVM option (-XX:MaxDirectMemorySize) and parse it.
// Note that we are using reflection because Android doesn’t have these classes.
Class<?> mgmtFactoryClass = Class.forName(
“java.lang.management.ManagementFactory”, true, systemClassLoader);
Class<?> runtimeClass = Class.forName(
“java.lang.management.RuntimeMXBean”, true, systemClassLoader);
Object runtime = mgmtFactoryClass.getDeclaredMethod(“getRuntimeMXBean”).invoke(null);
@SuppressWarnings(“unchecked”)
List<String> vmArgs = (List<String>) runtimeClass.getDeclaredMethod(“getInputArguments”).invoke(runtime);
for (int i = vmArgs.size() – 1; i >= 0; i –) {
Matcher m = MAX_DIRECT_MEMORY_SIZE_ARG_PATTERN.matcher(vmArgs.get(i));
if (!m.matches()) {
continue;
}
maxDirectMemory = Long.parseLong(m.group(1));
switch (m.group(2).charAt(0)) {
case ‘k’: case ‘K’:
maxDirectMemory *= 1024;
break;
case ‘m’: case ‘M’:
maxDirectMemory *= 1024 * 1024;
break;
case ‘g’: case ‘G’:
maxDirectMemory *= 1024 * 1024 * 1024;
break;
}
break;
}
} catch (Throwable ignored) {
// Ignore
}
if (maxDirectMemory <= 0) {
maxDirectMemory = Runtime.getRuntime().maxMemory();
logger.debug(“maxDirectMemory: {} bytes (maybe)”, maxDirectMemory);
} else {
logger.debug(“maxDirectMemory: {} bytes”, maxDirectMemory);
}
return maxDirectMemory;
}
/**
* Returns the maximum memory reserved for direct buffer allocation.
*/
public static long maxDirectMemory() {
return DIRECT_MEMORY_LIMIT;
}
//……
}
netty 的 PlatformDependent 有个静态属性 MAX_DIRECT_MEMORY，它是根据 maxDirectMemory0 方法来计算的
maxDirectMemory0 方法会根据 jvm 的类型来做不同处理，如果是 IBM J9 / Eclipse OpenJ9 的话，就不能使用 VM.maxDirectMemory() 来获取，正常 hotspot 则采用 VM.maxDirectMemory() 来获取 (VM.maxDirectMemory 是读取 -XX:MaxDirectMemorySize 配置，如果有设置且大于 0 则使用该值，如果没有设置该参数则默认值为 0，则默认是取的 Runtime.getRuntime().maxMemory())
static 代码块里头设置了 DIRECT_MEMORY_LIMIT；它首先从系统属性读取 io.netty.maxDirectMemory 到 maxDirectMemory，如果 maxDirectMemory 值小于 0，则设置 maxDirectMemory 为 MAX_DIRECT_MEMORY；DIRECT_MEMORY_LIMIT = maxDirectMemory >= 1 ? maxDirectMemory : MAX_DIRECT_MEMORY；maxDirectMemory 方法直接返回 DIRECT_MEMORY_LIMIT
ByteBuffer.allocateDirect
java.base/java/nio/ByteBuffer.java
public abstract class ByteBuffer
extends Buffer
implements Comparable<ByteBuffer>
{
//……
/**
* Allocates a new direct byte buffer.
*
* <p> The new buffer’s position will be zero, its limit will be its
* capacity, its mark will be undefined, each of its elements will be
* initialized to zero, and its byte order will be
* {@link ByteOrder#BIG_ENDIAN BIG_ENDIAN}. Whether or not it has a
* {@link #hasArray backing array} is unspecified.
*
* @param capacity
* The new buffer’s capacity, in bytes
*
* @return The new byte buffer
*
* @throws IllegalArgumentException
* If the {@code capacity} is a negative integer
*/
public static ByteBuffer allocateDirect(int capacity) {
return new DirectByteBuffer(capacity);
}
//……
}
ByteBuffer.allocateDirect 方法实际是创建了 DirectByteBuffer
DirectByteBuffer
java.base/java/nio/DirectByteBuffer.java
class DirectByteBuffer extends MappedByteBuffer implements DirectBuffer {
//……
// Primary constructor
//
DirectByteBuffer(int cap) {// package-private
super(-1, 0, cap, cap);
boolean pa = VM.isDirectMemoryPageAligned();
int ps = Bits.pageSize();
long size = Math.max(1L, (long)cap + (pa ? ps : 0));
Bits.reserveMemory(size, cap);
long base = 0;
try {
base = UNSAFE.allocateMemory(size);
} catch (OutOfMemoryError x) {
Bits.unreserveMemory(size, cap);
throw x;
}
UNSAFE.setMemory(base, size, (byte) 0);
if (pa && (base % ps != 0)) {
// Round up to page boundary
address = base + ps – (base & (ps – 1));
} else {
address = base;
}
cleaner = Cleaner.create(this, new Deallocator(base, size, cap));
att = null;
}
//……
}
DirectByteBuffer 的构造器里头会调用 Bits.reserveMemory，出现 OutOfMemoryError，则调用 Bits.unreserveMemory(size, cap)，然后抛出 OutOfMemoryError
Bits.reserveMemory
java.base/java/nio/Bits.java
/**
* Access to bits, native and otherwise.
*/
class Bits {// package-private
private Bits() {}
// — Direct memory management —
// A user-settable upper limit on the maximum amount of allocatable
// direct buffer memory. This value may be changed during VM
// initialization if it is launched with “-XX:MaxDirectMemorySize=<size>”.
private static volatile long MAX_MEMORY = VM.maxDirectMemory();
private static final AtomicLong RESERVED_MEMORY = new AtomicLong();
private static final AtomicLong TOTAL_CAPACITY = new AtomicLong();
private static final AtomicLong COUNT = new AtomicLong();
private static volatile boolean MEMORY_LIMIT_SET;
// max. number of sleeps during try-reserving with exponentially
// increasing delay before throwing OutOfMemoryError:
// 1, 2, 4, 8, 16, 32, 64, 128, 256 (total 511 ms ~ 0.5 s)
// which means that OOME will be thrown after 0.5 s of trying
private static final int MAX_SLEEPS = 9;
//……
// These methods should be called whenever direct memory is allocated or
// freed. They allow the user to control the amount of direct memory
// which a process may access. All sizes are specified in bytes.
static void reserveMemory(long size, int cap) {
if (!MEMORY_LIMIT_SET && VM.initLevel() >= 1) {
MAX_MEMORY = VM.maxDirectMemory();
MEMORY_LIMIT_SET = true;
}
// optimist!
if (tryReserveMemory(size, cap)) {
return;
}
final JavaLangRefAccess jlra = SharedSecrets.getJavaLangRefAccess();
boolean interrupted = false;
try {
// Retry allocation until success or there are no more
// references (including Cleaners that might free direct
// buffer memory) to process and allocation still fails.
boolean refprocActive;
do {
try {
refprocActive = jlra.waitForReferenceProcessing();
} catch (InterruptedException e) {
// Defer interrupts and keep trying.
interrupted = true;
refprocActive = true;
}
if (tryReserveMemory(size, cap)) {
return;
}
} while (refprocActive);
// trigger VM’s Reference processing
System.gc();
// A retry loop with exponential back-off delays.
// Sometimes it would suffice to give up once reference
// processing is complete. But if there are many threads
// competing for memory, this gives more opportunities for
// any given thread to make progress. In particular, this
// seems to be enough for a stress test like
// DirectBufferAllocTest to (usually) succeed, while
// without it that test likely fails. Since failure here
// ends in OOME, there’s no need to hurry.
long sleepTime = 1;
int sleeps = 0;
while (true) {
if (tryReserveMemory(size, cap)) {
return;
}
if (sleeps >= MAX_SLEEPS) {
break;
}
try {
if (!jlra.waitForReferenceProcessing()) {
Thread.sleep(sleepTime);
sleepTime <<= 1;
sleeps++;
}
} catch (InterruptedException e) {
interrupted = true;
}
}
// no luck
throw new OutOfMemoryError(“Direct buffer memory”);
} finally {
if (interrupted) {
// don’t swallow interrupts
Thread.currentThread().interrupt();
}
}
}
private static boolean tryReserveMemory(long size, int cap) {
// -XX:MaxDirectMemorySize limits the total capacity rather than the
// actual memory usage, which will differ when buffers are page
// aligned.
long totalCap;
while (cap <= MAX_MEMORY – (totalCap = TOTAL_CAPACITY.get())) {
if (TOTAL_CAPACITY.compareAndSet(totalCap, totalCap + cap)) {
RESERVED_MEMORY.addAndGet(size);
COUNT.incrementAndGet();
return true;
}
}
return false;
}
//……
}
Bits.reserveMemory 方法会先调用 tryReserveMemory 尝试分配 direct memory，不成功则继续往下执行 do while(refprocActive)
refprocActive 这段循环是不断尝试 allocation 直到分配成功，或者直到没有引用来处理且分配失败
如果 refprocActive 循环没有分配成功，则调用 System.gc()，然后进入最后一段循环尝试分配；最后这段循环如果分配成功则返回，分配不成功且 sleeps 大于等于 MAX_SLEEPS，则跳出循环，最后抛出 OutOfMemoryError(“Direct buffer memory”) 异常
小结
netty 的 PlatformDependent 有个静态属性 MAX_DIRECT_MEMORY，它是根据 maxDirectMemory0 方法来计算的；maxDirectMemory0 方法会根据 jvm 的类型来做不同处理，如果是 IBM J9 / Eclipse OpenJ9 的话，就不能使用 VM.maxDirectMemory() 来获取，正常 hotspot 则采用 VM.maxDirectMemory() 来获取 (VM.maxDirectMemory 是读取 -XX:MaxDirectMemorySize 配置，如果有设置且大于 0 则使用该值，如果没有设置该参数则默认值为 0，则默认是取的 Runtime.getRuntime().maxMemory())
static 代码块里头设置了 DIRECT_MEMORY_LIMIT；它首先从系统属性读取 io.netty.maxDirectMemory 到 maxDirectMemory，如果 maxDirectMemory 值小于 0，则设置 maxDirectMemory 为 MAX_DIRECT_MEMORY；DIRECT_MEMORY_LIMIT = maxDirectMemory >= 1 ? maxDirectMemory : MAX_DIRECT_MEMORY；maxDirectMemory 方法直接返回 DIRECT_MEMORY_LIMIT
ByteBuffer.allocateDirect 方法实际是创建了 DirectByteBuffer；DirectByteBuffer 的构造器里头会调用 Bits.reserveMemory，出现 OutOfMemoryError，则调用 Bits.unreserveMemory(size, cap)，然后抛出 OutOfMemoryError；Bits.reserveMemory 方法会先调用 tryReserveMemory 尝试分配 direct memory，不成功则继续往下执行 do while(refprocActive)；refprocActive 这段循环是不断尝试 allocation 直到分配成功，或者直到没有引用来处理且分配失败；如果 refprocActive 循环没有分配成功，则调用 System.gc()，然后进入最后一段循环尝试分配；最后这段循环如果分配成功则返回，分配不成功且 sleeps 大于等于 MAX_SLEEPS，则跳出循环，最后抛出 OutOfMemoryError(“Direct buffer memory”) 异常
doc
聊聊 jvm 的 -XX:MaxDirectMemorySize
In Netty 4, do I need to set option -XX:MaxDirectMemorySize?
Netty 之 Java 堆外内存扫盲贴
直播一次问题排查过程
Change default value of io.netty.maxDirectMemory ? #6349
LEAK: ByteBuf.release() was not called before it’s garbage-collected #422