Skip to content

Commit

Permalink
Merge branch 'issue_7812_2.2.x' of github.com:orientechnologies/orien…
Browse files Browse the repository at this point in the history 
…tdb into issue_7812_2.2.x
  • Loading branch information
@andrii0lomakin
andrii0lomakin committed Oct 26, 2017
2 parents 35bcf1a + 4fe171b commit 1b30b2c
Show file tree
Hide file tree
Showing 3 changed files with 240 additions and 35 deletions.
256 changes: 226 additions & 30 deletions core/src/main/java/com/orientechnologies/common/jna/ONative.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@
import com.sun.jna.Platform;

import javax.management.*;
import java.io.*;
import java.lang.management.ManagementFactory;

public class ONative {
private static final OCLibrary C_LIBRARY;
private static final String DEFAULT_MEMORY_CGROUP_PATH = "/sys/fs/memory";

static {
if (Platform.isLinux()) {
Expand All @@ -23,47 +25,241 @@ public static ONative instance() {
return InstanceHolder.INSTANCE;
}

public boolean isLinux() {
return Platform.isLinux();
}

/**
* @return PID if current functionality supported on running platform.
* @param printSteps Print all steps of discovering of memory limit in the log with {@code INFO} level.
*
* @return Amount of memory which are allowed to be consumed by application, if value <= 0 then limit is not set.
*/
public long getPid() {
if (C_LIBRARY != null) {
return C_LIBRARY.getpid();
} else {
final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
final int index = jvmName.indexOf('@');
public long getMemoryLimit(boolean printSteps) {
long memoryLimit = getPhysicalMemorySize();

if (index < 1) {
return -1;
} else {
if (memoryLimit > 0 && printSteps) {
OLogManager.instance().info(this, "%d bytes of physical memory were detected on machine", memoryLimit);
}

if (Platform.isLinux()) {
final OCLibrary.Rlimit rlimit = new OCLibrary.Rlimit();
int result = C_LIBRARY.getrlimit(OCLibrary.RLIMIT_AS, rlimit);

if (result == 0 && rlimit.rlim_cur > 0) {
if (printSteps)
OLogManager.instance().info(this, "Soft memory limit for this process is set to %d bytes", rlimit.rlim_cur);

if (rlimit.rlim_cur < memoryLimit || memoryLimit <= 0) {
memoryLimit = rlimit.rlim_cur;
}

}

String memoryCGroupPath = null;

final File cgroupList = new File("/proc/self/cgroup");
if (cgroupList.exists()) {
try {
return Long.parseLong(jvmName.substring(0, index));
} catch (NumberFormatException e) {
return -1;
final FileReader cgroupListReader = new FileReader(cgroupList);
final BufferedReader bufferedCGroupReader = new BufferedReader(cgroupListReader);
try {
String cgroupData;
try {
while ((cgroupData = bufferedCGroupReader.readLine()) != null) {
final String[] cgroupParts = cgroupData.split(":");
if (cgroupParts[1].equals("memory")) {
memoryCGroupPath = cgroupParts[2];
}
}
} catch (IOException ioe) {
OLogManager.instance().error(this, "Error during reading of details of list of cgroups for the current process, "
+ "no restrictions applied by cgroups will be taken into account", ioe);
memoryCGroupPath = null;
}

} finally {
try {
bufferedCGroupReader.close();
} catch (IOException ioe) {
OLogManager.instance()
.error(this, "Error during closing of reader which reads details of list of cgroups for the current process",
ioe);
}
}
} catch (FileNotFoundException fnfe) {
OLogManager.instance().warn(this, "Can not retrieve list of cgroups to which process belongs, "
+ "no restrictions applied by cgroups will be taken into account");
}
}
}
}

/**
* @return Amount of memory which are allowed to be consumed by application.
*/
public long getMemoryLimit() {
long memoryLimit = 0;
if (memoryCGroupPath != null) {
if (printSteps)
OLogManager.instance().info(this, "Path to 'memory' cgroup is %s, reading memory limits", memoryCGroupPath);

if (Platform.isLinux()) {
final OCLibrary.Rlimit rlimit = new OCLibrary.Rlimit();
C_LIBRARY.getrlimit(OCLibrary.RLIMIT_AS, rlimit);
String memoryCGroupRoot = null;

//find all mounting points
final File procMounts = new File("/proc/mounts");
if (procMounts.exists()) {
FileReader mountsReader;
try {
mountsReader = new FileReader(procMounts);
BufferedReader bufferedMountsReader = new BufferedReader(mountsReader);

try {
String fileSystem;
while ((fileSystem = bufferedMountsReader.readLine()) != null) {
final String[] fsParts = fileSystem.split("\\s+");
if (fsParts.length == 0) {
continue;
}

final String fsType = fsParts[0];
//all cgroup controllers have "cgroup" as file system type
if (fsType.equals("cgroup")) {
//get mounting path of cgroup
final String fsMountingPath = fsParts[1];
final String[] fsPathParts = fsMountingPath.split("/");
if (fsPathParts[fsPathParts.length - 1].equals("memory")) {
memoryCGroupRoot = fsMountingPath;
}
}
}
} catch (IOException e) {
OLogManager.instance().error(this, "Error during reading a list of mounted file systems", e);
memoryCGroupRoot = DEFAULT_MEMORY_CGROUP_PATH;
} finally {
try {
bufferedMountsReader.close();
} catch (IOException e) {
OLogManager.instance().error(this, "Error during closing of reader of list of mounted file systems", e);
}
}

} catch (FileNotFoundException fnfe) {
memoryCGroupRoot = DEFAULT_MEMORY_CGROUP_PATH;
}
}

if (memoryCGroupRoot == null) {
memoryCGroupRoot = DEFAULT_MEMORY_CGROUP_PATH;
}

if (printSteps)
OLogManager.instance().info(this, "Mounting path for memory cgroup controller is %s", memoryCGroupRoot);

File memoryCGroup = new File(memoryCGroupRoot, memoryCGroupPath);
if (!memoryCGroup.exists()) {
if (printSteps)
OLogManager.instance().info(this, "Can not find '%s' path for memory cgroup, it is supposed that "
+ "process is running in container, will try to read root '%s' memory cgroup data", memoryCGroupRoot);

memoryCGroup = new File(memoryCGroupRoot);
}

boolean readHardLimit = false;
File softMemoryCGroupLimit = new File(memoryCGroup, "memory.soft_limit_in_bytes");
if (softMemoryCGroupLimit.exists()) {
try {
final FileReader memoryLimitReader = new FileReader(softMemoryCGroupLimit);
final BufferedReader bufferedMemoryLimitReader = new BufferedReader(memoryLimitReader);
try {
try {
final String cgroupMemoryLimitValueStr = bufferedMemoryLimitReader.readLine();
try {
final long cgroupMemoryLimitValue = Long.parseLong(cgroupMemoryLimitValueStr);

if (printSteps)
OLogManager.instance().info(this, "cgroup soft memory limit is %d", cgroupMemoryLimitValue);

if (cgroupMemoryLimitValue < memoryLimit) {
memoryLimit = cgroupMemoryLimitValue;
}
} catch (NumberFormatException nfe) {
OLogManager.instance()
.error(this, "Can not read memory soft limit for cgroup %s, will try to read memory hard limit", nfe,
memoryCGroup);
readHardLimit = true;
}
} catch (IOException ioe) {
OLogManager.instance()
.error(this, "Can not read memory soft limit for cgroup %s, will try to read memory hard limit", ioe,
memoryCGroup);
readHardLimit = true;
}
} finally {
try {
bufferedMemoryLimitReader.close();
} catch (IOException ioe) {
OLogManager.instance().error(this, "Error on closing the reader of soft memory limit", ioe);
}
}
} catch (FileNotFoundException fnfe) {
OLogManager.instance()
.error(this, "Can not read memory soft limit for cgroup %s, will try to read memory hard limit", fnfe,
memoryCGroup);
readHardLimit = true;
}
} else {
if (printSteps)
OLogManager.instance()
.info(this, "Can not read memory soft limit for cgroup %s, will try to read memory hard limit", memoryCGroup);

readHardLimit = true;
}

memoryLimit = rlimit.rlim_cur;
if (readHardLimit) {
final File hardMemoryCGroupLimit = new File(memoryCGroup, "memory.limit_in_bytes");
if (hardMemoryCGroupLimit.exists()) {
try {
final FileReader memoryLimitReader = new FileReader(softMemoryCGroupLimit);
final BufferedReader bufferedMemoryLimitReader = new BufferedReader(memoryLimitReader);
try {
try {
final String cgroupMemoryLimitValueStr = bufferedMemoryLimitReader.readLine();
try {
final long cgroupMemoryLimitValue = Long.parseLong(cgroupMemoryLimitValueStr);

if (printSteps)
OLogManager.instance().info(this, "cgroup hard memory limit is %d", cgroupMemoryLimitValue);

if (cgroupMemoryLimitValue < memoryLimit) {
memoryLimit = cgroupMemoryLimitValue;
}
} catch (NumberFormatException nfe) {
OLogManager.instance().error(this,
"Can not read memory hard limit for cgroup %s, cgroup memory limits for current "
+ "process will not be applied", nfe, memoryCGroup);
}
} catch (IOException ioe) {
OLogManager.instance().error(this,
"Can not read memory hard limit for cgroup %s, cgroup memory limits for current process"
+ " will not be applied", ioe, memoryCGroup);
}
} finally {
try {
bufferedMemoryLimitReader.close();
} catch (IOException ioe) {
OLogManager.instance().error(this, "Error on closing the reader of hard memory limit", ioe);
}
}
} catch (FileNotFoundException fnfe) {
OLogManager.instance().error(this,
"Can not read memory hard limit for cgroup %s, cgroup memory limits for current process will not be applied",
fnfe, memoryCGroup);
}
} else {
if (printSteps) {
OLogManager.instance().info(this,
"Can not read memory hard limit for cgroup %s, cgroup memory limits for current process will not be applied",
memoryCGroup);
}
}
}
}
}

if (memoryLimit <= 0) {
memoryLimit = getPhysicalMemorySize();
if (printSteps) {
if (memoryLimit > 0)
OLogManager.instance().info(this, "Detected memory limit for current process is %d", memoryLimit);
else
OLogManager.instance().info(this, "Memory limit for current process is not set");
}

return memoryLimit;
Expand Down
8 changes: 4 additions & 4 deletions core/src/main/java/com/orientechnologies/common/util/OMemory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,11 @@ private static long getMaxCacheMemorySize() {
* Checks the direct memory configuration and emits a warning if configuration is invalid.
*/
public static void checkDirectMemoryConfiguration() {
final long physicalMemory = ONative.instance().getMemoryLimit();
final long physicalMemory = ONative.instance().getMemoryLimit(false);
final long maxDirectMemory = getConfiguredMaxDirectMemory();

if (maxDirectMemory == -1) {
if (physicalMemory != -1)
if (physicalMemory > 0)
OLogManager.instance().warn(OMemory.class, "MaxDirectMemorySize JVM option is not set or has invalid value, "
+ "that may cause out of memory errors. Please set the -XX:MaxDirectMemorySize=" + physicalMemory / (1024 * 1024)
+ "m option when you start the JVM.");
Expand All @@ -123,7 +123,7 @@ public static void checkDirectMemoryConfiguration() {
public static void checkCacheMemoryConfiguration() {
final long maxHeapSize = Runtime.getRuntime().maxMemory();
final long maxCacheSize = getMaxCacheMemorySize();
final long physicalMemory = ONative.instance().getMemoryLimit();
final long physicalMemory = ONative.instance().getMemoryLimit(false);
final long maxDirectMemory = getConfiguredMaxDirectMemory();

if (maxDirectMemory != -1 && maxCacheSize > maxDirectMemory)
Expand All @@ -133,7 +133,7 @@ public static void checkCacheMemoryConfiguration() {
+ "maximum direct memory size or storage.diskCache.bufferSize OrientDB option to lower memory requirements of the "
+ "cache.");

if (maxHeapSize != Long.MAX_VALUE && physicalMemory != -1 && maxHeapSize + maxCacheSize > physicalMemory)
if (maxHeapSize != Long.MAX_VALUE && physicalMemory > 0 && maxHeapSize + maxCacheSize > physicalMemory)
OLogManager.instance().warn(OMemory.class,
"The sum of the configured JVM maximum heap size (" + maxHeapSize + " bytes) " + "and the OrientDB maximum cache size ("
+ maxCacheSize + " bytes) is larger than the available physical memory size " + "(" + physicalMemory
Expand Down
11 changes: 10 additions & 1 deletion .../com/orientechnologies/orient/core/engine/OMemoryAndLocalPaginatedEnginesInitializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package com.orientechnologies.orient.core.engine;

import com.orientechnologies.common.collection.OLRUCache;
import com.orientechnologies.common.io.OFileUtils;
import com.orientechnologies.common.jna.ONative;
import com.orientechnologies.common.log.OLogManager;
Expand Down Expand Up @@ -56,11 +57,19 @@ public void initialize() {
OMemory.checkCacheMemoryConfiguration();

OMemory.fixCommonConfigurationProblems();

OLogManager.instance().info(this, "Disk cache size is %d mb", OGlobalConfiguration.DISK_CACHE_SIZE.getValueAsLong());
}

private void configureDefaults() {
OLogManager.instance().info(this, "Auto configuration of disk cache size.");

if (!OGlobalConfiguration.DISK_CACHE_SIZE.isChanged())
configureDefaultDiskCacheSize();
else {
OLogManager.instance()
.info(this, "Disk cache size is directly set by the user to the %d mb", OGlobalConfiguration.DISK_CACHE_SIZE.getValue());
}

if (!OGlobalConfiguration.WAL_RESTORE_BATCH_SIZE.isChanged())
configureDefaultWalRestoreBatchSize();
Expand All @@ -77,7 +86,7 @@ else if (jvmMaxMemory > 512 * OFileUtils.MEGABYTE)
}

private void configureDefaultDiskCacheSize() {
final long osMemory = ONative.instance().getMemoryLimit();
final long osMemory = ONative.instance().getMemoryLimit(true);
final long jvmMaxMemory = OMemory.getCappedRuntimeMaxMemory(2L * 1024 * 1024 * 1024 /* 2GB */);
final long maxDirectMemory = OMemory.getConfiguredMaxDirectMemory();

Expand Down

0 comments on commit 1b30b2c

Please sign in to comment.