mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
opendj-server-legacy/src/main/java/org/opends/server/api/Backend.java
@@ -114,7 +114,7 @@ /** * Temporarily sets up the server context for the first phase of add of a new configuration entry. * Will be needed for checking storage parameters before committing the change in configuration. * Needed for checking storage parameters before committing the change in configuration. * * @param context the server context for this instance */ opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/ConfigurableEnvironment.java
@@ -41,6 +41,8 @@ import org.opends.server.admin.std.meta.LocalDBBackendCfgDefn; import org.opends.server.admin.std.server.LocalDBBackendCfg; import org.opends.server.config.ConfigConstants; import org.opends.server.core.DirectoryServer; import org.opends.server.core.MemoryQuota; import org.forgerock.opendj.config.server.ConfigException; import com.sleepycat.je.Durability; import com.sleepycat.je.EnvironmentConfig; @@ -470,6 +472,12 @@ ERR_CONFIG_JEB_CACHE_SIZE_TOO_SMALL.get( cfg.getDBCacheSize(), MemoryBudget.MIN_MAX_MEMORY_SIZE)); } MemoryQuota memoryQuota = DirectoryServer.getInstance().getServerContext().getMemoryQuota(); if (!memoryQuota.acquireMemory(cfg.getDBCacheSize())) { logger.warn(ERR_CONFIG_JEB_CACHE_SIZE_GREATER_THAN_JVM_HEAP.get( cfg.getDBCacheSize(), memoryQuota.getMaxMemory())); } } EnvironmentConfig envConfig = defaultConfig(); opendj-server-legacy/src/main/java/org/opends/server/backends/persistit/PersistItStorage.java
@@ -30,6 +30,7 @@ import static org.opends.messages.ConfigMessages.*; import static org.opends.messages.JebMessages.*; import static org.opends.messages.BackendMessages.*; import static org.opends.server.util.ServerConstants.*; import static org.opends.server.util.StaticUtils.*; @@ -62,6 +63,7 @@ import org.opends.server.backends.pluggable.spi.WriteOperation; import org.opends.server.backends.pluggable.spi.WriteableStorage; import org.opends.server.core.DirectoryServer; import org.opends.server.core.MemoryQuota; import org.opends.server.core.ServerContext; import org.opends.server.extensions.DiskSpaceMonitor; import org.opends.server.types.DN; @@ -519,6 +521,7 @@ private Configuration dbCfg; private PersistitBackendCfg config; private DiskSpaceMonitor diskMonitor; private MemoryQuota memQuota; /** * Creates a new persistit storage with the provided configuration. @@ -541,13 +544,17 @@ BUFFER_SIZE, 4096, Long.MAX_VALUE / BUFFER_SIZE, 2048, true, false, false))); final BufferPoolConfiguration bufferPoolCfg = getBufferPoolCfg(); bufferPoolCfg.setMaximumCount(Integer.MAX_VALUE); memQuota = serverContext.getMemoryQuota(); if (cfg.getDBCacheSize() > 0) { bufferPoolCfg.setMaximumMemory(cfg.getDBCacheSize()); memQuota.acquireMemory(cfg.getDBCacheSize()); } else { bufferPoolCfg.setFraction(cfg.getDBCachePercent() / 100.0f); memQuota.acquireMemory(memQuota.memPercentToBytes(cfg.getDBCachePercent())); } dbCfg.setCommitPolicy(cfg.isDBTxnNoSync() ? SOFT : GROUP); cfg.addPersistitChangeListener(this); @@ -569,6 +576,14 @@ throw new IllegalStateException(e); } } if (config.getDBCacheSize() > 0) { memQuota.releaseMemory(config.getDBCacheSize()); } else { memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); } config.removePersistitChangeListener(this); DirectoryServer.deregisterMonitorProvider(diskMonitor); DirectoryServer.deregisterAlertGenerator(this); @@ -792,6 +807,41 @@ @Override public boolean isConfigurationChangeAcceptable(PersistitBackendCfg cfg, List<LocalizableMessage> unacceptableReasons) { return checkConfigurationDirectories(cfg, unacceptableReasons); } /** * Checks newly created backend has a valid configuration. * @param cfg the new configuration * @param unacceptableReasons the list of accumulated errors and their messages * @param context TODO * @return true if newly created backend has a valid configuration */ public static boolean isConfigurationAcceptable(PersistitBackendCfg cfg, List<LocalizableMessage> unacceptableReasons, ServerContext context) { if (context != null) { MemoryQuota memQuota = context.getMemoryQuota(); if (cfg.getDBCacheSize() > 0 && !memQuota.isMemoryAvailable(cfg.getDBCacheSize())) { unacceptableReasons.add(ERR_BACKEND_CONFIG_CACHE_SIZE_GREATER_THAN_JVM_HEAP.get( cfg.getDBCacheSize(), memQuota.getAvailableMemory())); return false; } else if (!memQuota.isMemoryAvailable(memQuota.memPercentToBytes(cfg.getDBCachePercent()))) { unacceptableReasons.add(ERR_BACKEND_CONFIG_CACHE_PERCENT_GREATER_THAN_JVM_HEAP.get( cfg.getDBCachePercent(), memQuota.memBytesToPercent(memQuota.getAvailableMemory()))); return false; } } return checkConfigurationDirectories(cfg, unacceptableReasons); } private static boolean checkConfigurationDirectories(PersistitBackendCfg cfg, List<LocalizableMessage> unacceptableReasons) { final ConfigChangeResult ccr = new ConfigChangeResult(); File parentDirectory = getFileForPath(cfg.getDBDirectory()); File newBackendDirectory = new File(parentDirectory, cfg.getBackendId()); @@ -813,7 +863,7 @@ * @param ccr the list of reasons to return upstream or null if called from setupStorage() * @param cleanup true if the directory should be deleted after creation */ private void checkDBDirExistsOrCanCreate(File backendDir, ConfigChangeResult ccr, boolean cleanup) private static void checkDBDirExistsOrCanCreate(File backendDir, ConfigChangeResult ccr, boolean cleanup) { if (!backendDir.exists()) { @@ -840,7 +890,7 @@ * @param ccr the current list of change results * @throws forwards a file exception */ private void checkDBDirPermissions(PersistitBackendCfg cfg, ConfigChangeResult ccr) private static void checkDBDirPermissions(PersistitBackendCfg cfg, ConfigChangeResult ccr) { try { @@ -887,7 +937,7 @@ } } private FilePermission decodeDBDirPermissions(PersistitBackendCfg curCfg) throws ConfigException private static FilePermission decodeDBDirPermissions(PersistitBackendCfg curCfg) throws ConfigException { try { @@ -947,7 +997,7 @@ return ccr; } private void addErrorMessage(final ConfigChangeResult ccr, LocalizableMessage message) private static void addErrorMessage(final ConfigChangeResult ccr, LocalizableMessage message) { ccr.setResultCode(DirectoryServer.getServerErrorResultCode()); ccr.addMessage(message); opendj-server-legacy/src/main/java/org/opends/server/backends/persistit/PitBackend.java
@@ -26,6 +26,9 @@ package org.opends.server.backends.persistit; import java.util.List; import org.forgerock.i18n.LocalizableMessage; import org.forgerock.opendj.config.server.ConfigException; import org.opends.server.admin.std.server.PersistitBackendCfg; import org.opends.server.backends.pluggable.BackendImpl; @@ -37,6 +40,12 @@ public final class PitBackend extends BackendImpl<PersistitBackendCfg> { @Override public boolean isConfigurationAcceptable(PersistitBackendCfg cfg, List<LocalizableMessage> unacceptableReasons) { return PersistItStorage.isConfigurationAcceptable(cfg, unacceptableReasons, serverContext); } @Override protected Storage configureStorage(PersistitBackendCfg cfg) throws ConfigException { return new PersistItStorage(cfg, serverContext); opendj-server-legacy/src/main/java/org/opends/server/core/BackendConfigManager.java
@@ -680,10 +680,10 @@ // backend implementation, then log an error and skip it. String className = configEntry.getJavaClass(); Backend<?> backend; Backend<BackendCfg> backend; try { backend = loadBackendClass(className).newInstance(); backend = (Backend<BackendCfg>)loadBackendClass(className).newInstance(); } catch (Exception e) { @@ -715,6 +715,11 @@ } } backend.setServerContext(serverContext); if (!backend.isConfigurationAcceptable(configEntry, unacceptableReason)) { return false; } // If we've gotten to this point, then it is acceptable as far as we are // concerned. If it is unacceptable according to the configuration for that opendj-server-legacy/src/main/java/org/opends/server/core/DirectoryServer.java
@@ -794,6 +794,9 @@ /** The writability mode for the Directory Server. */ private WritabilityMode writabilityMode; /** The memory reservation system */ private MemoryQuota memoryQuota; /** * The maximum size that internal buffers will be allowed to grow to until * they are trimmed. @@ -890,6 +893,11 @@ return serverManagementContext; } @Override public MemoryQuota getMemoryQuota() { return directoryServer.memoryQuota; } } @@ -921,6 +929,7 @@ operatingSystem = OperatingSystem.forName(System.getProperty("os.name")); serverContext = new DirectoryServerContext(); virtualAttributeConfigManager = new VirtualAttributeConfigManager(serverContext); memoryQuota = new MemoryQuota(); }
New file @@ -0,0 +1,169 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt * or http://forgerock.org/license/CDDLv1.0.html. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at legal-notices/CDDLv1_0.txt. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: * Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * * Copyright 2015 ForgeRock AS. */ package org.opends.server.core; import java.lang.management.ManagementFactory; import java.lang.management.MemoryPoolMXBean; import java.lang.management.MemoryUsage; import java.util.List; import java.util.concurrent.Semaphore; import static org.opends.server.util.ServerConstants.*; /** * Estimates the amount of memory in the running JVM for use of long term caches * by looking at the Old Generation, where implemented, or at the Runtime * information as fallback. Allows for reserving memory to avoid over commitment. * There is a fudge factor involved, so it is not byte exact. */ public final class MemoryQuota { private static final double INIT_FUDGE_FACTOR = 0.9; private static final double FUDGE_FACTOR = 1.3; private static final long ONE_MEGABYTE = 1024 * 1024; private Semaphore reservedMemory; private int reservableMemory; private boolean allowOvercommit; /** * Returns the memory quota reservation system for this server instance. */ public MemoryQuota() { allowOvercommit = System.getProperty(ENABLE_MEMORY_OVERCOMMIT) != null; reservableMemory = (int)(INIT_FUDGE_FACTOR * (getOldGenInfo().getMax() / ONE_MEGABYTE)); reservedMemory = new Semaphore(reservableMemory, true); } /** * Returns the maximum amount of memory the server will use when giving quotas. * @return the maximum amount of memory the server will use when giving quotas */ public long getMaxMemory() { return getOldGenInfo().getMax(); } private MemoryUsage getOldGenInfo() { List<MemoryPoolMXBean> mpools = ManagementFactory.getMemoryPoolMXBeans(); for (MemoryPoolMXBean mpool : mpools) { MemoryUsage usage = mpool.getUsage(); if (usage != null) { if (mpool.getName().endsWith("Old Gen")) { return usage; } } } Runtime runtime = Runtime.getRuntime(); return new MemoryUsage(0, runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory(), runtime.totalMemory(), runtime.maxMemory()); } /** * Check enough memory is available in the reservable pool. * @param size the amount of requested memory * @return true if enough memory is available in the reservable pool */ public boolean isMemoryAvailable(long size) { if (allowOvercommit) { return true; } if (acquireMemory(size)) { releaseMemory(size); return true; } return false; } /** * Reserves the requested amount of memory in OldGen. * * @param size the requested amount of memory in bytes * @return true if the requested amount of memory in OldGen could be reserved */ public boolean acquireMemory(long size) { if (allowOvercommit) { return true; } return reservedMemory.tryAcquire((int)(FUDGE_FACTOR * size / ONE_MEGABYTE)); } /** * Returns how much memory is currently not reserved (free) in OldGen. * @return how much memory is currently not reserved (free) in OldGen */ public long getAvailableMemory() { if (allowOvercommit) { return reservableMemory * ONE_MEGABYTE; } return reservedMemory.availablePermits() * ONE_MEGABYTE; } /** * Translates bytes to percent of reservable memory. * @param size the amount of memory in bytes * @return percent of reservable memory */ public int memBytesToPercent(long size) { return (int)(((size / ONE_MEGABYTE) * 100) / reservableMemory); } /** * Translates a percentage of memory to the equivalent number of bytes. * @param percent a percentage of memory * @return the equivalent number of bytes */ public long memPercentToBytes(int percent) { return (reservableMemory * percent / 100) * ONE_MEGABYTE; } /** * Declares OldGen memory is not needed anymore. * @param size the amount of memory to return */ public void releaseMemory(long size) { if (allowOvercommit) { return; } reservedMemory.release((int)(FUDGE_FACTOR * size / ONE_MEGABYTE)); } } opendj-server-legacy/src/main/java/org/opends/server/core/ServerContext.java
@@ -80,4 +80,10 @@ */ ServerManagementContext getServerManagementContext(); /** * Returns the memory quota system for reserving long term memory. * * @return the memory quota system */ MemoryQuota getMemoryQuota(); } opendj-server-legacy/src/main/java/org/opends/server/util/ServerConstants.java
@@ -3152,6 +3152,9 @@ */ public static final String DN_EXTERNAL_CHANGELOG_ROOT = "cn=changelog"; /** * Enable overcommit of memory in Old Gen space. */ public static final String ENABLE_MEMORY_OVERCOMMIT = "org.forgerock.opendj.EnableMemoryOvercommit"; }
@@ -20,7 +20,7 @@ # CDDL HEADER END # # Copyright 2006-2010 Sun Microsystems, Inc. # Portions Copyright 2011-2014 ForgeRock AS # Portions Copyright 2011-2015 ForgeRock AS # @@ -836,10 +836,6 @@ ERR_LDIF_BACKEND_ERROR_WRITING_FILE_346=An error occurred while \ trying to write updated data to file %s for the LDIF backend defined in \ configuration entry %s: %s ERR_LDIF_BACKEND_ERROR_CLOSING_FILE_426=An error occurred while trying to \ close file %s for the LDIF backend defined in configuration entry %s: %s ERR_LDIF_BACKEND_ERROR_EMPTY_FILE_427=The file %s written for the LDIF backend \ defined in configuration entry %s is 0 bytes long and unusable. ERR_LDIF_BACKEND_ERROR_RENAMING_FILE_347=An error occurred while \ attempting to rename file %s to %s while writing updated data for the LDIF \ backend defined in configuration entry %s: %s @@ -1019,4 +1015,14 @@ ERR_RECURRINGTASK_INVALID_WEEKDAY_TOKEN_SIMPLE_424=The provided \ recurring task schedule value has an invalid day of the week token ERR_SCHEMA_INVALID_REPLACE_MODIFICATION_425=The schema backend does \ not support the Replace modification type for the %s attribute type not support the Replace modification type for the %s attribute type ERR_LDIF_BACKEND_ERROR_CLOSING_FILE_426=An error occurred while trying to \ close file %s for the LDIF backend defined in configuration entry %s: %s ERR_LDIF_BACKEND_ERROR_EMPTY_FILE_427=The file %s written for the LDIF backend \ defined in configuration entry %s is 0 bytes long and unusable. ERR_BACKEND_CONFIG_CACHE_SIZE_GREATER_THAN_JVM_HEAP_428=Configuration \ attribute ds-cfg-db-cache-size has a value of %d but the JVM has only \ %d available. Consider using ds-cfg-db-cache-percent ERR_BACKEND_CONFIG_CACHE_PERCENT_GREATER_THAN_JVM_HEAP_429=Configuration \ attribute ds-cfg-db-cache-percent has a value of %d%% but the JVM has only \ %d%% available
