mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java
@@ -79,9 +79,9 @@ new DatabaseEntry("+".getBytes()); /** * The container in which this attribute index resides. * The entryContainer in which this attribute index resides. */ Container container; EntryContainer entryContainer; /** * The attribute index configuration. @@ -110,13 +110,13 @@ /** * Create a new attribute index object. * @param entryContainer The entryContainer of this attribute index. * @param indexConfig The attribute index configuration. * @param container The container of this attribute index. */ public AttributeIndex(IndexConfig indexConfig, Container container) public AttributeIndex(EntryContainer entryContainer, IndexConfig indexConfig) { this.entryContainer = entryContainer; this.indexConfig = indexConfig; this.container = container; AttributeType attrType = indexConfig.getAttributeType(); String name = attrType.getNameOrOID(); @@ -124,7 +124,7 @@ if (indexConfig.isEqualityIndex()) { Indexer equalityIndexer = new EqualityIndexer(indexConfig); this.equalityIndex = new Index(container, name + ".equality", this.equalityIndex = new Index(this.entryContainer, name + ".equality", equalityIndexer, indexConfig.getEqualityEntryLimit(), indexConfig.getCursorEntryLimit()); @@ -133,7 +133,7 @@ if (indexConfig.isPresenceIndex()) { Indexer presenceIndexer = new PresenceIndexer(indexConfig); this.presenceIndex = new Index(container, name + ".presence", this.presenceIndex = new Index(this.entryContainer, name + ".presence", presenceIndexer, indexConfig.getPresenceEntryLimit(), indexConfig.getCursorEntryLimit()); @@ -142,7 +142,7 @@ if (indexConfig.isSubstringIndex()) { Indexer substringIndexer = new SubstringIndexer(indexConfig); this.substringIndex = new Index(container, name + ".substring", this.substringIndex = new Index(this.entryContainer, name + ".substring", substringIndexer, indexConfig.getSubstringEntryLimit(), indexConfig.getCursorEntryLimit()); @@ -151,7 +151,7 @@ if (indexConfig.isOrderingIndex()) { Indexer orderingIndexer = new OrderingIndexer(indexConfig); this.orderingIndex = new Index(container, name + ".ordering", this.orderingIndex = new Index(this.entryContainer, name + ".ordering", orderingIndexer, indexConfig.getEqualityEntryLimit(), indexConfig.getCursorEntryLimit()); @@ -195,7 +195,7 @@ */ public void close() { // The container is responsible for closing the JE databases. // The entryContainer is responsible for closing the JE databases. } /** @@ -791,19 +791,19 @@ String name = attrType.getNameOrOID(); if (indexConfig.isEqualityIndex()) { container.removeDatabase(name + ".equality"); entryContainer.removeDatabase(name + ".equality"); } if (indexConfig.isPresenceIndex()) { container.removeDatabase(name + ".presence"); entryContainer.removeDatabase(name + ".presence"); } if (indexConfig.isSubstringIndex()) { container.removeDatabase(name + ".substring"); entryContainer.removeDatabase(name + ".substring"); } if (indexConfig.isOrderingIndex()) { container.removeDatabase(name + ".ordering"); entryContainer.removeDatabase(name + ".ordering"); } }
@@ -26,29 +26,13 @@ */ package org.opends.server.backends.jeb; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import com.sleepycat.je.CheckpointConfig; import com.sleepycat.je.Database; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentStats; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.PreloadConfig; import com.sleepycat.je.PreloadStats; import com.sleepycat.je.PreloadStatus; import com.sleepycat.je.StatsConfig; import com.sleepycat.je.config.EnvironmentParams; import com.sleepycat.je.config.ConfigParam; import org.opends.server.api.Backend; import org.opends.server.api.ConfigurableComponent; @@ -67,8 +51,6 @@ import org.opends.server.types.BackupDirectory; import org.opends.server.types.CancelledOperationException; import org.opends.server.types.ConfigChangeResult; import org.opends.server.types.DebugLogCategory; import org.opends.server.types.DebugLogSeverity; import org.opends.server.types.DN; import org.opends.server.types.DirectoryException; import org.opends.server.types.Entry; @@ -79,10 +61,7 @@ import org.opends.server.types.LDIFExportConfig; import org.opends.server.types.RestoreConfig; import org.opends.server.types.ResultCode; import org.opends.server.types.FilePermission; import org.opends.server.monitors.DatabaseEnvironmentMonitor; import org.opends.server.util.LDIFException; import org.opends.server.loggers.Debug; import static org.opends.server.messages.MessageHandler.*; import static org.opends.server.messages.JebMessages.*; @@ -117,26 +96,9 @@ private Config config; /** * The directory containing persistent storage for the backend. * The root JE container to use for this backend. */ private File backendDirectory; /** * The permissions mode for the directory containing persistent storage for * the backend. */ private FilePermission backendPermission; /** * The base DNs contained in this backend. */ private ConcurrentHashMap<DN, EntryContainer> baseDNs; /** * The JE environment for this backend instance. Each instance has its own * environment. */ private com.sleepycat.je.Environment dbEnv; private RootContainer rootContainer; /** * A configurable component to handle changes to the configuration of @@ -273,46 +235,6 @@ } } /** * Determine the container of an entry DN. * * @param dn The DN of an entry. * @return The container of the entry. * @throws DirectoryException If the entry DN does not match any of the base * DNs. */ private EntryContainer getContainer(DN dn) throws DirectoryException { assert debugEnter(CLASS_NAME, "getContainer"); EntryContainer ec = null; DN nodeDN = dn; while (ec == null && nodeDN != null) { ec = baseDNs.get(nodeDN); if (ec == null) { nodeDN = nodeDN.getParent(); } } if (ec == null) { // The operation should not have been routed to this backend. String message = getMessage(MSGID_JEB_INCORRECT_ROUTING, dn.toString()); throw new DirectoryException(ResultCode.NO_SUCH_OBJECT, message, MSGID_JEB_INCORRECT_ROUTING); } return ec; } /** * This method constructs a container name from a base DN. Only alphanumeric * characters are preserved, all other characters are replaced with an @@ -371,34 +293,6 @@ config = new Config(); config.initializeConfig(configEntry, baseDNs); // Get the backend database directory. backendDirectory = config.getBackendDirectory(); if (!backendDirectory.isDirectory()) { String message = getMessage(MSGID_JEB_DIRECTORY_INVALID, backendDirectory.getPath()); throw new InitializationException(MSGID_JEB_DIRECTORY_INVALID, message); } // Get the backend database directory permissions and apply try { backendPermission = config.getBackendPermission(); if(!FilePermission.setPermissions(backendDirectory, backendPermission)) { throw new Exception(); } } catch(Exception e) { // Log an warning that the permissions were not set. int msgID = MSGID_JEB_SET_PERMISSIONS_FAILED; String message = getMessage(msgID, backendDirectory.getPath()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_WARNING, message, msgID); } // FIXME: Currently assuming every base DN is also a suffix. for (DN dn : baseDNs) { @@ -416,14 +310,8 @@ // Open the database environment try { dbEnv = new Environment(backendDirectory, config.getEnvironmentConfig()); Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "initializeBackend", dbEnv.getConfig().toString()); // cleanDatabase(); rootContainer = new RootContainer(config, this); rootContainer.open(); } catch (DatabaseException e) { @@ -433,24 +321,15 @@ throw new InitializationException(MSGID_JEB_OPEN_ENV_FAIL, message, e); } // Create and register a monitor provider for the environment. String monitorName = this.getBackendID() + " Database Environment"; // Register a monitor provider for the environment. MonitorProvider monitorProvider = new DatabaseEnvironmentMonitor(monitorName, dbEnv); rootContainer.getMonitorProvider(); monitorProviders.add(monitorProvider); DirectoryServer.registerMonitorProvider(monitorProvider); // Open all the databases. this.baseDNs = new ConcurrentHashMap<DN, EntryContainer>(baseDNs.length); for (DN dn : baseDNs) { String containerName = getContainerName(dn); Container container = new Container(dbEnv, containerName); EntryContainer ec = new EntryContainer(this, config, container); try { ec.open(); rootContainer.openEntryContainers(baseDNs); } catch (DatabaseException databaseException) { @@ -462,16 +341,13 @@ databaseException); } this.baseDNs.put(dn, ec); } // Preload the database cache. preload(); rootContainer.preload(); // Determine the next entry ID and the total number of entries. EntryID highestID = null; long entryCount = 0; for (EntryContainer ec : this.baseDNs.values()) for (EntryContainer ec : rootContainer.getEntryContainers()) { try { @@ -502,76 +378,14 @@ DirectoryServer.registerConfigurableComponent(this); // Register the database environment as a configurable component. DN envConfigDN = config.getEnvConfigDN(); if (envConfigDN != null) ConfigurableEnvironment configurableEnv = rootContainer.getConfigurableEnvironment(); if (configurableEnv != null) { configurableEnv = new ConfigurableEnvironment(envConfigDN, dbEnv); DirectoryServer.registerConfigurableComponent(configurableEnv); } } /** * Synchronously invokes the cleaner on the database environment then forces a * checkpoint to delete the log files that are no longer in use. * * @throws DatabaseException If an error occurs while cleaning the database * environment. */ private void cleanDatabase() throws DatabaseException { assert debugEnter(CLASS_NAME, "cleanDatabase"); int msgID; String message; FilenameFilter filenameFilter = new FilenameFilter() { public boolean accept(File d, String name) { return name.endsWith(".jdb"); } }; int beforeLogfileCount = backendDirectory.list(filenameFilter).length; msgID = MSGID_JEB_CLEAN_DATABASE_START; message = getMessage(msgID, beforeLogfileCount, backendDirectory.getPath()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); int currentCleaned = 0; int totalCleaned = 0; while ((currentCleaned = dbEnv.cleanLog()) > 0) { totalCleaned += currentCleaned; } msgID = MSGID_JEB_CLEAN_DATABASE_MARKED; message = getMessage(msgID, totalCleaned); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); if (totalCleaned > 0) { CheckpointConfig force = new CheckpointConfig(); force.setForce(true); dbEnv.checkpoint(force); } int afterLogfileCount = backendDirectory.list(filenameFilter).length; msgID = MSGID_JEB_CLEAN_DATABASE_FINISH; message = getMessage(msgID, afterLogfileCount); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } /** * Performs any necessary work to finalize this backend, including closing any * underlying databases or connections and deregistering any suffixes that it @@ -595,7 +409,7 @@ // Deregister our suffixes. // FIXME: Currently assuming every base DN is also a suffix. for (DN dn : baseDNs.keySet()) for (DN dn : rootContainer.getBaseDNs()) { try { @@ -623,11 +437,7 @@ // Close the database. try { for (EntryContainer ec : baseDNs.values()) { ec.close(); } dbEnv.close(); rootContainer.close(); } catch (DatabaseException e) { @@ -850,7 +660,7 @@ readerBegin(); try { EntryContainer ec = getContainer(entryDN); EntryContainer ec = rootContainer.getEntryContainer(entryDN); Entry entry; try { @@ -903,7 +713,7 @@ try { DN entryDN = entry.getDN(); EntryContainer ec = getContainer(entryDN); EntryContainer ec = rootContainer.getEntryContainer(entryDN); try { @@ -954,7 +764,7 @@ writerBegin(); try { EntryContainer ec = getContainer(entryDN); EntryContainer ec = rootContainer.getEntryContainer(entryDN); try { ec.deleteEntry(entryDN, deleteOperation); @@ -1005,7 +815,7 @@ try { DN entryDN = entry.getDN(); EntryContainer ec = getContainer(entryDN); EntryContainer ec = rootContainer.getEntryContainer(entryDN); try { @@ -1061,12 +871,13 @@ writerBegin(); try { EntryContainer currentContainer = getContainer(currentDN); EntryContainer container = getContainer(entry.getDN()); EntryContainer currentContainer = rootContainer.getEntryContainer( currentDN); EntryContainer container = rootContainer.getEntryContainer(entry.getDN()); if (currentContainer != container) { // No reason why we cannot implement a move between containers // FIXME: No reason why we cannot implement a move between containers // since the containers share the same database environment. int msgID = MSGID_JEB_FUNCTION_NOT_SUPPORTED; String msg = getMessage(MSGID_JEB_FUNCTION_NOT_SUPPORTED); @@ -1115,7 +926,8 @@ readerBegin(); try { EntryContainer ec = getContainer(searchOperation.getBaseDN()); EntryContainer ec = rootContainer.getEntryContainer( searchOperation.getBaseDN()); ec.search(searchOperation); } catch (DatabaseException e) @@ -1166,48 +978,24 @@ e.getMessageID()); } // If the backend already has the environment open, we must use the same // underlying environment instance and its configuration. Environment env; try { EnvironmentConfig envConfig; Environment existingEnv = dbEnv; if (existingEnv == null) { // Open the environment read-only. envConfig = config.getEnvironmentConfig(); envConfig.setReadOnly(true); envConfig.setAllowCreate(false); envConfig.setTransactional(false); } else { envConfig = existingEnv.getConfig(); } // Open a new environment handle. File backendDirectory = config.getBackendDirectory(); env = new Environment(backendDirectory, envConfig); Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "exportLDIF", env.getConfig().toString()); } catch (DatabaseException e) { assert debugException(CLASS_NAME, "exportLDIF", e); String message = getMessage(MSGID_JEB_DATABASE_EXCEPTION, e.getMessage()); throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), message, MSGID_JEB_DATABASE_EXCEPTION); } // If the backend already has the root container open, we must use the same // underlying root container boolean openRootContainer = rootContainer == null; try { ExportJob exportJob = new ExportJob(this, config, exportConfig); exportJob.exportLDIF(env); if (openRootContainer) { // Open the database environment rootContainer = new RootContainer(config, this); rootContainer.open(config.getBackendDirectory(), config.getBackendPermission(), true, false, false, false, true, true); rootContainer.openEntryContainers(baseDNs); } ExportJob exportJob = new ExportJob(exportConfig); exportJob.exportLDIF(rootContainer); } catch (IOException ioe) { @@ -1241,9 +1029,14 @@ } finally { //If a root container was opened in this method as read only, close it //to leave the backend in the same state. if (openRootContainer && rootContainer != null) { try { env.close(); rootContainer.close(); rootContainer = null; } catch (DatabaseException e) { @@ -1251,6 +1044,7 @@ } } } } @@ -1342,10 +1136,24 @@ config = new Config(); config.initializeConfig(configEntry, baseDNs); VerifyJob verifyJob = new VerifyJob(this, config, verifyConfig); // If the backend already has the root container open, we must use the same // underlying root container boolean openRootContainer = rootContainer == null; try { verifyJob.verifyBackend(); if (openRootContainer) { // Open the database environment rootContainer = new RootContainer(config, this); rootContainer.open(config.getBackendDirectory(), config.getBackendPermission(), true, false, false, false, true, true); rootContainer.openEntryContainers(baseDNs); } VerifyJob verifyJob = new VerifyJob(config, verifyConfig); verifyJob.verifyBackend(rootContainer); } catch (DatabaseException e) { @@ -1362,6 +1170,23 @@ e.getMessage(), e.getMessageID()); } finally { //If a root container was opened in this method as read only, close it //to leave the backend in the same state. if (openRootContainer && rootContainer != null) { try { rootContainer.close(); rootContainer = null; } catch (DatabaseException e) { assert debugException(CLASS_NAME, "verifyBackend", e); } } } } @@ -1582,78 +1407,22 @@ // operation threads with a handle on the entry container being // closed. DirectoryServer.deregisterSuffix(baseDN); EntryContainer ec = this.baseDNs.remove(baseDN); ec.close(); ec.removeContainer(); rootContainer.removeEntryContainer(baseDN); } } for (DN baseDN : newConfig.getBaseDNs()) { if (!this.baseDNs.containsKey(baseDN)) if (!rootContainer.getBaseDNs().contains(baseDN)) { // The base DN was added. String containerName = getContainerName(baseDN); Container container = new Container(dbEnv, containerName); EntryContainer ec = new EntryContainer(this, config, container); ec.open(); this.baseDNs.put(baseDN, ec); rootContainer.openEntryContainer(baseDN); DirectoryServer.registerSuffix(baseDN, this); } } // Check for changes to the database directory permissions FilePermission oldPermission = config.getBackendPermission(); FilePermission newPermission = newConfig.getBackendPermission(); if(!FilePermission.toUNIXMode(oldPermission).equals( FilePermission.toUNIXMode(newPermission))) { try { if(!FilePermission.setPermissions(newConfig.getBackendDirectory(), newPermission)) { throw new Exception(); } } catch(Exception e) { // Log an warning that the permissions were not set. int msgID = MSGID_JEB_SET_PERMISSIONS_FAILED; String message = getMessage(msgID, backendDirectory.getPath()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_WARNING, message, msgID); } } // Check if any JE non-mutable properties were changed. EnvironmentConfig oldEnvConfig = config.getEnvironmentConfig(); EnvironmentConfig newEnvConfig = newConfig.getEnvironmentConfig(); Map paramsMap = EnvironmentParams.SUPPORTED_PARAMS; for (Object o : paramsMap.values()) { ConfigParam param = (ConfigParam)o; if (!param.isMutable()) { String oldValue = oldEnvConfig.getConfigParam(param.getName()); String newValue = newEnvConfig.getConfigParam(param.getName()); if (!oldValue.equalsIgnoreCase(newValue)) { System.out.println("The change to the following property will " + "take effect when the backend is restarted: " + param.getName()); } } } // This takes care of changes to the JE environment for those // properties that are mutable at runtime. dbEnv.setMutableConfig(newConfig.getEnvironmentConfig()); Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "applyNewConfiguration", dbEnv.getConfig().toString()); // Apply new JE configuration rootContainer.applyNewConfig(config); // Put the new configuration in place. config = newConfig; @@ -1671,76 +1440,15 @@ return ccr; } /** * Preload the database cache. There is no preload if the configured preload * time limit is zero. * Returns a handle to the JE root container currently used by this backend. * The rootContainer could be NULL if the backend is not initialized. * * @return The RootContainer object currently used by this backend. */ private void preload() public RootContainer getRootContainer() { assert debugEnter(CLASS_NAME, "preload"); long timeLimit = config.getPreloadTimeLimit(); if (timeLimit > 0) { // Get a list of all the databases used by the backend. ArrayList<Database> dbList = new ArrayList<Database>(); for (EntryContainer ec : baseDNs.values()) { ec.listDatabases(dbList); } // Sort the list in order of priority. Collections.sort(dbList, new DbPreloadComparator()); // Preload each database until we reach the time limit or the cache // is filled. try { long timeEnd = System.currentTimeMillis() + timeLimit; // Configure preload of Leaf Nodes (LNs) containing the data values. PreloadConfig preloadConfig = new PreloadConfig(); preloadConfig.setLoadLNs(true); for (Database db : dbList) { // Calculate the remaining time. long timeRemaining = timeEnd - System.currentTimeMillis(); if (timeRemaining <= 0) { break; } preloadConfig.setMaxMillisecs(timeRemaining); PreloadStats preloadStats = db.preload(preloadConfig); /* System.out.println("file=" + db.getDatabaseName() + " LNs=" + preloadStats.getNLNsLoaded()); */ // Stop if the cache is full or the time limit has been exceeded. if (preloadStats.getStatus() != PreloadStatus.SUCCESS) { break; } } // Log an informational message about the size of the cache. EnvironmentStats stats = dbEnv.getStats(new StatsConfig()); long total = stats.getCacheTotalBytes(); int msgID = MSGID_JEB_CACHE_SIZE_AFTER_PRELOAD; String message = getMessage(msgID, total / (1024 * 1024)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } catch (DatabaseException e) { assert debugException(CLASS_NAME, "preload", e); } } assert debugEnter(CLASS_NAME, "getRootContainer"); return rootContainer; } } opends/src/server/org/opends/server/backends/jeb/Config.java
@@ -356,6 +356,13 @@ throw new ConfigException(msgID, message); } backendDirectory = getFileForPath(backendDirectoryAttr.activeValue()); //Make sure the directory is valid. if (!backendDirectory.isDirectory()) { int msgID = MSGID_JEB_DIRECTORY_INVALID; String message = getMessage(msgID, backendDirectory.getPath()); throw new ConfigException(MSGID_JEB_DIRECTORY_INVALID, message); } // ds-cfg-backend-mode // Optional, single-valued config attribute requiring admin action on change opends/src/server/org/opends/server/backends/jeb/Container.java
File was deleted opends/src/server/org/opends/server/backends/jeb/DN2ID.java
@@ -49,9 +49,9 @@ public class DN2ID { /** * The database container. * The database entryContainer. */ private Container container; private EntryContainer entryContainer; /** * The JE database configuration. @@ -59,7 +59,7 @@ private DatabaseConfig dbConfig; /** * The name of the database within the container. * The name of the database within the entryContainer. */ private String name; @@ -70,16 +70,17 @@ new ThreadLocal<Database>(); /** * Create a DN2ID instance for the DN database in a given container. * Create a DN2ID instance for the DN database in a given entryContainer. * * @param container The container of the DN database. * @param entryContainer The entryContainer of the DN database. * @param dbConfig The JE database configuration which will be used to * open the database. * @param name The name of the DN database ("dn2id"). */ public DN2ID(Container container, DatabaseConfig dbConfig, String name) public DN2ID(EntryContainer entryContainer, DatabaseConfig dbConfig, String name) { this.container = container; this.entryContainer = entryContainer; this.dbConfig = dbConfig; this.name = name; } @@ -96,7 +97,7 @@ /** * Get a handle to the database. It returns a per-thread handle to avoid * any thread contention on the database handle. The container is * any thread contention on the database handle. The entryContainer is * responsible for closing all handles. * * @return A database handle. @@ -108,7 +109,7 @@ Database database = threadLocalDatabase.get(); if (database == null) { database = container.openDatabase(dbConfig, name); database = entryContainer.openDatabase(dbConfig, name); threadLocalDatabase.set(database); } return database; @@ -144,7 +145,7 @@ OperationStatus status; status = Container.insert(getDatabase(), txn, key, data); status = EntryContainer.insert(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -171,7 +172,7 @@ DatabaseEntry data = id.getDatabaseEntry(); OperationStatus status; status = Container.put(getDatabase(), txn, key, data); status = EntryContainer.put(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -194,7 +195,7 @@ throws DatabaseException { OperationStatus status; status = Container.put(getDatabase(), txn, key, data); status = EntryContainer.put(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -216,7 +217,7 @@ { DatabaseEntry key = DNdata(dn); OperationStatus status = Container.delete(getDatabase(), txn, key); OperationStatus status = EntryContainer.delete(getDatabase(), txn, key); if (status != OperationStatus.SUCCESS) { return false; @@ -239,7 +240,8 @@ DatabaseEntry data = new DatabaseEntry(); OperationStatus status; status = Container.read(getDatabase(), txn, key, data, LockMode.DEFAULT); status = EntryContainer.read(getDatabase(), txn, key, data, LockMode.DEFAULT); if (status != OperationStatus.SUCCESS) { return null; opends/src/server/org/opends/server/backends/jeb/DN2URI.java
@@ -90,9 +90,9 @@ DirectoryServer.getAttributeType(ATTR_REFERRAL_URL); /** * The database container. * The database entryContainer. */ private Container container; private EntryContainer entryContainer; /** * The JE database configuration. @@ -100,7 +100,7 @@ private DatabaseConfig dbConfig; /** * The name of the database within the container. * The name of the database within the entryContainer. */ private String name; @@ -116,16 +116,18 @@ new ThreadLocal<Database>(); /** * Create a new object representing a referral database in a given container. * Create a new object representing a referral database in a given * entryContainer. * * @param container The container of the referral database. * @param entryContainer The entryContainer of the referral database. * @param dbConfig The JE database configuration which will be used to * open the database. * @param name The name of the referral database. */ public DN2URI(Container container, DatabaseConfig dbConfig, String name) public DN2URI(EntryContainer entryContainer, DatabaseConfig dbConfig, String name) { this.container = container; this.entryContainer = entryContainer; this.dbConfig = dbConfig; this.name = name; } @@ -142,7 +144,7 @@ /** * Get a handle to the database. It returns a per-thread handle to avoid * any thread contention on the database handle. The container is * any thread contention on the database handle. The entryContainer is * responsible for closing all handles. * * @return A database handle. @@ -154,7 +156,7 @@ Database database = threadLocalDatabase.get(); if (database == null) { database = container.openDatabase(dbConfig, name); database = entryContainer.openDatabase(dbConfig, name); threadLocalDatabase.set(database); } return database; @@ -181,7 +183,7 @@ // The JE insert method does not permit duplicate keys so we must use the // put method. status = Container.put(getDatabase(), txn, key, data); status = EntryContainer.put(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -206,7 +208,7 @@ DatabaseEntry key = new DatabaseEntry(normDN); OperationStatus status; status = Container.delete(getDatabase(), txn, key); status = EntryContainer.delete(getDatabase(), txn, key); if (status != OperationStatus.SUCCESS) { return false; opends/src/server/org/opends/server/backends/jeb/EntryContainer.java
@@ -26,16 +26,7 @@ */ package org.opends.server.backends.jeb; import com.sleepycat.je.Cursor; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.DatabaseNotFoundException; import com.sleepycat.je.DeadlockException; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; import com.sleepycat.je.Transaction; import com.sleepycat.je.*; import org.opends.server.api.AttributeSyntax; import org.opends.server.api.Backend; @@ -47,6 +38,7 @@ import org.opends.server.core.ModifyDNOperation; import org.opends.server.core.Operation; import org.opends.server.core.SearchOperation; import org.opends.server.loggers.Debug; import org.opends.server.protocols.asn1.ASN1OctetString; import org.opends.server.protocols.ldap.LDAPException; import org.opends.server.controls.PagedResultsControl; @@ -55,6 +47,7 @@ import org.opends.server.types.AttributeValue; import org.opends.server.types.CancelledOperationException; import org.opends.server.types.Control; import org.opends.server.types.DebugLogCategory; import org.opends.server.types.DirectoryException; import org.opends.server.types.DN; import org.opends.server.types.Entry; @@ -77,6 +70,10 @@ import static org.opends.server.messages.MessageHandler.getMessage; import static org.opends.server.messages.JebMessages.*; import static org.opends.server.loggers.Debug.debugException; import static org.opends.server.types.DebugLogSeverity.VERBOSE; import static org.opends.server.types.DebugLogCategory.DATABASE_ACCESS; import static org.opends.server.types.DebugLogCategory.DATABASE_WRITE; import static org.opends.server.types.DebugLogCategory.DATABASE_READ; import static org.opends.server.util.ServerConstants.OID_SUBTREE_DELETE_CONTROL; import static org.opends.server.util.ServerConstants.OID_PAGED_RESULTS_CONTROL; @@ -94,6 +91,16 @@ "org.opends.server.backends.jeb.EntryContainer"; /** * The JE database environment. */ private static Environment env; /** * The backend configuration. */ private static Config config; /** * The name of the entry database. */ public static final String ID2ENTRY_DATABASE_NAME = "id2entry"; @@ -124,19 +131,26 @@ public static final String ATTR_DEBUG_SEARCH_INDEX = "debugsearchindex"; /** * The backend to which this entry container belongs. * The backend to which this entry entryContainer belongs. */ private Backend backend; /** * The database container. * The baseDN this entry container is responsible for. */ private Container container; private DN baseDN; /** * The backend configuration. * A list of JE database handles opened through this entryContainer. * They will be closed by the entryContainer. */ private Config config; private ArrayList<Database> databases; /** * A list of JE cursor handles registered with this entryContainer. * They will be closed by the entryContainer. */ private ArrayList<Cursor> cursors; /** * The DN database maps a normalized DN string to an entry ID (8 bytes). @@ -179,25 +193,34 @@ private HashMap<AttributeType, AttributeIndex> attrIndexMap; /** * Create a new entry container object. This method does not actually create * anything in the JE database environment. * Create a new entry entryContainer object. * * @param baseDN The baseDN this entry container will be responsible for * storing on disk. * @param backend A reference to the JE backend that is creating this entry * container. It is needed by the Directory Server entry cache methods. * container. It is needed by the Directory Server entry cache * methods. * @param config The configuration of the JE backend. * @param container The databases reside in this container. * @param env The JE environment to create this entryContainer in */ public EntryContainer(Backend backend, Config config, Container container) public EntryContainer(DN baseDN, Backend backend, Config config, Environment env) { this.backend = backend; this.baseDN = baseDN; this.config = config; this.container = container; this.env = env; // Instantiate database and cursor lists databases = new ArrayList<Database>(); cursors = new ArrayList<Cursor>(); // Instantiate indexes for id2children and id2subtree. id2children = new Index(container, ID2CHILDREN_DATABASE_NAME, id2children = new Index(this, ID2CHILDREN_DATABASE_NAME, new ID2CIndexer(), config.getBackendIndexEntryLimit(), 0); id2subtree = new Index(container, ID2SUBTREE_DATABASE_NAME, id2subtree = new Index(this, ID2SUBTREE_DATABASE_NAME, new ID2SIndexer(), config.getBackendIndexEntryLimit(), 0); @@ -208,7 +231,7 @@ { for (IndexConfig indexConfig : config.getIndexConfigMap().values()) { AttributeIndex index = new AttributeIndex(indexConfig, container); AttributeIndex index = new AttributeIndex(this, indexConfig); attrIndexMap.put(indexConfig.getAttributeType(), index); } } @@ -218,7 +241,7 @@ } /** * Opens the container for reading and writing. * Opens the entryContainer for reading and writing. * * @throws DatabaseException If an error occurs in the JE database. */ @@ -229,10 +252,10 @@ DatabaseConfig dbNodupsConfig = new DatabaseConfig(); dbNodupsConfig.setAllowCreate(true); dbNodupsConfig.setTransactional(true); container.open(); try { id2entry = new ID2Entry(container, dbNodupsConfig, entryDataConfig, id2entry = new ID2Entry(this, dbNodupsConfig, entryDataConfig, ID2ENTRY_DATABASE_NAME); id2entry.open(); @@ -242,7 +265,7 @@ dn2idConfig.setAllowCreate(true); dn2idConfig.setTransactional(true); dn2idConfig.setBtreeComparator(dn2idComparator.getClass()); dn2id = new DN2ID(container, dn2idConfig, DN2ID_DATABASE_NAME); dn2id = new DN2ID(this, dn2idConfig, DN2ID_DATABASE_NAME); dn2id.open(); id2children.open(dbNodupsConfig); @@ -253,7 +276,7 @@ dn2uriConfig.setAllowCreate(true); dn2uriConfig.setTransactional(true); dn2uriConfig.setBtreeComparator(dn2idComparator.getClass()); dn2uri = new DN2URI(container, dn2uriConfig, REFERRAL_DATABASE_NAME); dn2uri = new DN2URI(this, dn2uriConfig, REFERRAL_DATABASE_NAME); dn2uri.open(); for (AttributeIndex index : attrIndexMap.values()) @@ -264,16 +287,16 @@ catch (DatabaseException e) { assert debugException(CLASS_NAME, "open", e); container.close(); close(); throw e; } } /** * Opens the container for reading and writing without transactions. * Opens the entryContainer for reading and writing without transactions. * * @param deferredWrite Indicates whether to open the container using the * deferred write mode. * @param deferredWrite Indicates whether to open the entryContainer using * the deferred write mode. * * @throws DatabaseException If an error occurs in the JE database. */ @@ -286,10 +309,9 @@ dbNodupsConfig.setTransactional(false); dbNodupsConfig.setDeferredWrite(deferredWrite); container.open(); try { id2entry = new ID2Entry(container, dbNodupsConfig, entryDataConfig, id2entry = new ID2Entry(this, dbNodupsConfig, entryDataConfig, ID2ENTRY_DATABASE_NAME); id2entry.open(); @@ -300,7 +322,7 @@ dn2idConfig.setTransactional(false); dn2idConfig.setBtreeComparator(dn2idComparator.getClass()); dn2idConfig.setDeferredWrite(deferredWrite); dn2id = new DN2ID(container, dn2idConfig, DN2ID_DATABASE_NAME); dn2id = new DN2ID(this, dn2idConfig, DN2ID_DATABASE_NAME); dn2id.open(); id2children.open(dbNodupsConfig); @@ -312,7 +334,7 @@ dn2uriConfig.setTransactional(false); dn2uriConfig.setBtreeComparator(dn2idComparator.getClass()); dn2uriConfig.setDeferredWrite(deferredWrite); dn2uri = new DN2URI(container, dn2uriConfig, REFERRAL_DATABASE_NAME); dn2uri = new DN2URI(this, dn2uriConfig, REFERRAL_DATABASE_NAME); dn2uri.open(); for (AttributeIndex index : attrIndexMap.values()) @@ -323,13 +345,13 @@ catch (DatabaseException e) { assert debugException(CLASS_NAME, "open", e); container.close(); close(); throw e; } } /** * Opens the container for reading only. * Opens the entryContainer for reading only. * * @throws DatabaseException If an error occurs in the JE database. */ @@ -342,10 +364,9 @@ dbNodupsConfig.setAllowCreate(false); dbNodupsConfig.setTransactional(false); container.open(); try { id2entry = new ID2Entry(container, dbNodupsConfig, entryDataConfig, id2entry = new ID2Entry(this, dbNodupsConfig, entryDataConfig, ID2ENTRY_DATABASE_NAME); id2entry.open(); @@ -356,7 +377,7 @@ dn2idConfig.setAllowCreate(false); dn2idConfig.setTransactional(false); dn2idConfig.setBtreeComparator(dn2idComparator.getClass()); dn2id = new DN2ID(container, dn2idConfig, DN2ID_DATABASE_NAME); dn2id = new DN2ID(this, dn2idConfig, DN2ID_DATABASE_NAME); dn2id.open(); id2children.open(dbNodupsConfig); @@ -368,7 +389,7 @@ dn2uriConfig.setAllowCreate(false); dn2uriConfig.setTransactional(false); dn2uriConfig.setBtreeComparator(dn2idComparator.getClass()); dn2uri = new DN2URI(container, dn2uriConfig, REFERRAL_DATABASE_NAME); dn2uri = new DN2URI(this, dn2uriConfig, REFERRAL_DATABASE_NAME); dn2uri.open(); for (AttributeIndex index : attrIndexMap.values()) @@ -379,21 +400,36 @@ catch (DatabaseException e) { assert debugException(CLASS_NAME, "openReadOnly", e); container.close(); close(); throw e; } } /** * Closes the entry container. * Closes the entry entryContainer. * * @throws DatabaseException If an error occurs in the JE database. */ public void close() throws DatabaseException { // The database container is responsible for closing the JE databases. container.close(); // Close each cursor that has been registered. for (Cursor cursor : cursors) { cursor.close(); } // Close each database handle that has been opened. for (Database database : databases) { if (database.getConfig().getDeferredWrite()) { database.sync(); } database.close(); } for (AttributeIndex index : attrIndexMap.values()) { index.close(); @@ -401,8 +437,8 @@ } /** * Get the DN database used by this entry container. The container must * have been opened. * Get the DN database used by this entry entryContainer. The entryContainer * must have been opened. * * @return The DN database. */ @@ -412,8 +448,8 @@ } /** * Get the entry database used by this entry container. The container must * have been opened. * Get the entry database used by this entry entryContainer. The * entryContainer must have been opened. * * @return The entry database. */ @@ -423,8 +459,8 @@ } /** * Get the referral database used by this entry container. The container must * have been opened. * Get the referral database used by this entry entryContainer. The * entryContainer must have been opened. * * @return The referral database. */ @@ -434,8 +470,8 @@ } /** * Get the children database used by this entry container. * The container must have been opened. * Get the children database used by this entry entryContainer. * The entryContainer must have been opened. * * @return The children database. */ @@ -445,8 +481,8 @@ } /** * Get the subtree database used by this entry container. * The container must have been opened. * Get the subtree database used by this entry entryContainer. * The entryContainer must have been opened. * * @return The subtree database. */ @@ -467,8 +503,8 @@ } /** * Determine the highest entryID in the container. * The container must already be open. * Determine the highest entryID in the entryContainer. * The entryContainer must already be open. * * @return The highest entry ID. * @throws JebException If an error occurs in the JE backend. @@ -499,7 +535,7 @@ } /** * Processes the specified search in this container. * Processes the specified search in this entryContainer. * Matching entries should be provided back to the core server using the * <CODE>SearchOperation.returnEntry</CODE> method. * @@ -1269,12 +1305,12 @@ operation.invokeOperation(txn); // Commit the transaction. Container.transactionCommit(txn); EntryContainer.transactionCommit(txn); completed = true; } catch (DeadlockException deadlockException) { Container.transactionAbort(txn); EntryContainer.transactionAbort(txn); if (retryRemaining-- <= 0) { throw deadlockException; @@ -1284,22 +1320,22 @@ } catch (DatabaseException databaseException) { Container.transactionAbort(txn); EntryContainer.transactionAbort(txn); throw databaseException; } catch (DirectoryException directoryException) { Container.transactionAbort(txn); EntryContainer.transactionAbort(txn); throw directoryException; } catch (JebException jebException) { Container.transactionAbort(txn); EntryContainer.transactionAbort(txn); throw jebException; } catch (Exception e) { Container.transactionAbort(txn); EntryContainer.transactionAbort(txn); int messageID = MSGID_JEB_UNCHECKED_EXCEPTION; String message = getMessage(messageID); @@ -1378,7 +1414,7 @@ */ public Transaction beginTransaction() throws DatabaseException { return container.beginTransaction(); return EntryContainer.beginTransaction(); } /** @@ -1827,7 +1863,7 @@ */ public Transaction beginTransaction() throws DatabaseException { return container.beginTransaction(); return EntryContainer.beginTransaction(); } /** @@ -2356,7 +2392,7 @@ */ public Transaction beginTransaction() throws DatabaseException { return container.beginTransaction(); return EntryContainer.beginTransaction(); } /** @@ -2718,7 +2754,7 @@ */ public Transaction beginTransaction() throws DatabaseException { return container.beginTransaction(); return EntryContainer.beginTransaction(); } /** @@ -3196,9 +3232,9 @@ } /** * Get a count of the number of entries stored in this entry container. * Get a count of the number of entries stored in this entry entryContainer. * * @return The number of entries stored in this entry container. * @return The number of entries stored in this entry entryContainer. * @throws DatabaseException If an error occurs in the JE database. */ public long getEntryCount() throws DatabaseException @@ -3207,7 +3243,8 @@ } /** * Remove the entry container from disk. The container must not be open. * Remove the entry entryContainer from disk. The entryContainer must not be * open. * * @throws DatabaseException If an error occurs in the JE database. */ @@ -3215,7 +3252,7 @@ { try { container.removeDatabase(DN2ID_DATABASE_NAME); removeDatabase(DN2ID_DATABASE_NAME); } catch (DatabaseNotFoundException e) { @@ -3223,7 +3260,7 @@ } try { container.removeDatabase(ID2ENTRY_DATABASE_NAME); removeDatabase(ID2ENTRY_DATABASE_NAME); } catch (DatabaseNotFoundException e) { @@ -3231,7 +3268,7 @@ } try { container.removeDatabase(ID2CHILDREN_DATABASE_NAME); removeDatabase(ID2CHILDREN_DATABASE_NAME); } catch (DatabaseNotFoundException e) { @@ -3239,7 +3276,7 @@ } try { container.removeDatabase(ID2SUBTREE_DATABASE_NAME); removeDatabase(ID2SUBTREE_DATABASE_NAME); } catch (DatabaseNotFoundException e) { @@ -3260,7 +3297,7 @@ /** * Get the number of values for which the entry limit has been exceeded * since the entry container was opened. * since the entry entryContainer was opened. * @return The number of values for which the entry limit has been exceeded. */ public int getEntryLimitExceededCount() @@ -3276,53 +3313,18 @@ } /** * Begin a leaf transaction. * @return A JE transaction handle. * @throws DatabaseException If an error occurs in the JE database. */ protected Transaction beginTransaction() throws DatabaseException { return container.beginTransaction(); } /** * Commit a transaction. * @param txn The JE transaction handle. * @throws DatabaseException If an error occurs in the JE database. */ protected void transactionCommit(Transaction txn) throws DatabaseException { Container.transactionCommit(txn); } /** * Abort a transaction. * @param txn The JE transaction handle. * @throws DatabaseException If an error occurs in the JE database. */ protected void transactionAbort(Transaction txn) throws DatabaseException { Container.transactionAbort(txn); } /** * Get a list of the databases opened by this container. There will be * Get a list of the databases opened by this entryContainer. There will be * only one handle in the list for each database, regardless of the number * of handles open for a given database. * @param dbList A list of JE database handles. */ public void listDatabases(List<Database> dbList) { // The container has a list of all handles opened. List<Database> dbCompleteList = container.getDatabaseList(); // There may be more than one handle open for a given database // so we eliminate duplicates here. HashSet<String> set = new HashSet<String>(); for (Database db : dbCompleteList) for (Database db : databases) { try { @@ -3362,4 +3364,437 @@ return false; } /** * Constructs a full JE database name incorporating a entryContainer name. * * @param builder A string builder to which the full name will be appended. * @param name The short database name. */ private void buildDatabaseName(StringBuilder builder, String name) { builder.append(getContainerName()); builder.append('_'); builder.append(name); } /** * Opens a JE database in this entryContainer. The resulting database handle * must not be closed by the caller, as it will be closed by the * entryContainer. If the provided database configuration is transactional, * a transaction will be created and used to perform the open. * <p> * Note that a database can be opened multiple times and will result in * multiple unique handles to the database. This is used for example to * give each server thread its own database handle to eliminate contention * that could occur on a single handle. * * @param dbConfig The JE database configuration to be used to open the * database. * @param name The short database name, to which the entryContainer name * will be added. * @return A new JE database handle. * @throws DatabaseException If an error occurs while attempting to open the * database. */ public synchronized Database openDatabase(DatabaseConfig dbConfig, String name) throws DatabaseException { Database database; StringBuilder builder = new StringBuilder(); buildDatabaseName(builder, name); String fullName = builder.toString(); if (dbConfig.getTransactional()) { // Open the database under a transaction. Transaction txn = beginTransaction(); try { database = env.openDatabase(txn, fullName, dbConfig); assert Debug.debugMessage(DATABASE_ACCESS, VERBOSE, CLASS_NAME, "openDatabase", "open db=" + database.getDatabaseName() + " txnid=" + txn.getId()); transactionCommit(txn); } catch (DatabaseException e) { transactionAbort(txn); throw e; } } else { database = env.openDatabase(null, fullName, dbConfig); assert Debug.debugMessage(DATABASE_ACCESS, VERBOSE, CLASS_NAME, "openDatabase", "open db=" + database.getDatabaseName() + " txnid=none"); } // Insert into the list of database handles. databases.add(database); return database; } /** * Register a cursor with the entryContainer. The entryContainer will then * take care of closing the cursor when the entryContainer is closed. * * @param cursor A cursor to one of the databases in the entryContainer. */ public synchronized void addCursor(Cursor cursor) { cursors.add(cursor); } /** * Begin a leaf transaction using the default configuration. * Provides assertion debug logging. * @return A JE transaction handle. * @throws DatabaseException If an error occurs while attempting to begin * a new transaction. */ public static Transaction beginTransaction() throws DatabaseException { Transaction parentTxn = null; TransactionConfig txnConfig = null; Transaction txn = env.beginTransaction(parentTxn, txnConfig); assert Debug.debugMessage(DATABASE_ACCESS, VERBOSE, CLASS_NAME, "beginTransaction", "begin txnid=" + txn.getId()); return txn; } /** * Commit a transaction. * Provides assertion debug logging. * @param txn The JE transaction handle. * @throws DatabaseException If an error occurs while attempting to commit * the transaction. */ public static void transactionCommit(Transaction txn) throws DatabaseException { if (txn != null) { txn.commit(); assert Debug.debugMessage(DATABASE_ACCESS, VERBOSE, CLASS_NAME, "transactionCommit", "commit txnid=" + txn.getId()); } } /** * Abort a transaction. * Provides assertion debug logging. * @param txn The JE transaction handle. * @throws DatabaseException If an error occurs while attempting to abort the * transaction. */ public static void transactionAbort(Transaction txn) throws DatabaseException { if (txn != null) { txn.abort(); assert Debug.debugMessage(DATABASE_ACCESS, VERBOSE, CLASS_NAME, "transactionAbort", "abort txnid=" + txn.getId()); } } /** * Debug log a read or write access to the database. * @param operation The operation label: "read", "put", "insert". * @param category The log category for raw data value logging * @param status The JE return status code of the operation. * @param database The JE database handle operated on. * @param txn The JE transaction handle used in the operation. * @param key The database key operated on. * @param data The database value read or written. * @return true so that the method can be used in an assertion * @throws DatabaseException If an error occurs while retrieving information * about the JE objects provided to the method. */ private static boolean debugAccess(String operation, DebugLogCategory category, OperationStatus status, Database database, Transaction txn, DatabaseEntry key, DatabaseEntry data) throws DatabaseException { // Build the string that is common to category DATABASE_ACCESS and // DATABASE_READ/DATABASE_WRITE StringBuilder builder = new StringBuilder(); builder.append(operation); if (status == OperationStatus.SUCCESS) { builder.append(" (ok)"); } else { builder.append(" ("); builder.append(status.toString()); builder.append(")"); } builder.append(" db="); builder.append(database.getDatabaseName()); if (txn != null) { builder.append(" txnid="); builder.append(txn.getId()); } else { builder.append(" txnid=none"); } Debug.debugMessage(DATABASE_ACCESS, VERBOSE, CLASS_NAME, "debugAccess", builder.toString()); // If the operation was successful we log the same common information // plus the key and data under category DATABASE_READ or DATABASE_WRITE if (status == OperationStatus.SUCCESS) { builder.append(" key:"); builder.append(ServerConstants.EOL); StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 0); if (data != null) { builder.append("data(len="); builder.append(data.getSize()); builder.append("):"); builder.append(ServerConstants.EOL); StaticUtils.byteArrayToHexPlusAscii(builder, data.getData(), 0); } Debug.debugMessage(category, VERBOSE, CLASS_NAME, "debugAccess", builder.toString()); /* if (category == DATABASE_WRITE) { System.out.println(builder.toString()); } */ } return true; } /** * Insert a record into a JE database, with optional debug logging. This is a * simple wrapper around the JE Database.putNoOverwrite method. * @param database The JE database handle. * @param txn The JE transaction handle, or null if none. * @param key The record key. * @param data The record value. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus insert(Database database, Transaction txn, DatabaseEntry key, DatabaseEntry data) throws DatabaseException { OperationStatus status = database.putNoOverwrite(txn, key, data); assert debugAccess("insert", DATABASE_WRITE, status, database, txn, key, data); return status; } /** * Insert a record into a JE database through a cursor, with optional debug * logging. This is a simple wrapper around the JE Cursor.putNoOverwrite * method. * @param cursor The JE cursor handle. * @param key The record key. * @param data The record value. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus cursorInsert(Cursor cursor, DatabaseEntry key, DatabaseEntry data) throws DatabaseException { OperationStatus status = cursor.putNoOverwrite(key, data); assert debugAccess("cursorInsert", DATABASE_WRITE, status, cursor.getDatabase(), null, key, data); return status; } /** * Replace or insert a record into a JE database, with optional debug logging. * This is a simple wrapper around the JE Database.put method. * @param database The JE database handle. * @param txn The JE transaction handle, or null if none. * @param key The record key. * @param data The record value. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus put(Database database, Transaction txn, DatabaseEntry key, DatabaseEntry data) throws DatabaseException { OperationStatus status = database.put(txn, key, data); assert debugAccess("put", DATABASE_WRITE, status, database, txn, key, data); return status; } /** * Replace or insert a record into a JE database through a cursor, with * optional debug logging. This is a simple wrapper around the JE Cursor.put * method. * @param cursor The JE cursor handle. * @param key The record key. * @param data The record value. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus cursorPut(Cursor cursor, DatabaseEntry key, DatabaseEntry data) throws DatabaseException { OperationStatus status = cursor.put(key, data); assert debugAccess("cursorPut", DATABASE_WRITE, status, cursor.getDatabase(), null, key, data); return status; } /** * Read a record from a JE database, with optional debug logging. This is a * simple wrapper around the JE Database.get method. * @param database The JE database handle. * @param txn The JE transaction handle, or null if none. * @param key The key of the record to be read. * @param data The record value returned as output. Its byte array does not * need to be initialized by the caller. * @param lockMode The JE locking mode to be used for the read. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus read(Database database, Transaction txn, DatabaseEntry key, DatabaseEntry data, LockMode lockMode) throws DatabaseException { OperationStatus status = database.get(txn, key, data, lockMode); assert debugAccess("read", DATABASE_READ, status, database, txn, key, data); return status; } /** * Read a record from a JE database through a cursor, with optional debug * logging. This is a simple wrapper around the JE Cursor.getSearchKey method. * @param cursor The JE cursor handle. * @param key The key of the record to be read. * @param data The record value returned as output. Its byte array does not * need to be initialized by the caller. * @param lockMode The JE locking mode to be used for the read. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus cursorRead(Cursor cursor, DatabaseEntry key, DatabaseEntry data, LockMode lockMode) throws DatabaseException { OperationStatus status = cursor.getSearchKey(key, data, lockMode); assert debugAccess("cursorRead", DATABASE_READ, status, cursor.getDatabase(), null, key, data); return status; } /** * Delete a record from a JE database, with optional debug logging. This is a * simple wrapper around the JE Database.delete method. * @param database The JE database handle. * @param txn The JE transaction handle, or null if none. * @param key The key of the record to be read. * @return The operation status. * @throws DatabaseException If an error occurs in the JE operation. */ public static OperationStatus delete(Database database, Transaction txn, DatabaseEntry key) throws DatabaseException { OperationStatus status = database.delete(txn, key); assert debugAccess("delete", DATABASE_WRITE, status, database, txn, key, null); return status; } /** * Remove a database from disk. * * @param name The short database name, to which the entryContainer name will * be added. * @throws DatabaseException If an error occurs while attempting to delete the * database. */ public void removeDatabase(String name) throws DatabaseException { StringBuilder builder = new StringBuilder(); buildDatabaseName(builder, name); String fullName = builder.toString(); env.removeDatabase(null, fullName); } /** * Remove from disk all the databases in this entryContainer. * * @throws DatabaseException If an error occurs while attempting to delete any * database. */ private void removeAllDatabases() throws DatabaseException { for(Database database : databases) { String name = database.getDatabaseName(); env.removeDatabase(null, name); } } /** * This method constructs a container name from a base DN. Only alphanumeric * characters are preserved, all other characters are replaced with an * underscore. * * @return The container name for the base DN. */ public String getContainerName() { String normStr = baseDN.toNormalizedString(); StringBuilder builder = new StringBuilder(normStr.length()); for (int i = 0; i < normStr.length(); i++) { char ch = normStr.charAt(i); if (Character.isLetterOrDigit(ch)) { builder.append(ch); } else { builder.append('_'); } } return builder.toString(); } /** * Get the baseDN this entry container is responsible for. * * @return The Base DN for this entry container. */ public DN getBaseDN() { return baseDN; } } opends/src/server/org/opends/server/backends/jeb/ExportJob.java
@@ -30,11 +30,9 @@ import com.sleepycat.je.CursorConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.Environment; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; import org.opends.server.api.Backend; import org.opends.server.types.DN; import org.opends.server.types.Entry; import org.opends.server.types.ErrorLogCategory; @@ -44,10 +42,7 @@ import org.opends.server.util.StaticUtils; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Timer; import java.util.TimerTask; import java.util.*; import static org.opends.server.loggers.Debug.debugException; import static org.opends.server.messages.MessageHandler.getMessage; @@ -71,16 +66,6 @@ private LDIFExportConfig exportConfig; /** * The JE backend instance to be exported. */ private Backend backend; /** * The configuration of the JE backend instance. */ private Config config; /** * The number of milliseconds between job progress reports. */ private long progressInterval = 10000; @@ -98,43 +83,38 @@ /** * Create a new export job. * * @param backend The JE backend performing the export job. * @param config The JE backend configuration. * @param exportConfig The requested LDIF export configuration. */ public ExportJob(Backend backend, Config config, LDIFExportConfig exportConfig) public ExportJob(LDIFExportConfig exportConfig) { this.exportConfig = exportConfig; this.backend = backend; this.config = config; } /** * Export entries from the backend to an LDIF file. * @param env A handle to the JE database environment of the backend. * @param rootContainer The root container to export. * @throws DatabaseException If an error occurs in the JE database. * @throws IOException If an I/O error occurs while writing an entry. * @throws JebException If an error occurs in the JE backend. * @throws LDIFException If an error occurs while trying to determine whether * to write an entry. */ public void exportLDIF(Environment env) public void exportLDIF(RootContainer rootContainer) throws IOException, LDIFException, DatabaseException, JebException { // Open the containers read-only. List<DN> includeBranches = exportConfig.getIncludeBranches(); DN baseDNs[] = config.getBaseDNs(); ArrayList<EntryContainer> containers = new ArrayList<EntryContainer>(baseDNs.length); for (DN baseDN : baseDNs) DN baseDN; ArrayList<EntryContainer> exportContainers = new ArrayList<EntryContainer>(); for (EntryContainer entryContainer : rootContainer.getEntryContainers()) { // Skip containers that are not covered by the include branches. baseDN = entryContainer.getBaseDN(); boolean includeBase = false; if (includeBranches == null || includeBranches.isEmpty()) { includeBase = true; exportContainers.add(entryContainer); } else { @@ -143,35 +123,15 @@ if (includeBranch.isDescendantOf(baseDN) || includeBranch.isAncestorOf(baseDN)) { includeBase = true; exportContainers.add(entryContainer); } } } if (includeBase) { String containerName = BackendImpl.getContainerName(baseDN); Container container = new Container(env, containerName); EntryContainer entryContainer = new EntryContainer(backend, config, container); if (env.getConfig().getReadOnly()) { entryContainer.openReadOnly(); } else { entryContainer.open(); } containers.add(entryContainer); } } // Make a note of the time we started. long startTime = System.currentTimeMillis(); try { // Start a timer for the progress report. Timer timer = new Timer(); TimerTask progressTask = new ProgressTask(); @@ -181,23 +141,16 @@ // Iterate through the containers. try { for (EntryContainer ec : containers) for (EntryContainer exportContainer : exportContainers) { exportContainer(ec); exportContainer(exportContainer); } } finally { timer.cancel(); } } finally { for (EntryContainer ec : containers) { ec.close(); } } long finishTime = System.currentTimeMillis(); long totalTime = (finishTime - startTime); @@ -217,9 +170,10 @@ } /** * Export the entries in a single entry container, in other words from * Export the entries in a single entry entryContainer, in other words from * one of the base DNs. * @param entryContainer The entry container of those entries to be exported. * @param entryContainer The entry container that holds the entries to be * exported. * @throws DatabaseException If an error occurs in the JE database. * @throws IOException If an error occurs while writing an entry. * @throws LDIFException If an error occurs while trying to determine opends/src/server/org/opends/server/backends/jeb/ID2Entry.java
@@ -49,9 +49,9 @@ public class ID2Entry { /** * The database container. * The database entryContainer. */ private Container container; private EntryContainer entryContainer; /** * The JE database configuration. @@ -64,7 +64,7 @@ private DataConfig dataConfig; /** * The name of the database within the container. * The name of the database within the entryContainer. */ private String name; @@ -76,17 +76,17 @@ /** * Create a new ID2Entry object. * @param container The container of the entry database. * @param entryContainer The entryContainer of the entry database. * @param dbConfig The JE database configuration to be used to open the * underlying JE database. * @param dataConfig The desired compression and encryption options for data * stored in the entry database. * @param name The name of the entry database. */ public ID2Entry(Container container, DatabaseConfig dbConfig, public ID2Entry(EntryContainer entryContainer, DatabaseConfig dbConfig, DataConfig dataConfig, String name) { this.container = container; this.entryContainer = entryContainer; this.dbConfig = dbConfig; this.name = name; this.dataConfig = dataConfig; @@ -104,7 +104,7 @@ /** * Get a handle to the database. It returns a per-thread handle to avoid * any thread contention on the database handle. The container is * any thread contention on the database handle. The entryContainer is * responsible for closing all handles. * * @return A database handle. @@ -116,7 +116,7 @@ Database database = threadLocalDatabase.get(); if (database == null) { database = container.openDatabase(dbConfig, name); database = entryContainer.openDatabase(dbConfig, name); threadLocalDatabase.set(database); } return database; @@ -152,7 +152,7 @@ DatabaseEntry data = entryData(entry); OperationStatus status; status = Container.insert(getDatabase(), txn, key, data); status = EntryContainer.insert(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -176,7 +176,7 @@ DatabaseEntry data = entryData(entry); OperationStatus status; status = Container.put(getDatabase(), txn, key, data); status = EntryContainer.put(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -197,7 +197,7 @@ throws DatabaseException { OperationStatus status; status = Container.put(getDatabase(), txn, key, data); status = EntryContainer.put(getDatabase(), txn, key, data); if (status != OperationStatus.SUCCESS) { return false; @@ -218,7 +218,7 @@ { DatabaseEntry key = id.getDatabaseEntry(); OperationStatus status = Container.delete(getDatabase(), txn, key); OperationStatus status = EntryContainer.delete(getDatabase(), txn, key); if (status != OperationStatus.SUCCESS) { return false; @@ -242,7 +242,8 @@ DatabaseEntry data = new DatabaseEntry(); OperationStatus status; status = Container.read(getDatabase(), txn, key, data, LockMode.DEFAULT); status = EntryContainer.read(getDatabase(), txn, key, data, LockMode.DEFAULT); if (status != OperationStatus.SUCCESS) { @@ -311,7 +312,7 @@ key = id.getDatabaseEntry(); // Read the current count, if any. OperationStatus status = Container.read(getDatabase(), txn, OperationStatus status = EntryContainer.read(getDatabase(), txn, key, data, LockMode.DEFAULT); // Parse the current count. @@ -343,7 +344,7 @@ key = id.getDatabaseEntry(); // Read the current count, if any. OperationStatus status = Container.read(getDatabase(), txn, OperationStatus status = EntryContainer.read(getDatabase(), txn, key, data, LockMode.RMW); // Parse the current count. @@ -359,6 +360,6 @@ // Write it. byte[] bytes = JebFormat.entryIDToDatabase(count); data.setData(bytes); Container.put(getDatabase(), txn, key, data); EntryContainer.put(getDatabase(), txn, key, data); } } opends/src/server/org/opends/server/backends/jeb/ImportContext.java
@@ -41,7 +41,7 @@ public class ImportContext { /** * The name of the container for the destination base DN. * The name of the entryContainer for the destination base DN. */ private String containerName; @@ -66,7 +66,7 @@ private LDIFReader ldifReader; /** * The entry container for the destination base DN. * The entry entryContainer for the destination base DN. */ private EntryContainer entryContainer; @@ -120,8 +120,8 @@ } /** * Set the name of the container for the destination base DN. * @param containerName The container name. * Set the name of the entryContainer for the destination base DN. * @param containerName The entryContainer name. */ public void setContainerName(String containerName) { @@ -129,8 +129,8 @@ } /** * Get the name of the container for the destination base DN. * @return The container name. * Get the name of the entryContainer for the destination base DN. * @return The entryContainer name. */ public String getContainerName() { @@ -210,8 +210,8 @@ } /** * Set the entry container for the destination base DN. * @param entryContainer The entry container for the destination base DN. * Set the entry entryContainer for the destination base DN. * @param entryContainer The entry entryContainer for the destination base DN. */ public void setEntryContainer(EntryContainer entryContainer) { @@ -219,8 +219,8 @@ } /** * Get the entry container for the destination base DN. * @return The entry container for the destination base DN. * Get the entry entryContainer for the destination base DN. * @return The entry entryContainer for the destination base DN. */ public EntryContainer getEntryContainer() { opends/src/server/org/opends/server/backends/jeb/ImportJob.java
@@ -27,8 +27,6 @@ package org.opends.server.backends.jeb; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentStats; import com.sleepycat.je.StatsConfig; import com.sleepycat.je.Transaction; @@ -89,9 +87,9 @@ private Config config; /** * The database environment. * The root container used for this import job. */ private Environment env; private RootContainer rootContainer; /** * The LDIF import configuration. @@ -151,12 +149,9 @@ * reading, or while reading from the LDIF file. * @throws JebException If an error occurs in the JE backend. */ public void importLDIF() throws DatabaseException, IOException, JebException public void importLDIF() throws DatabaseException, IOException, JebException { File backendDirectory = config.getBackendDirectory(); EnvironmentConfig envConfig = config.getEnvironmentConfig(); envConfig.setConfigParam("je.env.runCheckpointer", "false"); /* envConfig.setConfigParam("je.env.runCleaner", "false"); envConfig.setConfigParam("je.log.numBuffers", "2"); @@ -164,28 +159,30 @@ envConfig.setConfigParam("je.log.totalBufferBytes", "30000000"); envConfig.setConfigParam("je.log.fileMax", "100000000"); */ rootContainer = new RootContainer(config, backend); if (ldifImportConfig.appendToExistingData()) { envConfig.setTransactional(true); envConfig.setTxnNoSync(true); rootContainer.open(config.getBackendDirectory(), config.getBackendPermission(), false, true, true, true, true, false); } else { envConfig.setTransactional(false); envConfig.setConfigParam("je.env.isLocking", "false"); rootContainer.open(config.getBackendDirectory(), config.getBackendPermission(), false, true, false, false, false, false); } env = new Environment(backendDirectory, envConfig); if (!ldifImportConfig.appendToExistingData()) { // We have the writer lock on the environment, now delete the // environment and re-open it. Only do this when we are // importing to all the base DNs in the backend. env.close(); EnvManager.removeFiles(backendDirectory.getPath()); env = new Environment(backendDirectory, envConfig); rootContainer.close(); EnvManager.removeFiles(config.getBackendDirectory().getPath()); rootContainer.open(config.getBackendDirectory(), config.getBackendPermission(), false, true, false, false, false, false); } // Divide the total buffer size by the number of threads @@ -205,49 +202,25 @@ message, msgID); msgID = MSGID_JEB_IMPORT_ENVIRONMENT_CONFIG; message = getMessage(msgID, env.getConfig().toString()); message = getMessage(msgID, rootContainer.getEnvironmentConfig().toString()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "importLDIF", env.getConfig().toString()); rootContainer.getEnvironmentConfig().toString()); // Create and open the containers for each base DN. rootContainer.openEntryContainers(config.getBaseDNs()); // Create the import contextes for each base DN. EntryID highestID = null; for (DN baseDN : config.getBaseDNs()) DN baseDN; for (EntryContainer entryContainer : rootContainer.getEntryContainers()) { String containerName = BackendImpl.getContainerName(baseDN); Container container = new Container(env, containerName); EntryContainer entryContainer = new EntryContainer(backend, config, container); if (ldifImportConfig.appendToExistingData()) { entryContainer.open(); } else { // We will need this code when the import can specify a subset // of the base DNs in the backend. /* long t1, t2; String msg = String.format("Removing existing data for base DN '%s'", baseDN); System.out.println(msg); t1 = System.currentTimeMillis(); container.removeAllDatabases(); t2 = System.currentTimeMillis(); msg = String.format("Data removed in %d seconds", (t2-t1)/1000); System.out.println(msg); */ entryContainer.openNonTransactional(true); } baseDN = entryContainer.getBaseDN(); // Keep track of the highest entry ID. EntryID id = entryContainer.getHighestEntryID(); @@ -263,7 +236,7 @@ importContext.setLDIFImportConfig(this.ldifImportConfig); importContext.setBaseDN(baseDN); importContext.setContainerName(containerName); importContext.setContainerName(entryContainer.getContainerName()); importContext.setEntryContainer(entryContainer); importContext.setBufferSize(bufferSize); @@ -344,17 +317,13 @@ } finally { for (ImportContext ic : importMap.values()) { ic.getEntryContainer().close(); } rootContainer.close(); // Sync the environment to disk. msgID = MSGID_JEB_IMPORT_CLOSING_DATABASE; message = getMessage(msgID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); env.close(); } long finishTime = System.currentTimeMillis(); @@ -923,7 +892,7 @@ public ProgressTask() throws DatabaseException { previousTime = System.currentTimeMillis(); prevEnvStats = env.getStats(new StatsConfig()); prevEnvStats = rootContainer.getEnvironmentStats(new StatsConfig()); } /** @@ -957,7 +926,8 @@ Runtime runtime = Runtime.getRuntime(); long freeMemory = runtime.freeMemory() / bytesPerMegabyte; EnvironmentStats envStats = env.getStats(new StatsConfig()); EnvironmentStats envStats = rootContainer.getEnvironmentStats(new StatsConfig()); long nCacheMiss = envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
@@ -58,17 +58,17 @@ private ImportContext importContext; /** * The destination entry container for entries read from the queue. * The destination entry entryContainer for entries read from the queue. */ private EntryContainer entryContainer; /** * The entry database of the destination entry container. * The entry database of the destination entry entryContainer. */ private ID2Entry id2entry; /** * The referral database of the destination entry container. * The referral database of the destination entry entryContainer. */ private DN2URI dn2uri;
@@ -65,9 +65,9 @@ /** * The database container holding this index database. * The database entryContainer holding this index database. */ private Container container; private EntryContainer entryContainer; /** * The JE database configuration. @@ -75,7 +75,7 @@ private DatabaseConfig dbConfig; /** * The name of the database within the container. * The name of the database within the entryContainer. */ private String name; @@ -114,8 +114,8 @@ /** * Create a new index object. * @param container The database container holding this index. * @param name The name of the index database within the container. * @param entryContainer The database entryContainer holding this index. * @param name The name of the index database within the entryContainer. * @param indexer The indexer object to construct index keys from LDAP * attribute values. * @param indexEntryLimit The configured limit on the number of entry IDs @@ -123,10 +123,10 @@ * @param cursorEntryLimit The configured limit on the number of entry IDs * that may be retrieved by cursoring through an index. */ public Index(Container container, String name, Indexer indexer, public Index(EntryContainer entryContainer, String name, Indexer indexer, int indexEntryLimit, int cursorEntryLimit) { this.container = container; this.entryContainer = entryContainer; this.name = name; this.indexer = indexer; this.comparator = indexer.getComparator(); @@ -148,7 +148,7 @@ /** * Get a handle to the database. It returns a per-thread handle to avoid * any thread contention on the database handle. The container is * any thread contention on the database handle. The entryContainer is * responsible for closing all handles. * * @return A database handle. @@ -160,7 +160,7 @@ Database database = threadLocalDatabase.get(); if (database == null) { database = container.openDatabase(dbConfig, name); database = entryContainer.openDatabase(dbConfig, name); threadLocalDatabase.set(database); } return database; @@ -182,7 +182,7 @@ DatabaseEntry entryIDData = entryID.getDatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); status = Container.read(getDatabase(), txn, key, data, lockMode); status = EntryContainer.read(getDatabase(), txn, key, data, lockMode); if (status == OperationStatus.SUCCESS) { @@ -202,12 +202,12 @@ byte[] after = entryIDList.toDatabase(); data.setData(after); Container.put(getDatabase(), txn, key, data); EntryContainer.put(getDatabase(), txn, key, data); } } else { Container.put(getDatabase(), txn, key, entryIDData); EntryContainer.put(getDatabase(), txn, key, entryIDData); } } @@ -226,7 +226,7 @@ LockMode lockMode = LockMode.RMW; DatabaseEntry data = new DatabaseEntry(); status = Container.read(getDatabase(), txn, key, data, lockMode); status = EntryContainer.read(getDatabase(), txn, key, data, lockMode); if (status == OperationStatus.SUCCESS) { @@ -244,12 +244,12 @@ if (after == null) { // No more IDs, so remove the key Container.delete(getDatabase(), txn, key); EntryContainer.delete(getDatabase(), txn, key); } else { data.setData(after); Container.put(getDatabase(), txn, key, data); EntryContainer.put(getDatabase(), txn, key, data); } } } @@ -280,7 +280,7 @@ LockMode lockMode = LockMode.DEFAULT; DatabaseEntry data = new DatabaseEntry(); status = Container.read(getDatabase(), txn, key, data, lockMode); status = EntryContainer.read(getDatabase(), txn, key, data, lockMode); if (status == OperationStatus.SUCCESS) { EntryIDSet entryIDList = @@ -320,7 +320,7 @@ { OperationStatus status; DatabaseEntry data = new DatabaseEntry(); status = Container.read(getDatabase(), txn, key, data, lockMode); status = EntryContainer.read(getDatabase(), txn, key, data, lockMode); if (status != OperationStatus.SUCCESS) { return new EntryIDSet(key.getData(), null); @@ -351,7 +351,7 @@ if (after == null) { // No more IDs, so remove the key. Container.delete(getDatabase(), txn, key); EntryContainer.delete(getDatabase(), txn, key); } else { @@ -360,7 +360,7 @@ entryLimitExceededCount++; } data.setData(after); Container.put(getDatabase(), txn, key, data); EntryContainer.put(getDatabase(), txn, key, data); } }
@@ -48,7 +48,7 @@ public static final int FILTER_CANDIDATE_THRESHOLD = 10; /** * The entry container holding the attribute indexes. * The entry entryContainer holding the attribute indexes. */ private EntryContainer entryContainer; @@ -67,7 +67,7 @@ /** * Construct an index filter for a search operation. * * @param entryContainer The entry container. * @param entryContainer The entry entryContainer. * @param searchOp The search operation to be evaluated. * * @param debugBuilder If not null, a diagnostic string will be written opends/src/server/org/opends/server/backends/jeb/RootContainer.java
New file @@ -0,0 +1,622 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at * trunk/opends/resource/legal-notices/OpenDS.LICENSE * or https://OpenDS.dev.java.net/OpenDS.LICENSE. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at * trunk/opends/resource/legal-notices/OpenDS.LICENSE. If applicable, * add the following below this CDDL HEADER, with the fields enclosed * by brackets "[]" replaced with your own identifying * information: * Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * * Portions Copyright 2006 Sun Microsystems, Inc. */ package org.opends.server.backends.jeb; import com.sleepycat.je.Database; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentStats; import com.sleepycat.je.PreloadConfig; import com.sleepycat.je.PreloadStats; import com.sleepycat.je.PreloadStatus; import com.sleepycat.je.config.EnvironmentParams; import com.sleepycat.je.config.ConfigParam; import com.sleepycat.je.StatsConfig; import com.sleepycat.je.CheckpointConfig; import java.util.concurrent.ConcurrentHashMap; import java.util.*; import java.io.File; import java.io.FilenameFilter; import org.opends.server.monitors.DatabaseEnvironmentMonitor; import org.opends.server.types.*; import org.opends.server.loggers.Debug; import static org.opends.server.loggers.Error.logError; import static org.opends.server.loggers.Debug.debugException; import static org.opends.server.loggers.Debug.debugEnter; import static org.opends.server.messages.MessageHandler.getMessage; import static org.opends.server.messages.JebMessages. MSGID_JEB_CACHE_SIZE_AFTER_PRELOAD; import static org.opends.server.messages.JebMessages. MSGID_JEB_CLEAN_DATABASE_START; import static org.opends.server.messages.JebMessages. MSGID_JEB_CLEAN_DATABASE_MARKED; import static org.opends.server.messages.JebMessages. MSGID_JEB_CLEAN_DATABASE_FINISH; import static org.opends.server.messages.JebMessages. MSGID_JEB_SET_PERMISSIONS_FAILED; import org.opends.server.api.Backend; /** * Wrapper class for the JE environment. Root container holds all the entry * containers for each base DN. It also maintains all the openings and closings * of the entry containers. */ public class RootContainer { /** * The fully-qualified name of this class for debugging purposes. */ private static final String CLASS_NAME = "org.opends.server.backends.jeb.RootContainer"; /** * The JE database environment. */ private Environment env; /** * The backend configuration. */ private Config config; /** * The backend to which this entry root container belongs. */ private Backend backend; /** * The database environment monitor for this JE environment. */ private DatabaseEnvironmentMonitor monitor; /** * A configurable component to handle changes to the configuration of * the database environment. */ private ConfigurableEnvironment configurableEnv; /** * The base DNs contained in this entryContainer. */ private ConcurrentHashMap<DN, EntryContainer> entryContainers; /** * Creates a new RootContainer object. Each root container represents a JE * environment. * * @param config The configuration of the JE backend. * @param backend A reference to the JE back end that is creating this * root container. */ public RootContainer(Config config, Backend backend) { this.env = null; this.configurableEnv = null; this.monitor = null; this.entryContainers = new ConcurrentHashMap<DN, EntryContainer>(); this.backend = backend; this.config = config; } /** * Helper method to apply database directory permissions and create a new * JE environment. * * @param backendDirectory The environment home directory for JE. * @param backendPermission The file permissions for the environment home * directory. * @param envConfig The JE environment configuration. * @throws DatabaseException If an error occurs when creating the environment. */ private void open(File backendDirectory, FilePermission backendPermission, EnvironmentConfig envConfig) throws DatabaseException { // Get the backend database backendDirectory permissions and apply try { if(!FilePermission.setPermissions(backendDirectory, backendPermission)) { throw new Exception(); } } catch(Exception e) { // Log an warning that the permissions were not set. int msgID = MSGID_JEB_SET_PERMISSIONS_FAILED; String message = getMessage(msgID, backendDirectory.getPath()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_WARNING, message, msgID); } // Open the database environment env = new Environment(backendDirectory, envConfig); Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "initializeBackend", env.getConfig().toString()); } /** * Opens the root container. * * @throws DatabaseException If an error occurs when opening the container. */ public void open() throws DatabaseException { open(config.getBackendDirectory(), config.getBackendPermission(), config.getEnvironmentConfig()); } /** * Opens the root container using the configuration parameters provided. Any * configuration parameters provided will override the parameters in the * JE configuration object. * * @param backendDirectory The environment home directory for JE. * @param backendPermission he file permissions for the environment home * directory. * @param readOnly Open the container in read only mode. * @param allowCreate Allow creating new entries in the container. * @param transactional Use transactions on operations. * @param txnNoSync Use asynchronous transactions. * @param isLocking Create the environment with locking. * @param runCheckPointer Start the checkpointer. * @throws DatabaseException If an error occurs when openinng the container. */ public void open(File backendDirectory, FilePermission backendPermission, boolean readOnly, boolean allowCreate, boolean transactional, boolean txnNoSync, boolean isLocking, boolean runCheckPointer) throws DatabaseException { EnvironmentConfig envConfig; if(config.getEnvironmentConfig() != null) { envConfig = config.getEnvironmentConfig(); } else { envConfig = new EnvironmentConfig(); } envConfig.setReadOnly(readOnly); envConfig.setAllowCreate(allowCreate); envConfig.setTransactional(transactional); envConfig.setTxnNoSync(txnNoSync); envConfig.setConfigParam("je.env.isLocking", String.valueOf(isLocking)); envConfig.setConfigParam("je.env.runCheckpointer", String.valueOf(runCheckPointer)); open(backendDirectory, backendPermission, envConfig); } /** * Opens the entry container for a base DN. If the entry container does not * exist for the base DN, it will be created. The entry container will be * opened with the same mode as the root container. Any entry containers * opened in a read only root container will also be read only. Any entry * containers opened in a non transactional root container will also be non * transactional. * * @param baseDN The base DN of the entry container to open. * @return The opened entry container. * @throws DatabaseException If an error occurs while opening the entry * container. */ public EntryContainer openEntryContainer(DN baseDN) throws DatabaseException { EntryContainer ec = new EntryContainer(baseDN, backend, config, env); if(env.getConfig().getReadOnly()) { ec.openReadOnly(); } else if(!env.getConfig().getTransactional()) { ec.openNonTransactional(true); } else { ec.open(); } this.entryContainers.put(baseDN, ec); return ec; } /** * Opens the entry containers for multiple base DNs. * * @param baseDNs The base DNs of the entry containers to open. * @throws DatabaseException If an error occurs while opening the entry * container. */ public void openEntryContainers(DN[] baseDNs) throws DatabaseException { for(DN baseDN : baseDNs) { openEntryContainer(baseDN); } } /** * Close the entry container for a base DN. * * @param baseDN The base DN of the entry container to close. * @throws DatabaseException If an error occurs while closing the entry * container. */ public void closeEntryContainer(DN baseDN) throws DatabaseException { getEntryContainer(baseDN).close(); entryContainers.remove(baseDN); } /** * Close and remove a entry container for a base DN from disk. * * @param baseDN The base DN of the entry container to remove. * @throws DatabaseException If an error occurs while removing the entry * container. */ public void removeEntryContainer(DN baseDN) throws DatabaseException { getEntryContainer(baseDN).close(); getEntryContainer(baseDN).removeContainer(); entryContainers.remove(baseDN); } /** * Get the ConfigurableEnvironment object for JE environment used by this * root container. * * @return The ConfigurableEnvironment object. */ public ConfigurableEnvironment getConfigurableEnvironment() { if(configurableEnv == null) { DN envConfigDN = config.getEnvConfigDN(); if (envConfigDN != null) { configurableEnv = new ConfigurableEnvironment(envConfigDN, env); } } return configurableEnv; } /** * Get the DatabaseEnvironmentMonitor object for JE environment used by this * root container. * * @return The DatabaseEnvironmentMonito object. */ public DatabaseEnvironmentMonitor getMonitorProvider() { if(monitor == null) { String monitorName = backend.getBackendID() + " Database Environment"; monitor = new DatabaseEnvironmentMonitor(monitorName, env); } return monitor; } /** * Preload the database cache. There is no preload if the configured preload * time limit is zero. */ public void preload() { assert debugEnter(CLASS_NAME, "preload"); long timeLimit = config.getPreloadTimeLimit(); if (timeLimit > 0) { // Get a list of all the databases used by the backend. ArrayList<Database> dbList = new ArrayList<Database>(); for (EntryContainer ec : entryContainers.values()) { ec.listDatabases(dbList); } // Sort the list in order of priority. Collections.sort(dbList, new DbPreloadComparator()); // Preload each database until we reach the time limit or the cache // is filled. try { long timeEnd = System.currentTimeMillis() + timeLimit; // Configure preload of Leaf Nodes (LNs) containing the data values. PreloadConfig preloadConfig = new PreloadConfig(); preloadConfig.setLoadLNs(true); for (Database db : dbList) { // Calculate the remaining time. long timeRemaining = timeEnd - System.currentTimeMillis(); if (timeRemaining <= 0) { break; } preloadConfig.setMaxMillisecs(timeRemaining); PreloadStats preloadStats = db.preload(preloadConfig); /* System.out.println("file=" + db.getDatabaseName() + " LNs=" + preloadStats.getNLNsLoaded()); */ // Stop if the cache is full or the time limit has been exceeded. if (preloadStats.getStatus() != PreloadStatus.SUCCESS) { break; } } // Log an informational message about the size of the cache. EnvironmentStats stats = env.getStats(new StatsConfig()); long total = stats.getCacheTotalBytes(); int msgID = MSGID_JEB_CACHE_SIZE_AFTER_PRELOAD; String message = getMessage(msgID, total / (1024 * 1024)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } catch (DatabaseException e) { assert debugException(CLASS_NAME, "preload", e); } } } /** * Synchronously invokes the cleaner on the database environment then forces a * checkpoint to delete the log files that are no longer in use. * * @throws DatabaseException If an error occurs while cleaning the database * environment. */ private void cleanDatabase() throws DatabaseException { assert debugEnter(CLASS_NAME, "cleanDatabase"); int msgID; String message; FilenameFilter filenameFilter = new FilenameFilter() { public boolean accept(File d, String name) { return name.endsWith(".jdb"); } }; File backendDirectory = env.getHome(); int beforeLogfileCount = backendDirectory.list(filenameFilter).length; msgID = MSGID_JEB_CLEAN_DATABASE_START; message = getMessage(msgID, beforeLogfileCount, backendDirectory.getPath()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); int currentCleaned = 0; int totalCleaned = 0; while ((currentCleaned = env.cleanLog()) > 0) { totalCleaned += currentCleaned; } msgID = MSGID_JEB_CLEAN_DATABASE_MARKED; message = getMessage(msgID, totalCleaned); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); if (totalCleaned > 0) { CheckpointConfig force = new CheckpointConfig(); force.setForce(true); env.checkpoint(force); } int afterLogfileCount = backendDirectory.list(filenameFilter).length; msgID = MSGID_JEB_CLEAN_DATABASE_FINISH; message = getMessage(msgID, afterLogfileCount); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } /** * Close the root entryContainer. * * @throws DatabaseException If an error occurs while attempting to close * the entryContainer. */ public void close() throws DatabaseException { for(DN baseDN : entryContainers.keySet()) { entryContainers.get(baseDN).close(); entryContainers.remove(baseDN); } env.close(); } /** * Return all the entry containers in this root container. * * @return The entry containers in this root container. */ public Collection<EntryContainer> getEntryContainers() { return entryContainers.values(); } /** * Returns all the baseDNs this root container stores. * * @return The set of DNs this root container stores. */ public Set<DN> getBaseDNs() { return entryContainers.keySet(); } /** * Return the entry container for a specific base DN. * * @param baseDN The base DN of the entry container to retrive. * @return The entry container for the base DN. */ public EntryContainer getEntryContainer(DN baseDN) { EntryContainer ec = null; DN nodeDN = baseDN; while (ec == null && nodeDN != null) { ec = entryContainers.get(nodeDN); if (ec == null) { nodeDN = nodeDN.getParent(); } } return ec; } /** * Apply new configuration to the JE environment. * * @param newConfig The new configuration to apply. * @throws DatabaseException If an error occurs while applying the new * configuration. */ public void applyNewConfig(Config newConfig) throws DatabaseException { // Check for changes to the database directory permissions FilePermission oldPermission = config.getBackendPermission(); FilePermission newPermission = newConfig.getBackendPermission(); if(!FilePermission.toUNIXMode(oldPermission).equals( FilePermission.toUNIXMode(newPermission))) { try { if(!FilePermission.setPermissions(newConfig.getBackendDirectory(), newPermission)) { throw new Exception(); } } catch(Exception e) { // Log an warning that the permissions were not set. int msgID = MSGID_JEB_SET_PERMISSIONS_FAILED; String message = getMessage(msgID, config.getBackendDirectory().getPath()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_WARNING, message, msgID); } } // Check if any JE non-mutable properties were changed. EnvironmentConfig oldEnvConfig = this.config.getEnvironmentConfig(); EnvironmentConfig newEnvConfig = newConfig.getEnvironmentConfig(); Map paramsMap = EnvironmentParams.SUPPORTED_PARAMS; for (Object o : paramsMap.values()) { ConfigParam param = (ConfigParam)o; if (!param.isMutable()) { String oldValue = oldEnvConfig.getConfigParam(param.getName()); String newValue = newEnvConfig.getConfigParam(param.getName()); if (!oldValue.equalsIgnoreCase(newValue)) { System.out.println("The change to the following property will " + "take effect when the backend is restarted: " + param.getName()); } } } // This takes care of changes to the JE environment for those // properties that are mutable at runtime. env.setMutableConfig(newConfig.getEnvironmentConfig()); config = newConfig; Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "applyNewConfiguration", env.getConfig().toString()); } /** * Get the environment stats of the JE environment used in this root * container. * * @param statsConfig The configuration to use for the EnvironmentStats * object. * @return The environment status of the JE environment. * @throws DatabaseException If an error occurs while retriving the stats * object. */ public EnvironmentStats getEnvironmentStats(StatsConfig statsConfig) throws DatabaseException { return env.getStats(statsConfig); } /** * Get the environment config of the JE environment used in this root * container. * * @return The environment config of the JE environment. * @throws DatabaseException If an error occurs while retriving the * configuration object. */ public EnvironmentConfig getEnvironmentConfig() throws DatabaseException { return env.getConfig(); } } opends/src/server/org/opends/server/backends/jeb/VerifyJob.java
@@ -32,26 +32,20 @@ import com.sleepycat.je.CursorConfig; import com.sleepycat.je.DatabaseEntry; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.Environment; import com.sleepycat.je.EnvironmentConfig; import com.sleepycat.je.EnvironmentStats; import com.sleepycat.je.LockMode; import com.sleepycat.je.OperationStatus; import com.sleepycat.je.StatsConfig; import com.sleepycat.je.Transaction; import org.opends.server.api.Backend; import org.opends.server.api.OrderingMatchingRule; import org.opends.server.core.DirectoryServer; import org.opends.server.loggers.Debug; import org.opends.server.protocols.asn1.ASN1OctetString; import org.opends.server.types.Attribute; import org.opends.server.types.AttributeType; import org.opends.server.types.AttributeValue; import org.opends.server.types.ByteString; import org.opends.server.types.ConditionResult; import org.opends.server.types.DebugLogCategory; import org.opends.server.types.DebugLogSeverity; import org.opends.server.types.DirectoryException; import org.opends.server.types.DN; import org.opends.server.types.Entry; @@ -65,7 +59,6 @@ import static org.opends.server.messages.MessageHandler.getMessage; import static org.opends.server.messages.JebMessages.*; import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -94,19 +87,14 @@ private VerifyConfig verifyConfig; /** * The JE backend to be verified. */ private Backend backend; /** * The configuration of the JE backend. */ private Config config; /** * A read-only JE database environment handle for the purpose of verification. * The root container used for the verify job. */ private Environment env; RootContainer rootContainer; /** * The number of milliseconds between job progress reports. @@ -193,44 +181,29 @@ /** * Construct a VerifyJob. * * @param backend The backend performing the verify process. * @param config The backend configuration. * @param verifyConfig The verify configuration. */ public VerifyJob(Backend backend, Config config, VerifyConfig verifyConfig) public VerifyJob(Config config, VerifyConfig verifyConfig) { this.verifyConfig = verifyConfig; this.backend = backend; this.config = config; } /** * Verify the backend. * * @param rootContainer The root container that holds the entries to verify. * @throws DatabaseException If an error occurs in the JE database. * @throws JebException If an error occurs in the JE backend. */ public void verifyBackend() throws DatabaseException, JebException public void verifyBackend(RootContainer rootContainer) throws DatabaseException, JebException { File backendDirectory = config.getBackendDirectory(); // Open the environment read-only. EnvironmentConfig envConfig = config.getEnvironmentConfig(); envConfig.setReadOnly(true); envConfig.setAllowCreate(false); envConfig.setTransactional(false); env = new Environment(backendDirectory, envConfig); Debug.debugMessage(DebugLogCategory.BACKEND, DebugLogSeverity.INFO, CLASS_NAME, "verifyBackend", env.getConfig().toString()); // Open a container read-only. String containerName = BackendImpl.getContainerName(verifyConfig.getBaseDN()); Container container = new Container(env, containerName); this.rootContainer = rootContainer; EntryContainer entryContainer = new EntryContainer(backend, config, container); entryContainer.openReadOnly(); rootContainer.getEntryContainer(verifyConfig.getBaseDN()); ArrayList<String> completeList = verifyConfig.getCompleteList(); ArrayList<String> cleanList = verifyConfig.getCleanList(); @@ -304,7 +277,7 @@ // We will be updating these files independently of the indexes // so we need direct access to them rather than going through // the entry container methods. // the entry entryContainer methods. id2entry = entryContainer.getID2Entry(); dn2id = entryContainer.getDN2ID(); id2c = entryContainer.getID2Children(); @@ -1567,7 +1540,8 @@ public ProgressTask() throws DatabaseException { previousTime = System.currentTimeMillis(); prevEnvStats = env.getStats(new StatsConfig()); prevEnvStats = rootContainer.getEnvironmentStats(new StatsConfig()); } /** @@ -1597,7 +1571,8 @@ Runtime runtime = Runtime.getRuntime(); long freeMemory = runtime.freeMemory() / bytesPerMegabyte; EnvironmentStats envStats = env.getStats(new StatsConfig()); EnvironmentStats envStats = rootContainer.getEnvironmentStats(new StatsConfig()); long nCacheMiss = envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
@@ -35,8 +35,11 @@ import java.util.ArrayList; import org.opends.server.TestCaseUtils; import org.opends.server.core.DirectoryServer; import org.opends.server.types.Entry; import org.opends.server.types.LDIFImportConfig; import org.opends.server.types.FilePermission; import org.opends.server.types.DN; import org.opends.server.util.LDIFReader; import org.testng.annotations.Test; import org.testng.annotations.BeforeClass; @@ -181,14 +184,14 @@ @Test() public void test1() throws Exception { EnvManager.createHomeDir(homeDirName); EnvironmentConfig envConfig = new EnvironmentConfig(); envConfig.setTransactional(true); envConfig.setAllowCreate(true); Environment env = new Environment(new File(homeDirName), envConfig); EntryContainer entryContainer = new EntryContainer(null, new Config(), new Container(env, null)); RootContainer rootContainer = new RootContainer(new Config(), null); rootContainer.open(new File(homeDirName), new FilePermission(true, true, true), false, true, true, false, true, true); entryContainer.open(); EntryContainer entryContainer = rootContainer.openEntryContainer(DirectoryServer.getSchemaDN()); EntryID actualHighestID = entryContainer.getHighestEntryID(); assertTrue(actualHighestID.equals(new EntryID(0))); @@ -201,9 +204,7 @@ actualHighestID = entryContainer.getHighestEntryID(); assertTrue(actualHighestID.equals(new EntryID(calculatedHighestID))); entryContainer.close(); env.close(); rootContainer.close(); EnvManager.removeFiles(homeDirName); } }
