mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
opends/resource/schema/02-config.ldif
@@ -1498,6 +1498,10 @@ attributeTypes: ( 1.3.6.1.4.1.26027.1.1.450 NAME 'ds-cfg-message-body' SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE X-ORIGIN 'OpenDS Directory Server' ) attributeTypes: ( 1.3.6.1.4.1.26027.1.1.451 NAME 'ds-task-import-clear-backend' SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE X-ORIGIN 'OpenDS Directory Server' ) objectClasses: ( 1.3.6.1.4.1.26027.1.2.1 NAME 'ds-cfg-access-control-handler' SUP top STRUCTURAL MUST ( cn $ ds-cfg-acl-handler-class $ ds-cfg-acl-handler-enabled ) @@ -1836,14 +1840,15 @@ X-ORIGIN 'OpenDS Directory Server' ) objectClasses: ( 1.3.6.1.4.1.26027.1.2.64 NAME 'ds-task-import' SUP ds-task MUST ( ds-task-import-ldif-file $ ds-task-import-backend-id ) MUST ( ds-task-import-ldif-file ) MAY ( ds-task-import-append $ ds-task-import-replace-existing $ ds-task-import-include-branch $ ds-task-import-exclude-branch $ ds-task-import-include-attribute $ ds-task-import-exclude-attribute $ ds-task-import-include-filter $ ds-task-import-exclude-filter $ ds-task-import-reject-file $ ds-task-import-overwrite-rejects $ ds-task-import-skip-schema-validation $ ds-task-import-is-compressed $ ds-task-import-is-encrypted ) ds-task-import-is-encrypted $ ds-task-import-backend-id $ ds-task-import-clear-backend ) X-ORIGIN 'OpenDS Directory Server' ) objectClasses: ( 1.3.6.1.4.1.26027.1.2.65 NAME ( 'ds-cfg-replication-server-config' opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java
@@ -139,7 +139,8 @@ this.state = state; AttributeType attrType = indexConfig.getIndexAttribute(); String name = attrType.getNameOrOID(); String name = entryContainer.getDatabasePrefix() + "_" + attrType.getNameOrOID(); int indexEntryLimit = indexConfig.getIndexEntryLimit(); if (indexConfig.getIndexType().contains(JEIndexCfgDefn.IndexType.EQUALITY)) @@ -1216,7 +1217,8 @@ try { AttributeType attrType = cfg.getIndexAttribute(); String name = attrType.getNameOrOID(); String name = entryContainer.getDatabasePrefix() + "_" + attrType.getNameOrOID(); int indexEntryLimit = cfg.getIndexEntryLimit(); if (cfg.getIndexType().contains(JEIndexCfgDefn.IndexType.EQUALITY)) @@ -1260,7 +1262,7 @@ entryContainer.exclusiveLock.lock(); try { entryContainer.removeDatabase(equalityIndex); entryContainer.deleteDatabase(equalityIndex); equalityIndex = null; } catch(DatabaseException de) @@ -1314,7 +1316,7 @@ entryContainer.exclusiveLock.lock(); try { entryContainer.removeDatabase(presenceIndex); entryContainer.deleteDatabase(presenceIndex); presenceIndex = null; } catch(DatabaseException de) @@ -1377,7 +1379,7 @@ entryContainer.exclusiveLock.lock(); try { entryContainer.removeDatabase(substringIndex); entryContainer.deleteDatabase(substringIndex); substringIndex = null; } catch(DatabaseException de) @@ -1431,7 +1433,7 @@ entryContainer.exclusiveLock.lock(); try { entryContainer.removeDatabase(orderingIndex); entryContainer.deleteDatabase(orderingIndex); orderingIndex = null; } catch(DatabaseException de) @@ -1485,7 +1487,7 @@ entryContainer.exclusiveLock.lock(); try { entryContainer.removeDatabase(approximateIndex); entryContainer.deleteDatabase(approximateIndex); approximateIndex = null; } catch(DatabaseException de) @@ -1595,7 +1597,7 @@ public String getName() { StringBuilder builder = new StringBuilder(); builder.append(entryContainer.getContainerName()); builder.append(entryContainer.getDatabasePrefix()); builder.append("_"); builder.append(indexConfig.getIndexAttribute().getNameOrOID()); return builder.toString(); opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
@@ -1070,7 +1070,7 @@ envConfig.setConfigParam("je.env.isLocking", "true"); envConfig.setConfigParam("je.env.runCheckpointer", "false"); } else else if(importConfig.clearBackend()) { // We have the writer lock on the environment, now delete the // environment and re-open it. Only do this when we are @@ -1450,7 +1450,9 @@ { // The base DN was deleted. DirectoryServer.deregisterBaseDN(baseDN, false); rootContainer.removeEntryContainer(baseDN); EntryContainer ec = rootContainer.unregisterEntryContainer(baseDN); ec.delete(); } } @@ -1461,7 +1463,9 @@ try { // The base DN was added. rootContainer.openEntryContainer(baseDN); EntryContainer ec = rootContainer.openEntryContainer(baseDN, null); rootContainer.registerEntryContainer(baseDN, ec); DirectoryServer.registerBaseDN(baseDN, this, false, false); } catch (Exception e) opends/src/server/org/opends/server/backends/jeb/DatabaseContainer.java
@@ -92,10 +92,7 @@ this.env = env; this.entryContainer = entryContainer; this.databases = new CopyOnWriteArrayList<Database>(); StringBuilder builder = new StringBuilder(); buildDatabaseName(builder, name); this.name = builder.toString(); this.name = name; } /** @@ -175,19 +172,6 @@ } /** * Constructs a full JE database name incorporating a entryContainer name. * * @param builder A string builder to which the full name will be appended. * @param name The short database name. */ private void buildDatabaseName(StringBuilder builder, String name) { builder.append(entryContainer.getContainerName()); builder.append('_'); builder.append(name); } /** * Flush any cached database information to disk and close the * database container. * @@ -385,4 +369,14 @@ { return getDatabase().preload(config); } /** * Set the JE database name to use for this container. * * @param name The database name to use for this container. */ void setName(String name) { this.name = name; } } opends/src/server/org/opends/server/backends/jeb/EntryContainer.java
@@ -186,6 +186,7 @@ private int indexEntryLimit; private String databasePrefix; /** * A read write lock to handle schema changes and bulk changes. */ @@ -198,6 +199,8 @@ * * @param baseDN The baseDN this entry container will be responsible for * storing on disk. * @param databasePrefix The prefix to use in the database names used by * this entry container. * @param backend A reference to the JE backend that is creating this entry * container. It is needed by the Directory Server entry cache * methods. @@ -206,8 +209,9 @@ * @param rootContainer The root container this entry container is in. * @throws ConfigException if a configuration related error occurs. */ public EntryContainer(DN baseDN, Backend backend, JEBackendCfg config, Environment env, RootContainer rootContainer) public EntryContainer(DN baseDN, String databasePrefix, Backend backend, JEBackendCfg config, Environment env, RootContainer rootContainer) throws ConfigException { this.backend = backend; @@ -215,6 +219,22 @@ this.config = config; this.env = env; this.rootContainer = rootContainer; StringBuilder builder = new StringBuilder(databasePrefix.length()); for (int i = 0; i < databasePrefix.length(); i++) { char ch = databasePrefix.charAt(i); if (Character.isLetterOrDigit(ch)) { builder.append(ch); } else { builder.append('_'); } } this.databasePrefix = builder.toString(); this.deadlockRetryLimit = config.getBackendDeadlockRetryLimit(); this.subtreeDeleteSizeLimit = config.getBackendSubtreeDeleteSizeLimit(); this.subtreeDeleteBatchSize = config.getBackendSubtreeDeleteBatchSize(); @@ -242,28 +262,29 @@ DataConfig entryDataConfig = new DataConfig(); entryDataConfig.setCompressed(config.isBackendEntriesCompressed()); id2entry = new ID2Entry(ID2ENTRY_DATABASE_NAME, entryDataConfig, env, this); id2entry = new ID2Entry(databasePrefix + "_" + ID2ENTRY_DATABASE_NAME, entryDataConfig, env, this); id2entry.open(); dn2id = new DN2ID(DN2ID_DATABASE_NAME, env, this); dn2id = new DN2ID(databasePrefix + "_" + DN2ID_DATABASE_NAME, env, this); dn2id.open(); state = new State(STATE_DATABASE_NAME, env, this); state = new State(databasePrefix + "_" + STATE_DATABASE_NAME, env, this); state.open(); id2children = new Index(ID2CHILDREN_DATABASE_NAME, id2children = new Index(databasePrefix + "_" + ID2CHILDREN_DATABASE_NAME, new ID2CIndexer(), state, indexEntryLimit, 0, env,this); id2children.open(); id2subtree = new Index(ID2SUBTREE_DATABASE_NAME, id2subtree = new Index(databasePrefix + "_" + ID2SUBTREE_DATABASE_NAME, new ID2SIndexer(), state, indexEntryLimit, 0, env, this); id2subtree.open(); dn2uri = new DN2URI(REFERRAL_DATABASE_NAME, env, this); dn2uri = new DN2URI(databasePrefix + "_" + REFERRAL_DATABASE_NAME, env, this); dn2uri.open(); for (String idx : config.listJEIndexes()) @@ -297,86 +318,11 @@ public void close() throws DatabaseException { try List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>(); listDatabases(databases); for(DatabaseContainer db : databases) { dn2id.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { id2entry.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { id2children.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { id2subtree.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { state.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { dn2uri.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } for (AttributeIndex index : attrIndexMap.values()) { try { index.close(); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } db.close(); } config.removeJEChangeListener(this); @@ -3438,86 +3384,6 @@ } /** * Remove the entry entryContainer from disk. The entryContainer must not be * open. * * @throws DatabaseException If an error occurs in the JE database. */ public void removeContainer() throws DatabaseException { try { removeDatabase(dn2id); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { removeDatabase(id2entry); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { removeDatabase(id2children); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { removeDatabase(id2subtree); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } try { removeDatabase(state); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } for (AttributeIndex index : attrIndexMap.values()) { try { removeAttributeIndex(index); } catch (DatabaseNotFoundException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } } attrIndexMap.clear(); } /** * Get the number of values for which the entry limit has been exceeded * since the entry entryContainer was opened. * @return The number of values for which the entry limit has been exceeded. @@ -3545,6 +3411,7 @@ dbList.add(dn2uri); dbList.add(id2children); dbList.add(id2subtree); dbList.add(state); for(AttributeIndex index : attrIndexMap.values()) { @@ -3579,19 +3446,6 @@ } /** * Constructs a full JE database name incorporating a entryContainer name. * * @param builder A string builder to which the full name will be appended. * @param name The short database name. */ private void buildDatabaseName(StringBuilder builder, String name) { builder.append(getContainerName()); builder.append('_'); builder.append(name); } /** * Begin a leaf transaction using the default configuration. * Provides assertion debug logging. * @return A JE transaction handle. @@ -3652,20 +3506,92 @@ } /** * Delete this entry container from disk. The entry container should be * closed before calling this method. * * @throws DatabaseException If an error occurs while removing the entry * container. */ public void delete() throws DatabaseException { List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>(); listDatabases(databases); for(DatabaseContainer db : databases) { db.close(); } if(env.getConfig().getTransactional()) { Transaction txn = beginTransaction(); try { for(DatabaseContainer db : databases) { env.removeDatabase(txn, db.getName()); } transactionCommit(txn); } catch(DatabaseException de) { transactionAbort(txn); throw de; } } else { for(DatabaseContainer db : databases) { env.removeDatabase(null, db.getName()); } } } /** * Remove a database from disk. * * @param database The database container to remove. * @throws DatabaseException If an error occurs while attempting to delete the * database. */ public void removeDatabase(DatabaseContainer database) public void deleteDatabase(DatabaseContainer database) throws DatabaseException { database.close(); env.removeDatabase(null, database.getName()); if(database instanceof Index) if(database == state) { state.removeIndexTrustState(null, (Index)database); // The state database can not be removed individually. return; } database.close(); if(env.getConfig().getTransactional()) { Transaction txn = beginTransaction(); try { env.removeDatabase(txn, database.getName()); if(database instanceof Index) { state.removeIndexTrustState(txn, (Index)database); } transactionCommit(txn); } catch(DatabaseException de) { transactionAbort(txn); throw de; } } else { env.removeDatabase(null, database.getName()); if(database instanceof Index) { state.removeIndexTrustState(null, (Index)database); } } } @@ -3676,34 +3602,75 @@ * @throws DatabaseException If an JE database error occurs while attempting * to delete the index. */ public void removeAttributeIndex(AttributeIndex index) public void deleteAttributeIndex(AttributeIndex index) throws DatabaseException { index.close(); if(index.equalityIndex != null) if(env.getConfig().getTransactional()) { env.removeDatabase(null, index.equalityIndex.getName()); state.removeIndexTrustState(null, index.equalityIndex); Transaction txn = beginTransaction(); try { if(index.equalityIndex != null) { env.removeDatabase(txn, index.equalityIndex.getName()); state.removeIndexTrustState(txn, index.equalityIndex); } if(index.presenceIndex != null) { env.removeDatabase(txn, index.presenceIndex.getName()); state.removeIndexTrustState(txn, index.presenceIndex); } if(index.substringIndex != null) { env.removeDatabase(txn, index.substringIndex.getName()); state.removeIndexTrustState(txn, index.substringIndex); } if(index.orderingIndex != null) { env.removeDatabase(txn, index.orderingIndex.getName()); state.removeIndexTrustState(txn, index.orderingIndex); } if(index.approximateIndex != null) { env.removeDatabase(txn, index.approximateIndex.getName()); state.removeIndexTrustState(txn, index.approximateIndex); } transactionCommit(txn); } catch(DatabaseException de) { transactionAbort(txn); throw de; } } if(index.presenceIndex != null) else { env.removeDatabase(null, index.presenceIndex.getName()); state.removeIndexTrustState(null, index.presenceIndex); } if(index.substringIndex != null) { env.removeDatabase(null, index.substringIndex.getName()); state.removeIndexTrustState(null, index.substringIndex); } if(index.orderingIndex != null) { env.removeDatabase(null, index.orderingIndex.getName()); state.removeIndexTrustState(null, index.orderingIndex); } if(index.approximateIndex != null) { env.removeDatabase(null, index.approximateIndex.getName()); state.removeIndexTrustState(null, index.approximateIndex); if(index.equalityIndex != null) { env.removeDatabase(null, index.equalityIndex.getName()); state.removeIndexTrustState(null, index.equalityIndex); } if(index.presenceIndex != null) { env.removeDatabase(null, index.presenceIndex.getName()); state.removeIndexTrustState(null, index.presenceIndex); } if(index.substringIndex != null) { env.removeDatabase(null, index.substringIndex.getName()); state.removeIndexTrustState(null, index.substringIndex); } if(index.orderingIndex != null) { env.removeDatabase(null, index.orderingIndex.getName()); state.removeIndexTrustState(null, index.orderingIndex); } if(index.approximateIndex != null) { env.removeDatabase(null, index.approximateIndex.getName()); state.removeIndexTrustState(null, index.approximateIndex); } } } @@ -3714,13 +3681,30 @@ * * @return The container name for the base DN. */ public String getContainerName() public String getDatabasePrefix() { String normStr = baseDN.toNormalizedString(); StringBuilder builder = new StringBuilder(normStr.length()); for (int i = 0; i < normStr.length(); i++) return databasePrefix; } /** * Sets a new database prefix for this entry container and rename all * existing databases in use by this entry container. * * @param newDatabasePrefix The new database prefix to use. * @throws DatabaseException If an error occurs in the JE database. * @throws JebException If an error occurs in the JE backend. */ public void setDatabasePrefix(String newDatabasePrefix) throws DatabaseException, JebException { List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>(); listDatabases(databases); StringBuilder builder = new StringBuilder(newDatabasePrefix.length()); for (int i = 0; i < newDatabasePrefix.length(); i++) { char ch = normStr.charAt(i); char ch = newDatabasePrefix.charAt(i); if (Character.isLetterOrDigit(ch)) { builder.append(ch); @@ -3730,9 +3714,75 @@ builder.append('_'); } } return builder.toString(); newDatabasePrefix = builder.toString(); // close the containers. for(DatabaseContainer db : databases) { db.close(); } try { if(env.getConfig().getTransactional()) { //Rename under transaction Transaction txn = beginTransaction(); try { for(DatabaseContainer db : databases) { String oldName = db.getName(); String newName = oldName.replace(databasePrefix, newDatabasePrefix); env.renameDatabase(txn, oldName, newName); } transactionCommit(txn); for(DatabaseContainer db : databases) { String oldName = db.getName(); String newName = oldName.replace(databasePrefix, newDatabasePrefix); db.setName(newName); } // Update the prefix. this.databasePrefix = newDatabasePrefix; } catch(Exception e) { transactionAbort(txn); int messageID = MSGID_JEB_UNCHECKED_EXCEPTION; String message = getMessage(messageID); throw new JebException(messageID, message, e); } } else { for(DatabaseContainer db : databases) { String oldName = db.getName(); String newName = oldName.replace(databasePrefix, newDatabasePrefix); env.renameDatabase(null, oldName, newName); db.setName(newName); } // Update the prefix. this.databasePrefix = newDatabasePrefix; } } finally { // Open the containers backup. for(DatabaseContainer db : databases) { db.open(); } } } /** * Get the baseDN this entry container is responsible for. * @@ -3876,7 +3926,7 @@ try { AttributeIndex index = attrIndexMap.get(cfg.getIndexAttribute()); removeAttributeIndex(index); deleteAttributeIndex(index); attrIndexMap.remove(cfg.getIndexAttribute()); } catch(DatabaseException de) @@ -3911,19 +3961,99 @@ } /** * Clear the contents of this entry container. * * @return The number of records deleted. * @throws DatabaseException If an error occurs while removing the entry * container. */ public long clear() throws DatabaseException { List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>(); listDatabases(databases); long count = 0; for(DatabaseContainer db : databases) { db.close(); } try { if(env.getConfig().getTransactional()) { Transaction txn = beginTransaction(); try { for(DatabaseContainer db : databases) { count += env.truncateDatabase(txn, db.getName(), true); } transactionCommit(txn); } catch(DatabaseException de) { transactionAbort(txn); throw de; } } else { for(DatabaseContainer db : databases) { count += env.truncateDatabase(null, db.getName(), true); } } } finally { for(DatabaseContainer db : databases) { db.open(); } } return count; } /** * Clear the contents for a database from disk. * * @param txn A transaction object or null if its not required. * @param database The database to clear. * @return The number of records deleted. * @throws DatabaseException if a JE database error occurs. */ public long clearDatabase(Transaction txn, DatabaseContainer database) public long clearDatabase(DatabaseContainer database) throws DatabaseException { long count = 0; database.close(); long count = env.truncateDatabase(txn, database.getName(), true); database.open(); try { if(env.getConfig().getTransactional()) { Transaction txn = beginTransaction(); try { count = env.truncateDatabase(txn, database.getName(), true); transactionCommit(txn); } catch(DatabaseException de) { transactionAbort(txn); throw de; } } else { count = env.truncateDatabase(null, database.getName(), true); } } finally { database.open(); } if(debugEnabled()) { TRACER.debugVerbose("Cleared %d existing records from the " + @@ -3935,39 +4065,89 @@ /** * Clear the contents for a attribute index from disk. * * @param txn A transaction object or null if its not required. * @param index The attribute index to clear. * @return The number of records deleted. * @throws DatabaseException if a JE database error occurs. */ public long clearAttributeIndex(Transaction txn, AttributeIndex index) public long clearAttributeIndex(AttributeIndex index) throws DatabaseException { long count = 0; index.close(); if(index.equalityIndex != null) try { count += env.truncateDatabase(txn, index.equalityIndex.getName(), true); if(env.getConfig().getTransactional()) { Transaction txn = beginTransaction(); try { if(index.equalityIndex != null) { count += env.truncateDatabase(txn, index.equalityIndex.getName(), true); } if(index.presenceIndex != null) { count += env.truncateDatabase(txn, index.presenceIndex.getName(), true); } if(index.substringIndex != null) { count += env.truncateDatabase(txn, index.substringIndex.getName(), true); } if(index.orderingIndex != null) { count += env.truncateDatabase(txn, index.orderingIndex.getName(), true); } if(index.approximateIndex != null) { count += env.truncateDatabase(txn, index.approximateIndex.getName(), true); } transactionCommit(txn); } catch(DatabaseException de) { transactionAbort(txn); throw de; } } else { if(index.equalityIndex != null) { count += env.truncateDatabase(null, index.equalityIndex.getName(), true); } if(index.presenceIndex != null) { count += env.truncateDatabase(null, index.presenceIndex.getName(), true); } if(index.substringIndex != null) { count += env.truncateDatabase(null, index.substringIndex.getName(), true); } if(index.orderingIndex != null) { count += env.truncateDatabase(null, index.orderingIndex.getName(), true); } if(index.approximateIndex != null) { count += env.truncateDatabase(null, index.approximateIndex.getName(), true); } } } if(index.presenceIndex != null) finally { count += env.truncateDatabase(txn, index.presenceIndex.getName(), true); index.open(); } if(index.substringIndex != null) { count += env.truncateDatabase(txn, index.substringIndex.getName(), true); } if(index.orderingIndex != null) { count += env.truncateDatabase(txn, index.orderingIndex.getName(), true); } if(index.approximateIndex != null) { count += env.truncateDatabase(txn, index.approximateIndex.getName(), true); } index.open(); if(debugEnabled()) { TRACER.debugVerbose("Cleared %d existing records from the " + opends/src/server/org/opends/server/backends/jeb/ImportContext.java
@@ -35,16 +35,13 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.ArrayList; import java.util.List; /** * This class represents the import context for a destination base DN. */ public class ImportContext { /** * The name of the entryContainer for the destination base DN. */ private String containerName; /** * The destination base DN. @@ -52,6 +49,16 @@ private DN baseDN; /** * The include branches below the base DN. */ private List<DN> includeBranches; /** * The exclude branches below the base DN. */ private List<DN> excludeBranches; /** * The configuration of the destination backend. */ private JEBackendCfg config; @@ -72,6 +79,11 @@ private EntryContainer entryContainer; /** * The source entryContainer if this is a partial import of a base DN. */ private EntryContainer srcEntryContainer; /** * The amount of buffer memory available in bytes. */ private long bufferSize; @@ -121,24 +133,6 @@ } /** * Set the name of the entryContainer for the destination base DN. * @param containerName The entryContainer name. */ public void setContainerName(String containerName) { this.containerName = containerName; } /** * Get the name of the entryContainer for the destination base DN. * @return The entryContainer name. */ public String getContainerName() { return containerName; } /** * Set the destination base DN. * @param baseDN The destination base DN. */ @@ -229,6 +223,25 @@ } /** * Set the source entry entryContainer for the destination base DN. * @param srcEntryContainer The entry source entryContainer for the * destination base DN. */ public void setSrcEntryContainer(EntryContainer srcEntryContainer) { this.srcEntryContainer = srcEntryContainer; } /** * Get the source entry entryContainer for the destination base DN. * @return The source entry entryContainer for the destination base DN. */ public EntryContainer getSrcEntryContainer() { return srcEntryContainer; } /** * Get the available buffer size in bytes. * @return The available buffer size. */ @@ -300,4 +313,75 @@ { this.IDs = IDs; } /** * Retrieves the set of base DNs that specify the set of entries to * exclude from the import. The contents of the returned list may * be altered by the caller. * * @return The set of base DNs that specify the set of entries to * exclude from the import. */ public List<DN> getExcludeBranches() { return excludeBranches; } /** * Specifies the set of base DNs that specify the set of entries to * exclude from the import. * * @param excludeBranches The set of base DNs that specify the set * of entries to exclude from the import. */ public void setExcludeBranches(List<DN> excludeBranches) { if (excludeBranches == null) { this.excludeBranches = new ArrayList<DN>(0); } else { this.excludeBranches = excludeBranches; } } /** * Retrieves the set of base DNs that specify the set of entries to * include in the import. The contents of the returned list may be * altered by the caller. * * @return The set of base DNs that specify the set of entries to * include in the import. */ public List<DN> getIncludeBranches() { return includeBranches; } /** * Specifies the set of base DNs that specify the set of entries to * include in the import. * * @param includeBranches The set of base DNs that specify the set * of entries to include in the import. */ public void setIncludeBranches(List<DN> includeBranches) { if (includeBranches == null) { this.includeBranches = new ArrayList<DN>(0); } else { this.includeBranches = includeBranches; } } } opends/src/server/org/opends/server/backends/jeb/ImportJob.java
@@ -26,10 +26,7 @@ */ package org.opends.server.backends.jeb; import com.sleepycat.je.DatabaseException; import com.sleepycat.je.EnvironmentStats; import com.sleepycat.je.StatsConfig; import com.sleepycat.je.Transaction; import com.sleepycat.je.*; import org.opends.server.types.DebugLogLevel; import org.opends.server.messages.JebMessages; @@ -48,10 +45,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Timer; import java.util.TimerTask; import java.util.*; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; @@ -66,6 +60,8 @@ import org.opends.server.loggers.debug.DebugTracer; import static org.opends.server.messages.JebMessages.*; import org.opends.server.admin.std.server.JEBackendCfg; import org.opends.server.protocols.asn1.ASN1OctetString; import org.opends.server.config.ConfigException; /** * Import from LDIF to a JE backend. @@ -108,12 +104,30 @@ */ private int importedCount; /** * The number of entries migrated. */ private int migratedCount; /** * The number of merge passes. */ int mergePassNumber = 1; /** * The number of milliseconds between job progress reports. */ private long progressInterval = 10000; /** * The progress report timer. */ private Timer timer; private int entriesProcessed; private int importPassSize; /** * The import worker threads. @@ -146,15 +160,25 @@ * @throws IOException If a problem occurs while opening the LDIF file for * reading, or while reading from the LDIF file. * @throws JebException If an error occurs in the JE backend. * @throws DirectoryException if a directory server related error occurs. * @throws ConfigException if a configuration related error occurs. */ public LDIFImportResult importLDIF(RootContainer rootContainer) throws DatabaseException, IOException, JebException throws DatabaseException, IOException, JebException, DirectoryException, ConfigException { // Create an LDIF reader. Throws an exception if the file does not exist. reader = new LDIFReader(ldifImportConfig); this.rootContainer = rootContainer; this.config = rootContainer.getConfiguration(); this.mergePassNumber = 1; this.entriesProcessed = 0; this.importPassSize = config.getBackendImportPassSize(); if (importPassSize <= 0) { importPassSize = Integer.MAX_VALUE; } int msgID; String message; @@ -185,31 +209,15 @@ TRACER.debugInfo(message); } // Create the import contexts for each base DN. DN baseDN; for (EntryContainer entryContainer : rootContainer.getEntryContainers()) { baseDN = entryContainer.getBaseDN(); ImportContext importContext = getImportContext(entryContainer, bufferSize); // Create an import context. ImportContext importContext = new ImportContext(); importContext.setBufferSize(bufferSize); importContext.setConfig(config); importContext.setLDIFImportConfig(this.ldifImportConfig); importContext.setLDIFReader(reader); importContext.setBaseDN(baseDN); importContext.setContainerName(entryContainer.getContainerName()); importContext.setEntryContainer(entryContainer); importContext.setBufferSize(bufferSize); // Create an entry queue. LinkedBlockingQueue<Entry> queue = new LinkedBlockingQueue<Entry>(config.getBackendImportQueueSize()); importContext.setQueue(queue); importMap.put(baseDN, importContext); if(importContext != null) { importMap.put(entryContainer.getBaseDN(), importContext); } } // Make a note of the time we started. @@ -232,53 +240,57 @@ } } startWorkerThreads(); try { importedCount = 0; int passNumber = 1; boolean moreData = true; while (moreData) { moreData = processLDIF(); if (moreData) { msgID = MSGID_JEB_IMPORT_BEGINNING_INTERMEDIATE_MERGE; message = getMessage(msgID, passNumber++); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } else { msgID = MSGID_JEB_IMPORT_BEGINNING_FINAL_MERGE; message = getMessage(msgID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } long mergeStartTime = System.currentTimeMillis(); merge(); long mergeEndTime = System.currentTimeMillis(); if (moreData) { msgID = MSGID_JEB_IMPORT_RESUMING_LDIF_PROCESSING; message = getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } else { msgID = MSGID_JEB_IMPORT_FINAL_MERGE_COMPLETED; message = getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } } migratedCount = 0; migrateExistingEntries(); processLDIF(); migrateExcludedEntries(); } finally { merge(false); tempDir.delete(); for(ImportContext importContext : importMap.values()) { DN baseDN = importContext.getBaseDN(); EntryContainer srcEntryContainer = importContext.getSrcEntryContainer(); if(srcEntryContainer != null) { if (debugEnabled()) { TRACER.debugInfo("Deleteing old entry container for base DN " + "%s and renaming temp entry container", baseDN); } EntryContainer unregEC = rootContainer.unregisterEntryContainer(baseDN); //Make sure the unregistered EC for the base DN is the same as //the one in the import context. if(unregEC != srcEntryContainer) { if(debugEnabled()) { TRACER.debugInfo("Current entry container used for base DN " + "%s is not the same as the source entry container used " + "during the migration process.", baseDN); } rootContainer.registerEntryContainer(baseDN, unregEC); continue; } srcEntryContainer.exclusiveLock.lock(); srcEntryContainer.delete(); srcEntryContainer.exclusiveLock.unlock(); EntryContainer newEC = importContext.getEntryContainer(); newEC.exclusiveLock.lock(); newEC.setDatabasePrefix(baseDN.toNormalizedString()); newEC.exclusiveLock.unlock(); rootContainer.registerEntryContainer(baseDN, newEC); } } } } finally @@ -296,9 +308,11 @@ } msgID = MSGID_JEB_IMPORT_FINAL_STATUS; message = getMessage(msgID, reader.getEntriesRead(), importedCount, message = getMessage(msgID, reader.getEntriesRead(), importedCount - migratedCount, reader.getEntriesIgnored(), reader.getEntriesRejected(), importTime/1000, rate); reader.getEntriesRejected(), migratedCount, importTime/1000, rate); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); @@ -314,135 +328,169 @@ /** * Merge the intermediate files to load the index databases. * * @param moreData <CODE>true</CODE> if this is a intermediate merge or * <CODE>false</CODE> if this is a final merge. * @throws DatabaseException If an error occurs in the JE database. */ public void merge() private void merge(boolean moreData) throws DatabaseException { ArrayList<IndexMergeThread> mergers = new ArrayList<IndexMergeThread>(); stopWorkerThreads(); // Create merge threads for each base DN. for (ImportContext importContext : importMap.values()) try { EntryContainer entryContainer = importContext.getEntryContainer(); // For each configured attribute index. for (AttributeIndex attrIndex : entryContainer.getAttributeIndexes()) if (moreData) { int indexEntryLimit = config.getBackendIndexEntryLimit(); if(attrIndex.getConfiguration().getIndexEntryLimit() != null) int msgID = MSGID_JEB_IMPORT_BEGINNING_INTERMEDIATE_MERGE; String message = getMessage(msgID, mergePassNumber++); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } else { int msgID = MSGID_JEB_IMPORT_BEGINNING_FINAL_MERGE; String message = getMessage(msgID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } long mergeStartTime = System.currentTimeMillis(); ArrayList<IndexMergeThread> mergers = new ArrayList<IndexMergeThread>(); // Create merge threads for each base DN. for (ImportContext importContext : importMap.values()) { EntryContainer entryContainer = importContext.getEntryContainer(); // For each configured attribute index. for (AttributeIndex attrIndex : entryContainer.getAttributeIndexes()) { indexEntryLimit = attrIndex.getConfiguration().getIndexEntryLimit(); int indexEntryLimit = config.getBackendIndexEntryLimit(); if(attrIndex.getConfiguration().getIndexEntryLimit() != null) { indexEntryLimit = attrIndex.getConfiguration().getIndexEntryLimit(); } if (attrIndex.equalityIndex != null) { Index index = attrIndex.equalityIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.presenceIndex != null) { Index index = attrIndex.presenceIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.substringIndex != null) { Index index = attrIndex.substringIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.orderingIndex != null) { Index index = attrIndex.orderingIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.approximateIndex != null) { Index index = attrIndex.approximateIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } } if (attrIndex.equalityIndex != null) // Id2Children index. Index id2Children = entryContainer.getID2Children(); IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, id2Children, config.getBackendIndexEntryLimit()); mergers.add(indexMergeThread); // Id2Subtree index. Index id2Subtree = entryContainer.getID2Subtree(); indexMergeThread = new IndexMergeThread(config, ldifImportConfig, id2Subtree, config.getBackendIndexEntryLimit()); mergers.add(indexMergeThread); } // Run all the merge threads. for (IndexMergeThread imt : mergers) { imt.start(); } // Wait for the threads to finish. for (IndexMergeThread imt : mergers) { try { Index index = attrIndex.equalityIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); imt.join(); } if (attrIndex.presenceIndex != null) catch (InterruptedException e) { Index index = attrIndex.presenceIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.substringIndex != null) { Index index = attrIndex.substringIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.orderingIndex != null) { Index index = attrIndex.orderingIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); } if (attrIndex.approximateIndex != null) { Index index = attrIndex.approximateIndex; IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, index, indexEntryLimit); mergers.add(indexMergeThread); if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } } // Id2Children index. Index id2Children = entryContainer.getID2Children(); IndexMergeThread indexMergeThread = new IndexMergeThread(config, ldifImportConfig, id2Children, config.getBackendIndexEntryLimit()); mergers.add(indexMergeThread); long mergeEndTime = System.currentTimeMillis(); // Id2Subtree index. Index id2Subtree = entryContainer.getID2Subtree(); indexMergeThread = new IndexMergeThread(config, ldifImportConfig, id2Subtree, config.getBackendIndexEntryLimit()); mergers.add(indexMergeThread); } // Run all the merge threads. for (IndexMergeThread imt : mergers) { imt.start(); } // Wait for the threads to finish. for (IndexMergeThread imt : mergers) { try if (moreData) { imt.join(); int msgID = MSGID_JEB_IMPORT_RESUMING_LDIF_PROCESSING; String message = getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } catch (InterruptedException e) else { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } int msgID = MSGID_JEB_IMPORT_FINAL_MERGE_COMPLETED; String message = getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); } } finally { if(moreData) { startWorkerThreads(); } } } /** * Create a set of worker threads, one set for each base DN. * Read each entry from the LDIF and determine which * base DN the entry belongs to. Write the dn2id database, then put the * entry on the appropriate queue for the worker threads to consume. * Record the entry count for each base DN when all entries have been * processed. * * @return true if thre is more data to be read from the LDIF file (the import * pass size was reached), false if the entire LDIF file has been read. * * @throws JebException If an error occurs in the JE backend. * @throws DatabaseException If an error occurs in the JE database. * @throws IOException If a problem occurs while opening the LDIF file for * reading, or while reading from the LDIF file. */ private boolean processLDIF() throws JebException, DatabaseException, IOException private void startWorkerThreads() throws DatabaseException { boolean moreData = false; // Create one set of worker threads for each base DN. int importThreadCount = config.getBackendImportThreadCount(); for (ImportContext ic : importMap.values()) @@ -457,122 +505,291 @@ } } try // Start a timer for the progress report. timer = new Timer(); TimerTask progressTask = new ImportJob.ProgressTask(); timer.scheduleAtFixedRate(progressTask, progressInterval, progressInterval); } private void stopWorkerThreads() { if(threads.size() > 0) { // Create a counter to use to determine whether we've hit the import // pass size. int entriesProcessed = 0; int importPassSize = config.getBackendImportPassSize(); if (importPassSize <= 0) // Wait for the queues to be drained. for (ImportContext ic : importMap.values()) { importPassSize = Integer.MAX_VALUE; } // Start a timer for the progress report. Timer timer = new Timer(); TimerTask progressTask = new ImportJob.ProgressTask(); timer.scheduleAtFixedRate(progressTask, progressInterval, progressInterval); try { do while (ic.getQueue().size() > 0) { if(threads.size() <= 0) { int msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS; String msg = getMessage(msgID); throw new JebException(msgID, msg); } try { // Read the next entry. Entry entry = reader.readEntry(); // Check for end of file. if (entry == null) { break; } // Route it according to base DN. ImportContext importContext = getImportConfig(entry.getDN()); processEntry(importContext, entry); entriesProcessed++; if (entriesProcessed >= importPassSize) { moreData = true; break; } } catch (LDIFException e) Thread.sleep(100); } catch (Exception e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } // No action needed. } catch (DirectoryException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } } while (true); } } } if(threads.size() > 0) // Order the threads to stop. for (ImportThread t : threads) { t.stopProcessing(); } // Wait for each thread to stop. for (ImportThread t : threads) { try { t.join(); importedCount += t.getImportedCount(); } catch (InterruptedException ie) { // No action needed? } } timer.cancel(); } /** * Create a set of worker threads, one set for each base DN. * Read each entry from the LDIF and determine which * base DN the entry belongs to. Write the dn2id database, then put the * entry on the appropriate queue for the worker threads to consume. * Record the entry count for each base DN when all entries have been * processed. * * pass size was reached), false if the entire LDIF file has been read. * * @throws JebException If an error occurs in the JE backend. * @throws DatabaseException If an error occurs in the JE database. * @throws IOException If a problem occurs while opening the LDIF file for * reading, or while reading from the LDIF file. */ private void processLDIF() throws JebException, DatabaseException, IOException { int msgID = MSGID_JEB_IMPORT_LDIF_START; String message = getMessage(msgID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); do { if(threads.size() <= 0) { msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS; message = getMessage(msgID); throw new JebException(msgID, message); } try { // Read the next entry. Entry entry = reader.readEntry(); // Check for end of file. if (entry == null) { // Wait for the queues to be drained. for (ImportContext ic : importMap.values()) msgID = MSGID_JEB_IMPORT_LDIF_END; message = getMessage(msgID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); break; } // Route it according to base DN. ImportContext importContext = getImportConfig(entry.getDN()); processEntry(importContext, entry); entriesProcessed++; if (entriesProcessed >= importPassSize) { merge(false); entriesProcessed = 0; } } catch (LDIFException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } catch (DirectoryException e) { if (debugEnabled()) { TRACER.debugCaught(DebugLogLevel.ERROR, e); } } } while (true); } private void migrateExistingEntries() throws JebException, DatabaseException, DirectoryException { for(ImportContext importContext : importMap.values()) { EntryContainer srcEntryContainer = importContext.getSrcEntryContainer(); if(srcEntryContainer != null && !importContext.getIncludeBranches().isEmpty()) { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); LockMode lockMode = LockMode.DEFAULT; OperationStatus status; int msgID = MSGID_JEB_IMPORT_MIGRATION_START; String message = getMessage(msgID, "existing", importContext.getBaseDN()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); Cursor cursor = srcEntryContainer.getDN2ID().openCursor(null, CursorConfig.READ_COMMITTED); try { status = cursor.getFirst(key, data, lockMode); while(status == OperationStatus.SUCCESS) { while (ic.getQueue().size() > 0) if(threads.size() <= 0) { try msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS; message = getMessage(msgID); throw new JebException(msgID, message); } DN dn = DN.decode(new ASN1OctetString(key.getData())); if(!importContext.getIncludeBranches().contains(dn)) { EntryID id = new EntryID(data); Entry entry = srcEntryContainer.getID2Entry().get(null, id); processEntry(importContext, entry); entriesProcessed++; migratedCount++; if (entriesProcessed >= importPassSize) { Thread.sleep(100); } catch (Exception e) merge(true); entriesProcessed = 0; } status = cursor.getNext(key, data, lockMode); } else { // This is the base entry for a branch that will be included // in the import so we don't want to copy the branch to the new // entry container. /** * Advance the cursor to next entry at the same level in the DIT * skipping all the entries in this branch. * Set the next starting value to a value of equal length but * slightly greater than the previous DN. Since keys are compared * in reverse order we must set the first byte (the comma). * No possibility of overflow here. */ byte[] begin = StaticUtils.getBytes("," + dn.toNormalizedString()); begin[0] = (byte) (begin[0] + 1); key.setData(begin); status = cursor.getSearchKeyRange(key, data, lockMode); } } } finally { cursor.close(); } } } } private void migrateExcludedEntries() throws JebException, DatabaseException { for(ImportContext importContext : importMap.values()) { EntryContainer srcEntryContainer = importContext.getSrcEntryContainer(); if(srcEntryContainer != null && !importContext.getExcludeBranches().isEmpty()) { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry data = new DatabaseEntry(); LockMode lockMode = LockMode.DEFAULT; OperationStatus status; int msgID = MSGID_JEB_IMPORT_MIGRATION_START; String message = getMessage(msgID, "excluded", importContext.getBaseDN()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID); Cursor cursor = srcEntryContainer.getDN2ID().openCursor(null, CursorConfig.READ_COMMITTED); Comparator<byte[]> dn2idComparator = srcEntryContainer.getDN2ID().getComparator(); try { for(DN excludedDN : importContext.getExcludeBranches()) { byte[] suffix = StaticUtils.getBytes(excludedDN.toNormalizedString()); key.setData(suffix); status = cursor.getSearchKeyRange(key, data, lockMode); if(status == OperationStatus.SUCCESS && Arrays.equals(key.getData(), suffix)) { // This is the base entry for a branch that was excluded in the // import so we must migrate all entries in this branch over to // the new entry container. byte[] end = StaticUtils.getBytes("," + excludedDN.toNormalizedString()); end[0] = (byte) (end[0] + 1); while(status == OperationStatus.SUCCESS && dn2idComparator.compare(key.getData(), end) < 0) { // No action needed. if(threads.size() <= 0) { msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS; message = getMessage(msgID); throw new JebException(msgID, message); } EntryID id = new EntryID(data); Entry entry = srcEntryContainer.getID2Entry().get(null, id); processEntry(importContext, entry); entriesProcessed++; migratedCount++; if (entriesProcessed >= importPassSize) { merge(true); entriesProcessed = 0; } status = cursor.getNext(key, data, lockMode); } } } } } finally { timer.cancel(); } } finally { // Order the threads to stop. for (ImportThread t : threads) { t.stopProcessing(); } // Wait for each thread to stop. for (ImportThread t : threads) { try finally { t.join(); importedCount += t.getImportedCount(); } catch (InterruptedException ie) { // No action needed? cursor.close(); } } } return moreData; } /** @@ -799,6 +1016,133 @@ return importContext; } private ImportContext getImportContext(EntryContainer entryContainer, long bufferSize) throws DatabaseException, JebException, ConfigException { DN baseDN = entryContainer.getBaseDN(); EntryContainer srcEntryContainer = null; List<DN> includeBranches = new ArrayList<DN>(); List<DN> excludeBranches = new ArrayList<DN>(); if(!ldifImportConfig.appendToExistingData() && !ldifImportConfig.clearBackend()) { for(DN dn : ldifImportConfig.getExcludeBranches()) { if(baseDN.equals(dn)) { // This entire base DN was explicitly excluded. Skip. return null; } if(baseDN.isAncestorOf(dn)) { excludeBranches.add(dn); } } if(!ldifImportConfig.getIncludeBranches().isEmpty()) { for(DN dn : ldifImportConfig.getIncludeBranches()) { if(baseDN.isAncestorOf(dn)) { includeBranches.add(dn); } } if(includeBranches.isEmpty()) { // There are no branches in the explicitly defined include list under // this base DN. Skip this base DN alltogether. return null; } // Remove any overlapping include branches. for(DN includeDN : includeBranches) { boolean keep = true; for(DN dn : includeBranches) { if(!dn.equals(includeDN) && dn.isAncestorOf(includeDN)) { keep = false; break; } } if(!keep) { includeBranches.remove(includeDN); } } // Remvoe any exclude branches that are not are not under a include // branch since they will be migrated as part of the existing entries // outside of the include branches anyways. for(DN excludeDN : excludeBranches) { boolean keep = false; for(DN includeDN : includeBranches) { if(includeDN.isAncestorOf(excludeDN)) { keep = true; break; } } if(!keep) { excludeBranches.remove(excludeDN); } } if(includeBranches.size() == 1 && excludeBranches.size() == 0 && includeBranches.get(0).equals(baseDN)) { // This entire base DN is explicitly included in the import with // no exclude branches that we need to migrate. Just clear the entry // container. entryContainer.exclusiveLock.lock(); entryContainer.clear(); entryContainer.exclusiveLock.unlock(); } else { // Create a temp entry container srcEntryContainer = entryContainer; entryContainer = rootContainer.openEntryContainer(baseDN, baseDN.toNormalizedString() + "_importTmp"); } } } // Create an import context. ImportContext importContext = new ImportContext(); importContext.setBufferSize(bufferSize); importContext.setConfig(config); importContext.setLDIFImportConfig(this.ldifImportConfig); importContext.setLDIFReader(reader); importContext.setBaseDN(baseDN); importContext.setEntryContainer(entryContainer); importContext.setSrcEntryContainer(srcEntryContainer); importContext.setBufferSize(bufferSize); // Create an entry queue. LinkedBlockingQueue<Entry> queue = new LinkedBlockingQueue<Entry>(config.getBackendImportQueueSize()); importContext.setQueue(queue); // Set the include and exclude branches importContext.setIncludeBranches(includeBranches); importContext.setExcludeBranches(excludeBranches); return importContext; } /** * This class reports progress of the import job at fixed intervals. */ @@ -841,7 +1185,7 @@ */ public void run() { long latestCount = reader.getEntriesRead(); long latestCount = reader.getEntriesRead() + migratedCount; long deltaCount = (latestCount - previousCount); long latestTime = System.currentTimeMillis(); long deltaTime = latestTime - previousTime; @@ -858,7 +1202,7 @@ int msgID = MSGID_JEB_IMPORT_PROGRESS_REPORT; String message = getMessage(msgID, numRead, numIgnored, numRejected, rate); migratedCount, rate); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE, message, msgID);
@@ -196,6 +196,12 @@ String message = getMessage(msgID, index.getName()); TRACER.debugInfo(message); } if(!ldifImportConfig.appendToExistingData()) { index.setTrusted(null, true); } return; } @@ -307,7 +313,7 @@ { } if(replaceExisting) if(!ldifImportConfig.appendToExistingData()) { index.setTrusted(txn, true); } opends/src/server/org/opends/server/backends/jeb/IndexRebuildThread.java
@@ -76,11 +76,6 @@ Index index = null; /** * Name of the indexType being rebuilt. */ String indexName = null; /** * The ID2ENTRY database. */ ID2Entry id2entry = null; @@ -128,12 +123,11 @@ */ IndexRebuildThread(EntryContainer ec, IndexType index) { super("Index Rebuild Thread " + ec.getContainerName() + "_" + super("Index Rebuild Thread " + ec.getDatabasePrefix() + "_" + index.toString()); this.ec = ec; this.indexType = index; this.id2entry = ec.getID2Entry(); this.indexName = ec.getContainerName() + "_" + index.toString(); } /** @@ -149,7 +143,6 @@ this.indexType = IndexType.INDEX; this.index = index; this.id2entry = ec.getID2Entry(); this.indexName = ec.getContainerName() + "_" + index.toString(); } /** @@ -165,7 +158,6 @@ this.indexType = IndexType.ATTRIBUTEINDEX; this.attrIndex = index; this.id2entry = ec.getID2Entry(); this.indexName = ec.getContainerName() + "_" + index.toString(); } /** @@ -213,25 +205,25 @@ switch(indexType) { case DN2ID : ec.clearDatabase(null, ec.getDN2ID()); ec.clearDatabase(ec.getDN2ID()); break; case DN2URI : ec.clearDatabase(null, ec.getDN2URI()); ec.clearDatabase(ec.getDN2URI()); break; case ID2CHILDREN : ec.clearDatabase(null, ec.getID2Children()); ec.clearDatabase(ec.getID2Children()); ec.getID2Children().setRebuildStatus(true); break; case ID2SUBTREE : ec.clearDatabase(null, ec.getID2Subtree()); ec.clearDatabase(ec.getID2Subtree()); ec.getID2Subtree().setRebuildStatus(true); break; case ATTRIBUTEINDEX : ec.clearAttributeIndex(null, attrIndex); ec.clearAttributeIndex(attrIndex); attrIndex.setRebuildStatus(true); break; case INDEX : ec.clearDatabase(null, index); ec.clearDatabase(index); index.setRebuildStatus(true); } } @@ -278,12 +270,6 @@ try { totalEntries = getTotalEntries(); if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s indexType/database", indexName); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } switch(indexType) { @@ -329,6 +315,14 @@ { DN2ID dn2id = ec.getDN2ID(); if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s database", dn2id.getName()); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } //Iterate through the id2entry database and insert associated dn2id //records. Cursor cursor = id2entry.openCursor(null, null); @@ -379,7 +373,7 @@ skippedEntries++; int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED; String message = getMessage(msgID, indexName, String message = getMessage(msgID, dn2id.getName(), stackTraceToSingleLineString(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR, message, msgID); @@ -406,6 +400,14 @@ { DN2URI dn2uri = ec.getDN2URI(); if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s database", dn2uri.getName()); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } //Iterate through the id2entry database and insert associated dn2uri //records. Cursor cursor = id2entry.openCursor(null, null); @@ -457,7 +459,7 @@ skippedEntries++; int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED; String message = getMessage(msgID, indexName, String message = getMessage(msgID, dn2uri.getName(), stackTraceToSingleLineString(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR, message, msgID); @@ -485,6 +487,14 @@ { Index id2children = ec.getID2Children(); if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s index", id2children.getName()); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } DN2ID dn2id = ec.getDN2ID(); DN2URI dn2uri = ec.getDN2URI(); @@ -559,7 +569,7 @@ skippedEntries++; int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED; String message = getMessage(msgID, indexName, String message = getMessage(msgID, id2children.getName(), stackTraceToSingleLineString(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR, message, msgID); @@ -589,6 +599,14 @@ { Index id2subtree = ec.getID2Subtree(); if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s index", id2subtree.getName()); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } DN2ID dn2id = ec.getDN2ID(); DN2URI dn2uri = ec.getDN2URI(); @@ -693,7 +711,7 @@ skippedEntries++; int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED; String message = getMessage(msgID, indexName, String message = getMessage(msgID, id2subtree.getName(), stackTraceToSingleLineString(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR, message, msgID); @@ -722,6 +740,12 @@ private void rebuildAttributeIndex(AttributeIndex index) throws DatabaseException { if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s index", index.getName()); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } //Iterate through the id2entry database and insert associated indexType //records. @@ -770,7 +794,7 @@ skippedEntries++; int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED; String message = getMessage(msgID, indexName, String message = getMessage(msgID, index.getName(), stackTraceToSingleLineString(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR, message, msgID); @@ -799,6 +823,12 @@ private void rebuildAttributeIndex(Index index) throws DatabaseException { if(debugEnabled()) { TRACER.debugInfo("Initiating rebuild of the %s attribute index", index.getName()); TRACER.debugVerbose("%d entries will be rebuilt", totalEntries); } //Iterate through the id2entry database and insert associated indexType //records. @@ -847,7 +877,7 @@ skippedEntries++; int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED; String message = getMessage(msgID, indexName, String message = getMessage(msgID, index.getName(), stackTraceToSingleLineString(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR, message, msgID); opends/src/server/org/opends/server/backends/jeb/RootContainer.java
@@ -223,7 +223,7 @@ TRACER.debugInfo("Free memory in heap: %d bytes", heapFreeSize); } openEntryContainers(config.getBackendBaseDN()); openAndRegisterEntryContainers(config.getBackendBaseDN()); } /** @@ -235,28 +235,55 @@ * transactional. * * @param baseDN The base DN of the entry container to open. * @param name The name of the entry container or <CODE>NULL</CODE> to open * the default entry container for the given base DN. * @return The opened entry container. * @throws DatabaseException If an error occurs while opening the entry * container. * @throws ConfigException If an configuration error occurs while opening * the entry container. */ public EntryContainer openEntryContainer(DN baseDN) public EntryContainer openEntryContainer(DN baseDN, String name) throws DatabaseException, ConfigException { EntryContainer ec = new EntryContainer(baseDN, backend, config, env, this); String databasePrefix; if(name == null || name.equals("")) { databasePrefix = baseDN.toNormalizedString(); } else { databasePrefix = name; } EntryContainer ec = new EntryContainer(baseDN, databasePrefix, backend, config, env, this); ec.open(); return ec; } /** * Registeres the entry container for a base DN. * * @param baseDN The base DN of the entry container to close. * @param entryContainer The entry container to register for the baseDN. * @throws DatabaseException If an error occurs while opening the entry * container. */ public void registerEntryContainer(DN baseDN, EntryContainer entryContainer) throws DatabaseException { EntryContainer ec1=this.entryContainers.get(baseDN); //If an entry container for this baseDN is already open we don't allow //another to be opened. if (ec1 != null) throw new DatabaseException("Entry container for baseDN " + baseDN.toString() + " already is open."); throw new DatabaseException("An entry container named " + ec1.getDatabasePrefix() + " is alreadly registered for base DN " + baseDN.toString()); ec.open(); this.entryContainers.put(baseDN, ec); return ec; this.entryContainers.put(baseDN, entryContainer); } /** @@ -268,15 +295,16 @@ * @throws ConfigException if a configuration error occurs while opening the * container. */ private void openEntryContainers(Set<DN> baseDNs) private void openAndRegisterEntryContainers(Set<DN> baseDNs) throws DatabaseException, ConfigException { EntryID id; EntryID highestID = null; for(DN baseDN : baseDNs) { EntryContainer ec = openEntryContainer(baseDN); EntryContainer ec = openEntryContainer(baseDN, null); id = ec.getHighestEntryID(); registerEntryContainer(baseDN, ec); if(highestID == null || id.compareTo(highestID) > 0) { highestID = id; @@ -287,48 +315,15 @@ } /** * Close the entry container for a base DN. * Unregisteres the entry container for a base DN. * * @param baseDN The base DN of the entry container to close. * @throws DatabaseException If an error occurs while closing the entry * container. * @return The entry container that was unregistered or NULL if a entry * container for the base DN was not registered. */ public void closeEntryContainer(DN baseDN) throws DatabaseException public EntryContainer unregisterEntryContainer(DN baseDN) { EntryContainer ec = getEntryContainer(baseDN); ec.exclusiveLock.lock(); try { ec.close(); entryContainers.remove(baseDN); } finally { ec.exclusiveLock.unlock(); } } /** * Close and remove a entry container for a base DN from disk. * * @param baseDN The base DN of the entry container to remove. * @throws DatabaseException If an error occurs while removing the entry * container. */ public void removeEntryContainer(DN baseDN) throws DatabaseException { EntryContainer ec = getEntryContainer(baseDN); ec.exclusiveLock.lock(); try { ec.close(); ec.removeContainer(); entryContainers.remove(baseDN); } finally { ec.exclusiveLock.unlock(); } return entryContainers.remove(baseDN); } @@ -499,7 +494,16 @@ { for(DN baseDN : entryContainers.keySet()) { closeEntryContainer(baseDN); EntryContainer ec = unregisterEntryContainer(baseDN); ec.exclusiveLock.lock(); try { ec.close(); } finally { ec.exclusiveLock.unlock(); } } if (env != null) opends/src/server/org/opends/server/backends/jeb/State.java
@@ -117,8 +117,10 @@ public boolean getIndexTrustState(Transaction txn, Index index) throws DatabaseException { String sortName = index.getName().replace(entryContainer.getDatabasePrefix(), ""); DatabaseEntry key = new DatabaseEntry(StaticUtils.getBytes(index.getName())); new DatabaseEntry(StaticUtils.getBytes(sortName)); DatabaseEntry data = new DatabaseEntry(); OperationStatus status; @@ -145,8 +147,10 @@ boolean trusted) throws DatabaseException { String sortName = index.getName().replace(entryContainer.getDatabasePrefix(), ""); DatabaseEntry key = new DatabaseEntry(StaticUtils.getBytes(index.getName())); new DatabaseEntry(StaticUtils.getBytes(sortName)); DatabaseEntry data = new DatabaseEntry(); if(trusted) @@ -162,4 +166,6 @@ } return true; } // TODO: Make sure to update the VLV state access methods to use shortname. } opends/src/server/org/opends/server/config/ConfigConstants.java
@@ -4159,5 +4159,12 @@ */ public static final String ATTR_REBUILD_MAX_THREADS = NAME_PREFIX_TASK + "rebuild-max-threads"; /** * The name of the attribute in an import task definition that specifies * whether the backend should be cleared before the import. */ public static final String ATTR_IMPORT_CLEAR_BACKEND = NAME_PREFIX_TASK + "import-clear-backend"; }
@@ -1215,6 +1215,28 @@ public static final int MSGID_JEB_INVALID_LOGGING_LEVEL = CATEGORY_MASK_JEB | SEVERITY_MASK_SEVERE_ERROR | 156; /** * The message ID of the message indicating the migration stage has started * during an import process. This takes two arguments, which are the type * of entries being migrated and the base DN of the entry container they * are from. */ public static final int MSGID_JEB_IMPORT_MIGRATION_START = CATEGORY_MASK_JEB | SEVERITY_MASK_INFORMATIONAL | 157; /** * The message ID of the message indicating the LDIF reading stage has * started during an import process. */ public static final int MSGID_JEB_IMPORT_LDIF_START = CATEGORY_MASK_JEB | SEVERITY_MASK_INFORMATIONAL | 158; /** * The message ID of the message indicating the LDIF reading stage has * ended during an import process. */ public static final int MSGID_JEB_IMPORT_LDIF_END = CATEGORY_MASK_JEB | SEVERITY_MASK_INFORMATIONAL | 159; /** * Associates a set of generic messages with the message IDs defined in this @@ -1426,7 +1448,7 @@ "Exported %d records and skipped %d " + "(recent rate %.1f/sec)"); registerMessage(MSGID_JEB_IMPORT_THREAD_COUNT, "Starting LDIF import (using %d threads)"); "Starting import (using %d threads)"); registerMessage(MSGID_JEB_IMPORT_BUFFER_SIZE, "Buffer size per thread = %,d"); registerMessage(MSGID_JEB_IMPORT_LDIF_PROCESSING_TIME, @@ -1434,10 +1456,10 @@ registerMessage(MSGID_JEB_IMPORT_INDEX_PROCESSING_TIME, "Index processing took %d seconds"); registerMessage(MSGID_JEB_IMPORT_BEGINNING_INTERMEDIATE_MERGE, "Ending LDIF import pass %d because the pass size has " + "Ending import pass %d because the pass size has " + "been reached. Beginning the intermediate index merge"); registerMessage(MSGID_JEB_IMPORT_BEGINNING_FINAL_MERGE, "End of LDIF reached. Beginning final index merge"); "Beginning final index merge"); registerMessage(MSGID_JEB_IMPORT_RESUMING_LDIF_PROCESSING, "Intermediate index merge processing complete (index " + "processing time %d seconds). Resuming LDIF processing"); @@ -1446,13 +1468,14 @@ registerMessage(MSGID_JEB_IMPORT_CLOSING_DATABASE, "Flushing data to disk"); registerMessage(MSGID_JEB_IMPORT_FINAL_STATUS, "Processed %d entries, imported %d, skipped %d, and " + "rejected %d in %d seconds (average rate %.1f/sec)"); "Processed %d entries, imported %d, skipped %d, " + "rejected %d and migrated %d in %d seconds " + "(average rate %.1f/sec)"); registerMessage(MSGID_JEB_IMPORT_ENTRY_LIMIT_EXCEEDED_COUNT, "Number of index values that exceeded the entry limit: %d"); registerMessage(MSGID_JEB_IMPORT_PROGRESS_REPORT, "Processed %d entries, skipped %d, and rejected %d " + "(recent rate %.1f/sec)"); "Processed %d entries, skipped %d, rejected %d, and " + "migrated %d (recent rate %.1f/sec)"); registerMessage(MSGID_JEB_IMPORT_CACHE_AND_MEMORY_REPORT, "Free memory = %d MB, Cache miss rate = %.1f/entry"); registerMessage(MSGID_JEB_INDEX_MERGE_NO_DATA, @@ -1575,5 +1598,11 @@ "imported entries"); registerMessage(MSGID_JEB_IMPORT_CREATE_TMPDIR_ERROR, "Unable to create the temporary directory %s"); registerMessage(MSGID_JEB_IMPORT_MIGRATION_START, "Migrating %s entries for base DN %s"); registerMessage(MSGID_JEB_IMPORT_LDIF_START, "Processing LDIF"); registerMessage(MSGID_JEB_IMPORT_LDIF_END, "End of LDIF reached"); } } opends/src/server/org/opends/server/messages/ToolMessages.java
@@ -9240,6 +9240,7 @@ public static final int MSGID_DSCFG_ERROR_MISSING_NON_INTERACTIVE_ARG = CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1223; /** * The message ID for the message that will be used if dsconfig cannot * read input from the console. This takes a single argument, which @@ -9378,6 +9379,32 @@ CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1239; /** * The message ID for the message that will be used as the description of the * clearBackend argument. This does not take any arguments. */ public static final int MSGID_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND = CATEGORY_MASK_TOOLS | SEVERITY_MASK_INFORMATIONAL | 1240; /** * The message ID for the message that will be used if neither the * includeBranchStrings or the backendID arguments was provided. * This takes two arguments, which are the long identifiers for the * includeBranchStrings and backendID options. */ public static final int MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT = CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1241; /** * The message ID for the message that will be used if the clearBackend * argument was not provided when the backendID argument was provided without * an append argument. This takes the long identifier of the clearBackend * option as an argument. */ public static final int MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND = CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1242; /** * Associates a set of generic messages with the message IDs defined in this * class. */ @@ -9664,11 +9691,12 @@ "Unable to decode include filter string \"%s\" as a " + "valid search filter: %s"); registerMessage(MSGID_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID, "Multiple Directory Server backends are configured with " + "backend ID \"%s\""); "Imported branches or backend IDs can not span across " + "multiple Directory Server backends"); registerMessage(MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID, "None of the Directory Server backends are configured " + "with the requested backend ID \"%s\""); "with the requested backend ID or base DNs that include " + "the specified branches"); registerMessage(MSGID_LDIFIMPORT_CANNOT_IMPORT, "The Directory Server backend with backend ID %s does " + "not provide a mechanism for performing LDIF imports"); @@ -12404,6 +12432,18 @@ "Invalid response. Please enter a value between 1 and %s"); registerMessage(MSGID_DSCFG_ERROR_GENERAL_CONFIRM, "Invalid response. Please enter \"%s\" or \"%s\""); registerMessage(MSGID_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND, "Remove all entries for all base DNs in the backend " + "before importing"); registerMessage(MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT, "Neither the %s or the %s argument was provided. One " + "of these arguments must be given to specify the backend " + "for the LDIF data to be imported to"); registerMessage(MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND, "Importing to a backend without the append argument will " + "remove all entries for all base DNs (%s) in the " + "backend. The %s argument must be given to continue with " + "import"); } }
@@ -80,6 +80,7 @@ boolean overwrite = false; boolean replaceExisting = false; boolean skipSchemaValidation = false; boolean clearBackend = false; String backendID = null; String rejectFile = null; String skipFile = null; @@ -132,6 +133,7 @@ AttributeType typeSkipSchemaValidation; AttributeType typeIsCompressed; AttributeType typeIsEncrypted; AttributeType typeClearBackend; typeLdifFile = getAttributeType(ATTR_IMPORT_LDIF_FILE, true); @@ -165,6 +167,8 @@ getAttributeType(ATTR_IMPORT_IS_COMPRESSED, true); typeIsEncrypted = getAttributeType(ATTR_IMPORT_IS_ENCRYPTED, true); typeClearBackend = getAttributeType(ATTR_IMPORT_CLEAR_BACKEND, true); List<Attribute> attrList; @@ -216,6 +220,113 @@ attrList = taskEntry.getAttribute(typeIsEncrypted); isEncrypted = TaskUtils.getBoolean(attrList, false); attrList = taskEntry.getAttribute(typeClearBackend); clearBackend = TaskUtils.getBoolean(attrList, false); // Make sure that either the "includeBranchStrings" argument or the // "backendID" argument was provided, but not both. if(!includeBranchStrings.isEmpty()) { if(backendID != null) { int msgID = MSGID_LDIFIMPORT_CONFLICTING_OPTIONS; String message = getMessage(msgID, typeIncludeBranch.getNameOrOID(), typeBackendID.getNameOrOID()); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } } else if(backendID == null) { int msgID = MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT; String message = getMessage(msgID, typeIncludeBranch.getNameOrOID(), typeBackendID.getNameOrOID()); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } if(backendID != null) { Backend backend = DirectoryServer.getBackend(backendID); if (backend == null) { int msgID = MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID; String message = getMessage(msgID, backendID); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } else if (! backend.supportsLDIFImport()) { int msgID = MSGID_LDIFIMPORT_CANNOT_IMPORT; String message = getMessage(msgID, backendID); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } // Make sure that if the "backendID" argument was provided and the // "append" option was not provided, the "clearBackend" argument was also // provided if there are more then one baseDNs for the backend being // imported. else if(!append && backend.getBaseDNs().length > 1 && !clearBackend) { StringBuilder builder = new StringBuilder(); for(DN dn : backend.getBaseDNs()) { builder.append(dn.toNormalizedString()); builder.append(" "); } int msgID = MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND; String message = getMessage(msgID, builder.toString(), typeClearBackend.getNameOrOID()); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } } else { // Find the backend that includes all the branches. Backend backend = null; for (String s : includeBranchStrings) { DN includeBranch; try { includeBranch = DN.decode(s); } catch (DirectoryException de) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, de.getErrorMessage()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } catch (Exception e) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, getExceptionMessage(e)); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } Backend locatedBackend = DirectoryServer.getBackend(includeBranch); if(locatedBackend != null) { if(backend == null) { backend = locatedBackend; } else if(backend != locatedBackend) { // The include branches span across multiple backends. int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE; String message = getMessage(msgID, s, backend.getBackendID()); throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message, msgID); } } } } } @@ -295,27 +406,106 @@ // Get the backend into which the LDIF should be imported. Backend backend; Backend backend = null; ArrayList<DN> defaultIncludeBranches; ArrayList<DN> excludeBranches = new ArrayList<DN>(); ArrayList<DN> includeBranches = new ArrayList<DN>(); backend = DirectoryServer.getBackend(backendID); if (backend == null) includeBranches = new ArrayList<DN>(includeBranchStrings.size()); for (String s : includeBranchStrings) { int msgID = MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID; String message = getMessage(msgID, backendID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; DN includeBranch; try { includeBranch = DN.decode(s); } catch (DirectoryException de) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, de.getErrorMessage()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } catch (Exception e) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, getExceptionMessage(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } includeBranches.add(includeBranch); } else if (! backend.supportsLDIFImport()) if(backendID != null) { int msgID = MSGID_LDIFIMPORT_CANNOT_IMPORT; String message = getMessage(msgID, backendID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; backend = DirectoryServer.getBackend(backendID); if (backend == null) { int msgID = MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID; String message = getMessage(msgID, backendID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } else if (! backend.supportsLDIFImport()) { int msgID = MSGID_LDIFIMPORT_CANNOT_IMPORT; String message = getMessage(msgID, backendID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } // Make sure that if the "backendID" argument was provided and the // "append" option was not provided, the "clearBackend" argument was also // provided if there are more then one baseDNs for the backend being // imported. else if(!append && backend.getBaseDNs().length > 1 && !clearBackend) { StringBuilder builder = new StringBuilder(); builder.append(backend.getBaseDNs()[0].toNormalizedString()); for(int i = 1; i < backend.getBaseDNs().length; i++) { builder.append(" / "); builder.append(backend.getBaseDNs()[i].toNormalizedString()); } int msgID = MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND; String message = getMessage(msgID, builder.toString(), ATTR_IMPORT_CLEAR_BACKEND); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } } else { // Find the backend that includes all the branches. for(DN includeBranch : includeBranches) { Backend locatedBackend = DirectoryServer.getBackend(includeBranch); if(locatedBackend != null) { if(backend == null) { backend = locatedBackend; } else if(backend != locatedBackend) { // The include branches span across multiple backends. int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE; String message = getMessage(msgID, includeBranch.toNormalizedString(), backend.getBackendID()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } } } } // Find backends with subordinate base DNs that should be excluded from the @@ -380,53 +570,10 @@ } } ArrayList<DN> includeBranches; if (includeBranchStrings.isEmpty()) { includeBranches = defaultIncludeBranches; } else { includeBranches = new ArrayList<DN>(includeBranchStrings.size()); for (String s : includeBranchStrings) { DN includeBranch; try { includeBranch = DN.decode(s); } catch (DirectoryException de) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, de.getErrorMessage()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } catch (Exception e) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, getExceptionMessage(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches, excludeBranches)) { int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE; String message = getMessage(msgID, s, backendID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return TaskState.STOPPED_BY_ERROR; } includeBranches.add(includeBranch); } } // Create the LDIF import configuration to use when reading the LDIF. ArrayList<String> fileList = new ArrayList<String>(ldifFiles); @@ -435,6 +582,7 @@ importConfig.setReplaceExistingEntries(replaceExisting); importConfig.setCompressed(isCompressed); importConfig.setEncrypted(isEncrypted); importConfig.setClearBackend(clearBackend); importConfig.setExcludeAttributes(excludeAttributes); importConfig.setExcludeBranches(excludeBranches); importConfig.setExcludeFilters(excludeFilters); @@ -511,7 +659,7 @@ // Disable the backend. try { TaskUtils.disableBackend(backendID); TaskUtils.disableBackend(backend.getBackendID()); } catch (DirectoryException e) { @@ -631,11 +779,11 @@ // Enable the backend. try { TaskUtils.enableBackend(backendID); TaskUtils.enableBackend(backend.getBackendID()); // It is necessary to retrieve the backend structure again // because disabling and enabling it again may have resulted // in a new backend being registered to the server. backend = DirectoryServer.getBackend(backendID); backend = DirectoryServer.getBackend(backend.getBackendID()); } catch (DirectoryException e) { opends/src/server/org/opends/server/tools/ImportLDIF.java
@@ -172,6 +172,7 @@ BooleanArgument quietMode = null; BooleanArgument replaceExisting = null; BooleanArgument skipSchemaValidation = null; BooleanArgument clearBackend = null; IntegerArgument randomSeed = null; StringArgument backendID = null; StringArgument configClass = null; @@ -245,11 +246,16 @@ backendID = new StringArgument("backendid", 'n', "backendID", true, false, true, new StringArgument("backendid", 'n', "backendID", false, false, true, "{backendID}", null, null, MSGID_LDIFIMPORT_DESCRIPTION_BACKEND_ID); argParser.addArgument(backendID); clearBackend = new BooleanArgument("clearbackend", 'F', "clearBackend", MSGID_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND); argParser.addArgument(clearBackend); includeBranchStrings = new StringArgument("includebranch", 'b', "includeBranch", false, @@ -413,6 +419,29 @@ return 1; } // Make sure that either the "includeBranchStrings" argument or the // "backendID" argument was provided, but not both. if(includeBranchStrings.isPresent()) { if(backendID.isPresent()) { int msgID = MSGID_LDIFIMPORT_CONFLICTING_OPTIONS; String message = getMessage(msgID, includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier()); err.println(wrapText(message, MAX_LINE_WIDTH)); return 1; } } else if(! backendID.isPresent()) { int msgID = MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT; String message = getMessage(msgID, includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier()); err.println(wrapText(message, MAX_LINE_WIDTH)); return 1; } // Perform the initial bootstrap of the Directory Server and process the // configuration. @@ -766,6 +795,38 @@ Backend backend = null; List<DN> defaultIncludeBranches = null; List<DN> excludeBranches = new ArrayList<DN>(); List<DN> includeBranches = new ArrayList<DN>(); if (includeBranchStrings.isPresent()) { includeBranches = new ArrayList<DN>(); for (String s : includeBranchStrings.getValues()) { DN includeBranch; try { includeBranch = DN.decode(s); } catch (DirectoryException de) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, de.getErrorMessage()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return 1; } catch (Exception e) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, getExceptionMessage(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return 1; } includeBranches.add(includeBranch); } } ArrayList<Backend> backendList = new ArrayList<Backend>(); ArrayList<BackendCfg> entryList = new ArrayList<BackendCfg>(); @@ -780,9 +841,36 @@ for (int i=0; i < numBackends; i++) { Backend b = backendList.get(i); if (! backendID.getValue().equals(b.getBackendID())) if(backendID.isPresent()) { continue; if (! backendID.getValue().equals(b.getBackendID())) { continue; } } else { boolean useBackend = false; for(DN baseDN : dnList.get(i)) { for(DN includeDN : includeBranches) { if(baseDN.isAncestorOf(includeDN)) { useBackend = true; break; } } if(useBackend) { break; } } if(!useBackend) { continue; } } if (backend == null) @@ -793,7 +881,7 @@ else { int msgID = MSGID_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID; String message = getMessage(msgID, backendID.getValue()); String message = getMessage(msgID); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return 1; @@ -837,6 +925,27 @@ } } // Make sure that if the "backendID" argument was provided and the "append" // option was not provided, the "clearBackend" argument was also // provided if there are more then one baseDNs for the backend being // imported. if(backendID.isPresent() && !append.isPresent() && defaultIncludeBranches.size() > 1 && !clearBackend.isPresent()) { StringBuilder builder = new StringBuilder(); builder.append(backend.getBaseDNs()[0].toNormalizedString()); for(int i = 1; i < backend.getBaseDNs().length; i++) { builder.append(" / "); builder.append(backend.getBaseDNs()[i].toNormalizedString()); } int msgID = MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND; String message = getMessage(msgID, builder.toString(), clearBackend.getLongIdentifier()); err.println(wrapText(message, MAX_LINE_WIDTH)); return 1; } for (String s : excludeBranchStrings.getValues()) { DN excludeBranch; @@ -867,50 +976,25 @@ } } List<DN> includeBranches; if (! includeBranchStrings.isPresent()) { includeBranches = defaultIncludeBranches; } else { includeBranches = new ArrayList<DN>(); for (String s : includeBranchStrings.getValues()) // Make sure the selected backend will handle all the include branches for(DN includeBranch : includeBranches) { DN includeBranch; try { includeBranch = DN.decode(s); } catch (DirectoryException de) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, de.getErrorMessage()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return 1; } catch (Exception e) { int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE; String message = getMessage(msgID, s, getExceptionMessage(e)); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return 1; } if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches, excludeBranches)) { int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE; String message = getMessage(msgID, s, backendID.getValue()); String message = getMessage(msgID, includeBranch.toNormalizedString(), backendID.getValue()); logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message, msgID); return 1; } includeBranches.add(includeBranch); } } @@ -983,6 +1067,7 @@ importConfig.setAppendToExistingData(append.isPresent()); importConfig.setReplaceExistingEntries(replaceExisting.isPresent()); importConfig.setCompressed(isCompressed.isPresent()); importConfig.setClearBackend(clearBackend.isPresent()); importConfig.setEncrypted(isEncrypted.isPresent()); importConfig.setExcludeAttributes(excludeAttributes); importConfig.setExcludeBranches(excludeBranches); opends/src/server/org/opends/server/types/LDIFImportConfig.java
@@ -95,6 +95,9 @@ // Indicates whether the import is encrypted. private boolean isEncrypted; // Indicates whether to clear all base DNs in a backend. private boolean clearBackend; // Indicates whether to replace existing entries when appending // data. private boolean replaceExistingEntries; @@ -184,6 +187,7 @@ invokeImportPlugins = false; isCompressed = false; isEncrypted = false; clearBackend = false; validateSchema = true; reader = null; rejectWriter = null; @@ -795,6 +799,34 @@ /** * Indicates whether to clear the entire backend if importing to a * backend with more than one base DNs. * * @return <CODE>true</code> if the entire backend should be * cleared or <CODE>false</CODE> if not. */ public boolean clearBackend() { return clearBackend; } /** * Specifies whether to clear the entire backend if importing to a * backend. * * @param clearBackend Indicates whether to clear the entire * backend. */ public void setClearBackend(boolean clearBackend) { this.clearBackend = clearBackend; } /** * Indicates whether to perform schema validation on entries as they * are read. * opends/src/server/org/opends/server/util/StaticUtils.java
@@ -1601,7 +1601,7 @@ buffer.append("("); for (StackTraceElement e : t.getStackTrace()) { if (i > 4) if (i > 20) { buffer.append(" ..."); break;
