From f8ef0eed366445c5a341dbcc7882a7104c1cac1b Mon Sep 17 00:00:00 2001
From: boli <boli@localhost>
Date: Thu, 26 Jul 2007 16:31:34 +0000
Subject: [PATCH] This fixes issue 1971 and allows partial non-append imports of a backend:
---
opends/resource/schema/02-config.ldif | 9
opends/src/server/org/opends/server/util/StaticUtils.java | 2
opends/src/server/org/opends/server/backends/jeb/State.java | 10
opends/src/server/org/opends/server/backends/jeb/ImportContext.java | 128 ++
opends/src/server/org/opends/server/backends/jeb/ImportJob.java | 896 ++++++++++++++------
opends/src/server/org/opends/server/backends/jeb/IndexRebuildThread.java | 84 +
opends/src/server/org/opends/server/types/LDIFImportConfig.java | 32
opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java | 18
opends/src/server/org/opends/server/tools/ImportLDIF.java | 151 ++
opends/src/server/org/opends/server/backends/jeb/RootContainer.java | 104 +-
opends/src/server/org/opends/server/messages/ToolMessages.java | 46 +
opends/src/server/org/opends/server/backends/jeb/EntryContainer.java | 658 +++++++++-----
opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java | 8
opends/src/server/org/opends/server/messages/JebMessages.java | 43
opends/src/server/org/opends/server/backends/jeb/BackendImpl.java | 10
opends/src/server/org/opends/server/tasks/ImportTask.java | 268 ++++-
opends/src/server/org/opends/server/config/ConfigConstants.java | 7
opends/src/server/org/opends/server/backends/jeb/DatabaseContainer.java | 28
18 files changed, 1,751 insertions(+), 751 deletions(-)
diff --git a/opends/resource/schema/02-config.ldif b/opends/resource/schema/02-config.ldif
index a510e36..5fb22e9 100644
--- a/opends/resource/schema/02-config.ldif
+++ b/opends/resource/schema/02-config.ldif
@@ -1498,6 +1498,10 @@
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.450 NAME 'ds-cfg-message-body'
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE
X-ORIGIN 'OpenDS Directory Server' )
+attributeTypes: ( 1.3.6.1.4.1.26027.1.1.451
+ NAME 'ds-task-import-clear-backend'
+ SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE
+ X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.1
NAME 'ds-cfg-access-control-handler' SUP top STRUCTURAL
MUST ( cn $ ds-cfg-acl-handler-class $ ds-cfg-acl-handler-enabled )
@@ -1836,14 +1840,15 @@
X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.64
NAME 'ds-task-import' SUP ds-task
- MUST ( ds-task-import-ldif-file $ ds-task-import-backend-id )
+ MUST ( ds-task-import-ldif-file )
MAY ( ds-task-import-append $ ds-task-import-replace-existing $
ds-task-import-include-branch $ ds-task-import-exclude-branch $
ds-task-import-include-attribute $ ds-task-import-exclude-attribute $
ds-task-import-include-filter $ ds-task-import-exclude-filter $
ds-task-import-reject-file $ ds-task-import-overwrite-rejects $
ds-task-import-skip-schema-validation $ ds-task-import-is-compressed $
- ds-task-import-is-encrypted )
+ ds-task-import-is-encrypted $ ds-task-import-backend-id $
+ ds-task-import-clear-backend )
X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.65 NAME
( 'ds-cfg-replication-server-config'
diff --git a/opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java b/opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java
index 4aa1c95..5cdbe74 100644
--- a/opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java
+++ b/opends/src/server/org/opends/server/backends/jeb/AttributeIndex.java
@@ -139,7 +139,8 @@
this.state = state;
AttributeType attrType = indexConfig.getIndexAttribute();
- String name = attrType.getNameOrOID();
+ String name =
+ entryContainer.getDatabasePrefix() + "_" + attrType.getNameOrOID();
int indexEntryLimit = indexConfig.getIndexEntryLimit();
if (indexConfig.getIndexType().contains(JEIndexCfgDefn.IndexType.EQUALITY))
@@ -1216,7 +1217,8 @@
try
{
AttributeType attrType = cfg.getIndexAttribute();
- String name = attrType.getNameOrOID();
+ String name =
+ entryContainer.getDatabasePrefix() + "_" + attrType.getNameOrOID();
int indexEntryLimit = cfg.getIndexEntryLimit();
if (cfg.getIndexType().contains(JEIndexCfgDefn.IndexType.EQUALITY))
@@ -1260,7 +1262,7 @@
entryContainer.exclusiveLock.lock();
try
{
- entryContainer.removeDatabase(equalityIndex);
+ entryContainer.deleteDatabase(equalityIndex);
equalityIndex = null;
}
catch(DatabaseException de)
@@ -1314,7 +1316,7 @@
entryContainer.exclusiveLock.lock();
try
{
- entryContainer.removeDatabase(presenceIndex);
+ entryContainer.deleteDatabase(presenceIndex);
presenceIndex = null;
}
catch(DatabaseException de)
@@ -1377,7 +1379,7 @@
entryContainer.exclusiveLock.lock();
try
{
- entryContainer.removeDatabase(substringIndex);
+ entryContainer.deleteDatabase(substringIndex);
substringIndex = null;
}
catch(DatabaseException de)
@@ -1431,7 +1433,7 @@
entryContainer.exclusiveLock.lock();
try
{
- entryContainer.removeDatabase(orderingIndex);
+ entryContainer.deleteDatabase(orderingIndex);
orderingIndex = null;
}
catch(DatabaseException de)
@@ -1485,7 +1487,7 @@
entryContainer.exclusiveLock.lock();
try
{
- entryContainer.removeDatabase(approximateIndex);
+ entryContainer.deleteDatabase(approximateIndex);
approximateIndex = null;
}
catch(DatabaseException de)
@@ -1595,7 +1597,7 @@
public String getName()
{
StringBuilder builder = new StringBuilder();
- builder.append(entryContainer.getContainerName());
+ builder.append(entryContainer.getDatabasePrefix());
builder.append("_");
builder.append(indexConfig.getIndexAttribute().getNameOrOID());
return builder.toString();
diff --git a/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java b/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
index e9fd080..bcf83cb 100644
--- a/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
+++ b/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
@@ -1070,7 +1070,7 @@
envConfig.setConfigParam("je.env.isLocking", "true");
envConfig.setConfigParam("je.env.runCheckpointer", "false");
}
- else
+ else if(importConfig.clearBackend())
{
// We have the writer lock on the environment, now delete the
// environment and re-open it. Only do this when we are
@@ -1450,7 +1450,9 @@
{
// The base DN was deleted.
DirectoryServer.deregisterBaseDN(baseDN, false);
- rootContainer.removeEntryContainer(baseDN);
+ EntryContainer ec =
+ rootContainer.unregisterEntryContainer(baseDN);
+ ec.delete();
}
}
@@ -1461,7 +1463,9 @@
try
{
// The base DN was added.
- rootContainer.openEntryContainer(baseDN);
+ EntryContainer ec =
+ rootContainer.openEntryContainer(baseDN, null);
+ rootContainer.registerEntryContainer(baseDN, ec);
DirectoryServer.registerBaseDN(baseDN, this, false, false);
}
catch (Exception e)
diff --git a/opends/src/server/org/opends/server/backends/jeb/DatabaseContainer.java b/opends/src/server/org/opends/server/backends/jeb/DatabaseContainer.java
index 6e62932..131bbaf 100644
--- a/opends/src/server/org/opends/server/backends/jeb/DatabaseContainer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/DatabaseContainer.java
@@ -92,10 +92,7 @@
this.env = env;
this.entryContainer = entryContainer;
this.databases = new CopyOnWriteArrayList<Database>();
-
- StringBuilder builder = new StringBuilder();
- buildDatabaseName(builder, name);
- this.name = builder.toString();
+ this.name = name;
}
/**
@@ -175,19 +172,6 @@
}
/**
- * Constructs a full JE database name incorporating a entryContainer name.
- *
- * @param builder A string builder to which the full name will be appended.
- * @param name The short database name.
- */
- private void buildDatabaseName(StringBuilder builder, String name)
- {
- builder.append(entryContainer.getContainerName());
- builder.append('_');
- builder.append(name);
- }
-
- /**
* Flush any cached database information to disk and close the
* database container.
*
@@ -385,4 +369,14 @@
{
return getDatabase().preload(config);
}
+
+ /**
+ * Set the JE database name to use for this container.
+ *
+ * @param name The database name to use for this container.
+ */
+ void setName(String name)
+ {
+ this.name = name;
+ }
}
diff --git a/opends/src/server/org/opends/server/backends/jeb/EntryContainer.java b/opends/src/server/org/opends/server/backends/jeb/EntryContainer.java
index 207e6b7..69fb221 100644
--- a/opends/src/server/org/opends/server/backends/jeb/EntryContainer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/EntryContainer.java
@@ -186,6 +186,7 @@
private int indexEntryLimit;
+ private String databasePrefix;
/**
* A read write lock to handle schema changes and bulk changes.
*/
@@ -198,6 +199,8 @@
*
* @param baseDN The baseDN this entry container will be responsible for
* storing on disk.
+ * @param databasePrefix The prefix to use in the database names used by
+ * this entry container.
* @param backend A reference to the JE backend that is creating this entry
* container. It is needed by the Directory Server entry cache
* methods.
@@ -206,8 +209,9 @@
* @param rootContainer The root container this entry container is in.
* @throws ConfigException if a configuration related error occurs.
*/
- public EntryContainer(DN baseDN, Backend backend, JEBackendCfg config,
- Environment env, RootContainer rootContainer)
+ public EntryContainer(DN baseDN, String databasePrefix, Backend backend,
+ JEBackendCfg config, Environment env,
+ RootContainer rootContainer)
throws ConfigException
{
this.backend = backend;
@@ -215,6 +219,22 @@
this.config = config;
this.env = env;
this.rootContainer = rootContainer;
+
+ StringBuilder builder = new StringBuilder(databasePrefix.length());
+ for (int i = 0; i < databasePrefix.length(); i++)
+ {
+ char ch = databasePrefix.charAt(i);
+ if (Character.isLetterOrDigit(ch))
+ {
+ builder.append(ch);
+ }
+ else
+ {
+ builder.append('_');
+ }
+ }
+ this.databasePrefix = builder.toString();
+
this.deadlockRetryLimit = config.getBackendDeadlockRetryLimit();
this.subtreeDeleteSizeLimit = config.getBackendSubtreeDeleteSizeLimit();
this.subtreeDeleteBatchSize = config.getBackendSubtreeDeleteBatchSize();
@@ -242,28 +262,29 @@
DataConfig entryDataConfig = new DataConfig();
entryDataConfig.setCompressed(config.isBackendEntriesCompressed());
- id2entry = new ID2Entry(ID2ENTRY_DATABASE_NAME, entryDataConfig,
- env, this);
+ id2entry = new ID2Entry(databasePrefix + "_" + ID2ENTRY_DATABASE_NAME,
+ entryDataConfig, env, this);
id2entry.open();
- dn2id = new DN2ID(DN2ID_DATABASE_NAME, env, this);
+ dn2id = new DN2ID(databasePrefix + "_" + DN2ID_DATABASE_NAME, env, this);
dn2id.open();
- state = new State(STATE_DATABASE_NAME, env, this);
+ state = new State(databasePrefix + "_" + STATE_DATABASE_NAME, env, this);
state.open();
- id2children = new Index(ID2CHILDREN_DATABASE_NAME,
+ id2children = new Index(databasePrefix + "_" + ID2CHILDREN_DATABASE_NAME,
new ID2CIndexer(), state,
indexEntryLimit, 0,
env,this);
id2children.open();
- id2subtree = new Index(ID2SUBTREE_DATABASE_NAME,
+ id2subtree = new Index(databasePrefix + "_" + ID2SUBTREE_DATABASE_NAME,
new ID2SIndexer(), state,
indexEntryLimit, 0,
env, this);
id2subtree.open();
- dn2uri = new DN2URI(REFERRAL_DATABASE_NAME, env, this);
+ dn2uri = new DN2URI(databasePrefix + "_" + REFERRAL_DATABASE_NAME,
+ env, this);
dn2uri.open();
for (String idx : config.listJEIndexes())
@@ -297,86 +318,11 @@
public void close()
throws DatabaseException
{
- try
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+ for(DatabaseContainer db : databases)
{
- dn2id.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- id2entry.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- id2children.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- id2subtree.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- state.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- dn2uri.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
-
- for (AttributeIndex index : attrIndexMap.values())
- {
- try
- {
- index.close();
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
+ db.close();
}
config.removeJEChangeListener(this);
@@ -3438,86 +3384,6 @@
}
/**
- * Remove the entry entryContainer from disk. The entryContainer must not be
- * open.
- *
- * @throws DatabaseException If an error occurs in the JE database.
- */
- public void removeContainer() throws DatabaseException
- {
- try
- {
- removeDatabase(dn2id);
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- removeDatabase(id2entry);
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- removeDatabase(id2children);
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- removeDatabase(id2subtree);
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- try
- {
- removeDatabase(state);
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- for (AttributeIndex index : attrIndexMap.values())
- {
- try
- {
- removeAttributeIndex(index);
- }
- catch (DatabaseNotFoundException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- }
- attrIndexMap.clear();
- }
-
- /**
* Get the number of values for which the entry limit has been exceeded
* since the entry entryContainer was opened.
* @return The number of values for which the entry limit has been exceeded.
@@ -3545,6 +3411,7 @@
dbList.add(dn2uri);
dbList.add(id2children);
dbList.add(id2subtree);
+ dbList.add(state);
for(AttributeIndex index : attrIndexMap.values())
{
@@ -3579,19 +3446,6 @@
}
/**
- * Constructs a full JE database name incorporating a entryContainer name.
- *
- * @param builder A string builder to which the full name will be appended.
- * @param name The short database name.
- */
- private void buildDatabaseName(StringBuilder builder, String name)
- {
- builder.append(getContainerName());
- builder.append('_');
- builder.append(name);
- }
-
- /**
* Begin a leaf transaction using the default configuration.
* Provides assertion debug logging.
* @return A JE transaction handle.
@@ -3652,20 +3506,92 @@
}
/**
+ * Delete this entry container from disk. The entry container should be
+ * closed before calling this method.
+ *
+ * @throws DatabaseException If an error occurs while removing the entry
+ * container.
+ */
+ public void delete() throws DatabaseException
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+
+ for(DatabaseContainer db : databases)
+ {
+ db.close();
+ }
+
+ if(env.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+
+ try
+ {
+ for(DatabaseContainer db : databases)
+ {
+ env.removeDatabase(txn, db.getName());
+ }
+
+ transactionCommit(txn);
+ }
+ catch(DatabaseException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ for(DatabaseContainer db : databases)
+ {
+ env.removeDatabase(null, db.getName());
+ }
+ }
+ }
+
+ /**
* Remove a database from disk.
*
* @param database The database container to remove.
* @throws DatabaseException If an error occurs while attempting to delete the
* database.
*/
- public void removeDatabase(DatabaseContainer database)
+ public void deleteDatabase(DatabaseContainer database)
throws DatabaseException
{
- database.close();
- env.removeDatabase(null, database.getName());
- if(database instanceof Index)
+ if(database == state)
{
- state.removeIndexTrustState(null, (Index)database);
+ // The state database can not be removed individually.
+ return;
+ }
+
+ database.close();
+ if(env.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+ try
+ {
+ env.removeDatabase(txn, database.getName());
+ if(database instanceof Index)
+ {
+ state.removeIndexTrustState(txn, (Index)database);
+ }
+ transactionCommit(txn);
+ }
+ catch(DatabaseException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ env.removeDatabase(null, database.getName());
+ if(database instanceof Index)
+ {
+ state.removeIndexTrustState(null, (Index)database);
+ }
}
}
@@ -3676,34 +3602,75 @@
* @throws DatabaseException If an JE database error occurs while attempting
* to delete the index.
*/
- public void removeAttributeIndex(AttributeIndex index)
+ public void deleteAttributeIndex(AttributeIndex index)
throws DatabaseException
{
index.close();
- if(index.equalityIndex != null)
+ if(env.getConfig().getTransactional())
{
- env.removeDatabase(null, index.equalityIndex.getName());
- state.removeIndexTrustState(null, index.equalityIndex);
+ Transaction txn = beginTransaction();
+ try
+ {
+ if(index.equalityIndex != null)
+ {
+ env.removeDatabase(txn, index.equalityIndex.getName());
+ state.removeIndexTrustState(txn, index.equalityIndex);
+ }
+ if(index.presenceIndex != null)
+ {
+ env.removeDatabase(txn, index.presenceIndex.getName());
+ state.removeIndexTrustState(txn, index.presenceIndex);
+ }
+ if(index.substringIndex != null)
+ {
+ env.removeDatabase(txn, index.substringIndex.getName());
+ state.removeIndexTrustState(txn, index.substringIndex);
+ }
+ if(index.orderingIndex != null)
+ {
+ env.removeDatabase(txn, index.orderingIndex.getName());
+ state.removeIndexTrustState(txn, index.orderingIndex);
+ }
+ if(index.approximateIndex != null)
+ {
+ env.removeDatabase(txn, index.approximateIndex.getName());
+ state.removeIndexTrustState(txn, index.approximateIndex);
+ }
+ transactionCommit(txn);
+ }
+ catch(DatabaseException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
}
- if(index.presenceIndex != null)
+ else
{
- env.removeDatabase(null, index.presenceIndex.getName());
- state.removeIndexTrustState(null, index.presenceIndex);
- }
- if(index.substringIndex != null)
- {
- env.removeDatabase(null, index.substringIndex.getName());
- state.removeIndexTrustState(null, index.substringIndex);
- }
- if(index.orderingIndex != null)
- {
- env.removeDatabase(null, index.orderingIndex.getName());
- state.removeIndexTrustState(null, index.orderingIndex);
- }
- if(index.approximateIndex != null)
- {
- env.removeDatabase(null, index.approximateIndex.getName());
- state.removeIndexTrustState(null, index.approximateIndex);
+ if(index.equalityIndex != null)
+ {
+ env.removeDatabase(null, index.equalityIndex.getName());
+ state.removeIndexTrustState(null, index.equalityIndex);
+ }
+ if(index.presenceIndex != null)
+ {
+ env.removeDatabase(null, index.presenceIndex.getName());
+ state.removeIndexTrustState(null, index.presenceIndex);
+ }
+ if(index.substringIndex != null)
+ {
+ env.removeDatabase(null, index.substringIndex.getName());
+ state.removeIndexTrustState(null, index.substringIndex);
+ }
+ if(index.orderingIndex != null)
+ {
+ env.removeDatabase(null, index.orderingIndex.getName());
+ state.removeIndexTrustState(null, index.orderingIndex);
+ }
+ if(index.approximateIndex != null)
+ {
+ env.removeDatabase(null, index.approximateIndex.getName());
+ state.removeIndexTrustState(null, index.approximateIndex);
+ }
}
}
@@ -3714,13 +3681,30 @@
*
* @return The container name for the base DN.
*/
- public String getContainerName()
+ public String getDatabasePrefix()
{
- String normStr = baseDN.toNormalizedString();
- StringBuilder builder = new StringBuilder(normStr.length());
- for (int i = 0; i < normStr.length(); i++)
+ return databasePrefix;
+ }
+
+ /**
+ * Sets a new database prefix for this entry container and rename all
+ * existing databases in use by this entry container.
+ *
+ * @param newDatabasePrefix The new database prefix to use.
+ * @throws DatabaseException If an error occurs in the JE database.
+ * @throws JebException If an error occurs in the JE backend.
+ */
+ public void setDatabasePrefix(String newDatabasePrefix)
+ throws DatabaseException, JebException
+
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+
+ StringBuilder builder = new StringBuilder(newDatabasePrefix.length());
+ for (int i = 0; i < newDatabasePrefix.length(); i++)
{
- char ch = normStr.charAt(i);
+ char ch = newDatabasePrefix.charAt(i);
if (Character.isLetterOrDigit(ch))
{
builder.append(ch);
@@ -3730,9 +3714,75 @@
builder.append('_');
}
}
- return builder.toString();
+ newDatabasePrefix = builder.toString();
+
+ // close the containers.
+ for(DatabaseContainer db : databases)
+ {
+ db.close();
+ }
+
+ try
+ {
+ if(env.getConfig().getTransactional())
+ {
+ //Rename under transaction
+ Transaction txn = beginTransaction();
+ try
+ {
+ for(DatabaseContainer db : databases)
+ {
+ String oldName = db.getName();
+ String newName = oldName.replace(databasePrefix, newDatabasePrefix);
+ env.renameDatabase(txn, oldName, newName);
+ }
+
+ transactionCommit(txn);
+
+ for(DatabaseContainer db : databases)
+ {
+ String oldName = db.getName();
+ String newName = oldName.replace(databasePrefix, newDatabasePrefix);
+ db.setName(newName);
+ }
+
+ // Update the prefix.
+ this.databasePrefix = newDatabasePrefix;
+ }
+ catch(Exception e)
+ {
+ transactionAbort(txn);
+
+ int messageID = MSGID_JEB_UNCHECKED_EXCEPTION;
+ String message = getMessage(messageID);
+ throw new JebException(messageID, message, e);
+ }
+ }
+ else
+ {
+ for(DatabaseContainer db : databases)
+ {
+ String oldName = db.getName();
+ String newName = oldName.replace(databasePrefix, newDatabasePrefix);
+ env.renameDatabase(null, oldName, newName);
+ db.setName(newName);
+ }
+
+ // Update the prefix.
+ this.databasePrefix = newDatabasePrefix;
+ }
+ }
+ finally
+ {
+ // Open the containers backup.
+ for(DatabaseContainer db : databases)
+ {
+ db.open();
+ }
+ }
}
+
/**
* Get the baseDN this entry container is responsible for.
*
@@ -3876,7 +3926,7 @@
try
{
AttributeIndex index = attrIndexMap.get(cfg.getIndexAttribute());
- removeAttributeIndex(index);
+ deleteAttributeIndex(index);
attrIndexMap.remove(cfg.getIndexAttribute());
}
catch(DatabaseException de)
@@ -3911,19 +3961,99 @@
}
/**
+ * Clear the contents of this entry container.
+ *
+ * @return The number of records deleted.
+ * @throws DatabaseException If an error occurs while removing the entry
+ * container.
+ */
+ public long clear() throws DatabaseException
+ {
+ List<DatabaseContainer> databases = new ArrayList<DatabaseContainer>();
+ listDatabases(databases);
+ long count = 0;
+
+ for(DatabaseContainer db : databases)
+ {
+ db.close();
+ }
+ try
+ {
+ if(env.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+
+ try
+ {
+ for(DatabaseContainer db : databases)
+ {
+ count += env.truncateDatabase(txn, db.getName(), true);
+ }
+
+ transactionCommit(txn);
+ }
+ catch(DatabaseException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ for(DatabaseContainer db : databases)
+ {
+ count += env.truncateDatabase(null, db.getName(), true);
+ }
+ }
+ }
+ finally
+ {
+ for(DatabaseContainer db : databases)
+ {
+ db.open();
+ }
+ }
+
+ return count;
+ }
+
+ /**
* Clear the contents for a database from disk.
*
- * @param txn A transaction object or null if its not required.
* @param database The database to clear.
* @return The number of records deleted.
* @throws DatabaseException if a JE database error occurs.
*/
- public long clearDatabase(Transaction txn, DatabaseContainer database)
+ public long clearDatabase(DatabaseContainer database)
throws DatabaseException
{
+ long count = 0;
database.close();
- long count = env.truncateDatabase(txn, database.getName(), true);
- database.open();
+ try
+ {
+ if(env.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+ try
+ {
+ count = env.truncateDatabase(txn, database.getName(), true);
+ transactionCommit(txn);
+ }
+ catch(DatabaseException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ count = env.truncateDatabase(null, database.getName(), true);
+ }
+ }
+ finally
+ {
+ database.open();
+ }
if(debugEnabled())
{
TRACER.debugVerbose("Cleared %d existing records from the " +
@@ -3935,39 +4065,89 @@
/**
* Clear the contents for a attribute index from disk.
*
- * @param txn A transaction object or null if its not required.
* @param index The attribute index to clear.
* @return The number of records deleted.
* @throws DatabaseException if a JE database error occurs.
*/
- public long clearAttributeIndex(Transaction txn, AttributeIndex index)
+ public long clearAttributeIndex(AttributeIndex index)
throws DatabaseException
{
long count = 0;
index.close();
- if(index.equalityIndex != null)
+ try
{
- count += env.truncateDatabase(txn, index.equalityIndex.getName(), true);
+ if(env.getConfig().getTransactional())
+ {
+ Transaction txn = beginTransaction();
+ try
+ {
+ if(index.equalityIndex != null)
+ {
+ count += env.truncateDatabase(txn, index.equalityIndex.getName(),
+ true);
+ }
+ if(index.presenceIndex != null)
+ {
+ count += env.truncateDatabase(txn, index.presenceIndex.getName(),
+ true);
+ }
+ if(index.substringIndex != null)
+ {
+ count += env.truncateDatabase(txn, index.substringIndex.getName(),
+ true);
+ }
+ if(index.orderingIndex != null)
+ {
+ count += env.truncateDatabase(txn, index.orderingIndex.getName(),
+ true);
+ }
+ if(index.approximateIndex != null)
+ {
+ count += env.truncateDatabase(txn, index.approximateIndex.getName(),
+ true);
+ }
+ transactionCommit(txn);
+ }
+ catch(DatabaseException de)
+ {
+ transactionAbort(txn);
+ throw de;
+ }
+ }
+ else
+ {
+ if(index.equalityIndex != null)
+ {
+ count += env.truncateDatabase(null, index.equalityIndex.getName(),
+ true);
+ }
+ if(index.presenceIndex != null)
+ {
+ count += env.truncateDatabase(null, index.presenceIndex.getName(),
+ true);
+ }
+ if(index.substringIndex != null)
+ {
+ count += env.truncateDatabase(null, index.substringIndex.getName(),
+ true);
+ }
+ if(index.orderingIndex != null)
+ {
+ count += env.truncateDatabase(null, index.orderingIndex.getName(),
+ true);
+ }
+ if(index.approximateIndex != null)
+ {
+ count += env.truncateDatabase(null, index.approximateIndex.getName(),
+ true);
+ }
+ }
}
- if(index.presenceIndex != null)
+ finally
{
- count += env.truncateDatabase(txn, index.presenceIndex.getName(), true);
+ index.open();
}
- if(index.substringIndex != null)
- {
- count += env.truncateDatabase(txn, index.substringIndex.getName(), true);
- }
- if(index.orderingIndex != null)
- {
- count += env.truncateDatabase(txn, index.orderingIndex.getName(), true);
- }
- if(index.approximateIndex != null)
- {
- count += env.truncateDatabase(txn, index.approximateIndex.getName(),
- true);
- }
- index.open();
if(debugEnabled())
{
TRACER.debugVerbose("Cleared %d existing records from the " +
diff --git a/opends/src/server/org/opends/server/backends/jeb/ImportContext.java b/opends/src/server/org/opends/server/backends/jeb/ImportContext.java
index 7298ac7..d7e8392 100644
--- a/opends/src/server/org/opends/server/backends/jeb/ImportContext.java
+++ b/opends/src/server/org/opends/server/backends/jeb/ImportContext.java
@@ -35,16 +35,13 @@
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.atomic.AtomicLong;
import java.util.ArrayList;
+import java.util.List;
/**
* This class represents the import context for a destination base DN.
*/
public class ImportContext
{
- /**
- * The name of the entryContainer for the destination base DN.
- */
- private String containerName;
/**
* The destination base DN.
@@ -52,6 +49,16 @@
private DN baseDN;
/**
+ * The include branches below the base DN.
+ */
+ private List<DN> includeBranches;
+
+ /**
+ * The exclude branches below the base DN.
+ */
+ private List<DN> excludeBranches;
+
+ /**
* The configuration of the destination backend.
*/
private JEBackendCfg config;
@@ -72,6 +79,11 @@
private EntryContainer entryContainer;
/**
+ * The source entryContainer if this is a partial import of a base DN.
+ */
+ private EntryContainer srcEntryContainer;
+
+ /**
* The amount of buffer memory available in bytes.
*/
private long bufferSize;
@@ -121,24 +133,6 @@
}
/**
- * Set the name of the entryContainer for the destination base DN.
- * @param containerName The entryContainer name.
- */
- public void setContainerName(String containerName)
- {
- this.containerName = containerName;
- }
-
- /**
- * Get the name of the entryContainer for the destination base DN.
- * @return The entryContainer name.
- */
- public String getContainerName()
- {
- return containerName;
- }
-
- /**
* Set the destination base DN.
* @param baseDN The destination base DN.
*/
@@ -229,6 +223,25 @@
}
/**
+ * Set the source entry entryContainer for the destination base DN.
+ * @param srcEntryContainer The entry source entryContainer for the
+ * destination base DN.
+ */
+ public void setSrcEntryContainer(EntryContainer srcEntryContainer)
+ {
+ this.srcEntryContainer = srcEntryContainer;
+ }
+
+ /**
+ * Get the source entry entryContainer for the destination base DN.
+ * @return The source entry entryContainer for the destination base DN.
+ */
+ public EntryContainer getSrcEntryContainer()
+ {
+ return srcEntryContainer;
+ }
+
+ /**
* Get the available buffer size in bytes.
* @return The available buffer size.
*/
@@ -300,4 +313,75 @@
{
this.IDs = IDs;
}
+
+ /**
+ * Retrieves the set of base DNs that specify the set of entries to
+ * exclude from the import. The contents of the returned list may
+ * be altered by the caller.
+ *
+ * @return The set of base DNs that specify the set of entries to
+ * exclude from the import.
+ */
+ public List<DN> getExcludeBranches()
+ {
+ return excludeBranches;
+ }
+
+
+
+ /**
+ * Specifies the set of base DNs that specify the set of entries to
+ * exclude from the import.
+ *
+ * @param excludeBranches The set of base DNs that specify the set
+ * of entries to exclude from the import.
+ */
+ public void setExcludeBranches(List<DN> excludeBranches)
+ {
+ if (excludeBranches == null)
+ {
+ this.excludeBranches = new ArrayList<DN>(0);
+ }
+ else
+ {
+ this.excludeBranches = excludeBranches;
+ }
+ }
+
+
+
+ /**
+ * Retrieves the set of base DNs that specify the set of entries to
+ * include in the import. The contents of the returned list may be
+ * altered by the caller.
+ *
+ * @return The set of base DNs that specify the set of entries to
+ * include in the import.
+ */
+ public List<DN> getIncludeBranches()
+ {
+ return includeBranches;
+ }
+
+
+
+ /**
+ * Specifies the set of base DNs that specify the set of entries to
+ * include in the import.
+ *
+ * @param includeBranches The set of base DNs that specify the set
+ * of entries to include in the import.
+ */
+ public void setIncludeBranches(List<DN> includeBranches)
+ {
+ if (includeBranches == null)
+ {
+ this.includeBranches = new ArrayList<DN>(0);
+ }
+ else
+ {
+ this.includeBranches = includeBranches;
+ }
+ }
+
}
diff --git a/opends/src/server/org/opends/server/backends/jeb/ImportJob.java b/opends/src/server/org/opends/server/backends/jeb/ImportJob.java
index 8e077a1..b06c2c2 100644
--- a/opends/src/server/org/opends/server/backends/jeb/ImportJob.java
+++ b/opends/src/server/org/opends/server/backends/jeb/ImportJob.java
@@ -26,10 +26,7 @@
*/
package org.opends.server.backends.jeb;
-import com.sleepycat.je.DatabaseException;
-import com.sleepycat.je.EnvironmentStats;
-import com.sleepycat.je.StatsConfig;
-import com.sleepycat.je.Transaction;
+import com.sleepycat.je.*;
import org.opends.server.types.DebugLogLevel;
import org.opends.server.messages.JebMessages;
@@ -48,10 +45,7 @@
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Timer;
-import java.util.TimerTask;
+import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
@@ -66,6 +60,8 @@
import org.opends.server.loggers.debug.DebugTracer;
import static org.opends.server.messages.JebMessages.*;
import org.opends.server.admin.std.server.JEBackendCfg;
+import org.opends.server.protocols.asn1.ASN1OctetString;
+import org.opends.server.config.ConfigException;
/**
* Import from LDIF to a JE backend.
@@ -108,12 +104,30 @@
*/
private int importedCount;
+ /**
+ * The number of entries migrated.
+ */
+ private int migratedCount;
+
+ /**
+ * The number of merge passes.
+ */
+ int mergePassNumber = 1;
+
/**
* The number of milliseconds between job progress reports.
*/
private long progressInterval = 10000;
+ /**
+ * The progress report timer.
+ */
+ private Timer timer;
+
+ private int entriesProcessed;
+ private int importPassSize;
+
/**
* The import worker threads.
@@ -146,15 +160,25 @@
* @throws IOException If a problem occurs while opening the LDIF file for
* reading, or while reading from the LDIF file.
* @throws JebException If an error occurs in the JE backend.
+ * @throws DirectoryException if a directory server related error occurs.
+ * @throws ConfigException if a configuration related error occurs.
*/
public LDIFImportResult importLDIF(RootContainer rootContainer)
- throws DatabaseException, IOException, JebException
+ throws DatabaseException, IOException, JebException, DirectoryException,
+ ConfigException
{
// Create an LDIF reader. Throws an exception if the file does not exist.
reader = new LDIFReader(ldifImportConfig);
this.rootContainer = rootContainer;
this.config = rootContainer.getConfiguration();
+ this.mergePassNumber = 1;
+ this.entriesProcessed = 0;
+ this.importPassSize = config.getBackendImportPassSize();
+ if (importPassSize <= 0)
+ {
+ importPassSize = Integer.MAX_VALUE;
+ }
int msgID;
String message;
@@ -185,31 +209,15 @@
TRACER.debugInfo(message);
}
- // Create the import contexts for each base DN.
- DN baseDN;
-
for (EntryContainer entryContainer : rootContainer.getEntryContainers())
{
- baseDN = entryContainer.getBaseDN();
+ ImportContext importContext =
+ getImportContext(entryContainer, bufferSize);
- // Create an import context.
- ImportContext importContext = new ImportContext();
- importContext.setBufferSize(bufferSize);
- importContext.setConfig(config);
- importContext.setLDIFImportConfig(this.ldifImportConfig);
- importContext.setLDIFReader(reader);
-
- importContext.setBaseDN(baseDN);
- importContext.setContainerName(entryContainer.getContainerName());
- importContext.setEntryContainer(entryContainer);
- importContext.setBufferSize(bufferSize);
-
- // Create an entry queue.
- LinkedBlockingQueue<Entry> queue =
- new LinkedBlockingQueue<Entry>(config.getBackendImportQueueSize());
- importContext.setQueue(queue);
-
- importMap.put(baseDN, importContext);
+ if(importContext != null)
+ {
+ importMap.put(entryContainer.getBaseDN(), importContext);
+ }
}
// Make a note of the time we started.
@@ -232,53 +240,57 @@
}
}
+ startWorkerThreads();
try
{
importedCount = 0;
- int passNumber = 1;
- boolean moreData = true;
- while (moreData)
- {
- moreData = processLDIF();
- if (moreData)
- {
- msgID = MSGID_JEB_IMPORT_BEGINNING_INTERMEDIATE_MERGE;
- message = getMessage(msgID, passNumber++);
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
- message, msgID);
- }
- else
- {
- msgID = MSGID_JEB_IMPORT_BEGINNING_FINAL_MERGE;
- message = getMessage(msgID);
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
- message, msgID);
- }
-
-
- long mergeStartTime = System.currentTimeMillis();
- merge();
- long mergeEndTime = System.currentTimeMillis();
-
- if (moreData)
- {
- msgID = MSGID_JEB_IMPORT_RESUMING_LDIF_PROCESSING;
- message = getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000));
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
- message, msgID);
- }
- else
- {
- msgID = MSGID_JEB_IMPORT_FINAL_MERGE_COMPLETED;
- message = getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000));
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
- message, msgID);
- }
- }
+ migratedCount = 0;
+ migrateExistingEntries();
+ processLDIF();
+ migrateExcludedEntries();
}
finally
{
+ merge(false);
tempDir.delete();
+
+ for(ImportContext importContext : importMap.values())
+ {
+ DN baseDN = importContext.getBaseDN();
+ EntryContainer srcEntryContainer =
+ importContext.getSrcEntryContainer();
+ if(srcEntryContainer != null)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugInfo("Deleteing old entry container for base DN " +
+ "%s and renaming temp entry container", baseDN);
+ }
+ EntryContainer unregEC =
+ rootContainer.unregisterEntryContainer(baseDN);
+ //Make sure the unregistered EC for the base DN is the same as
+ //the one in the import context.
+ if(unregEC != srcEntryContainer)
+ {
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Current entry container used for base DN " +
+ "%s is not the same as the source entry container used " +
+ "during the migration process.", baseDN);
+ }
+ rootContainer.registerEntryContainer(baseDN, unregEC);
+ continue;
+ }
+ srcEntryContainer.exclusiveLock.lock();
+ srcEntryContainer.delete();
+ srcEntryContainer.exclusiveLock.unlock();
+ EntryContainer newEC = importContext.getEntryContainer();
+ newEC.exclusiveLock.lock();
+ newEC.setDatabasePrefix(baseDN.toNormalizedString());
+ newEC.exclusiveLock.unlock();
+ rootContainer.registerEntryContainer(baseDN, newEC);
+ }
+ }
}
}
finally
@@ -296,9 +308,11 @@
}
msgID = MSGID_JEB_IMPORT_FINAL_STATUS;
- message = getMessage(msgID, reader.getEntriesRead(), importedCount,
+ message = getMessage(msgID, reader.getEntriesRead(),
+ importedCount - migratedCount,
reader.getEntriesIgnored(),
- reader.getEntriesRejected(), importTime/1000, rate);
+ reader.getEntriesRejected(),
+ migratedCount, importTime/1000, rate);
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
message, msgID);
@@ -314,135 +328,169 @@
/**
* Merge the intermediate files to load the index databases.
+ *
+ * @param moreData <CODE>true</CODE> if this is a intermediate merge or
+ * <CODE>false</CODE> if this is a final merge.
+ * @throws DatabaseException If an error occurs in the JE database.
*/
- public void merge()
+ private void merge(boolean moreData) throws DatabaseException
{
- ArrayList<IndexMergeThread> mergers = new ArrayList<IndexMergeThread>();
+ stopWorkerThreads();
- // Create merge threads for each base DN.
- for (ImportContext importContext : importMap.values())
+ try
{
- EntryContainer entryContainer = importContext.getEntryContainer();
-
- // For each configured attribute index.
- for (AttributeIndex attrIndex : entryContainer.getAttributeIndexes())
+ if (moreData)
{
- int indexEntryLimit = config.getBackendIndexEntryLimit();
- if(attrIndex.getConfiguration().getIndexEntryLimit() != null)
+ int msgID = MSGID_JEB_IMPORT_BEGINNING_INTERMEDIATE_MERGE;
+ String message = getMessage(msgID, mergePassNumber++);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+ }
+ else
+ {
+ int msgID = MSGID_JEB_IMPORT_BEGINNING_FINAL_MERGE;
+ String message = getMessage(msgID);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+ }
+
+
+ long mergeStartTime = System.currentTimeMillis();
+
+ ArrayList<IndexMergeThread> mergers = new ArrayList<IndexMergeThread>();
+
+ // Create merge threads for each base DN.
+ for (ImportContext importContext : importMap.values())
+ {
+ EntryContainer entryContainer = importContext.getEntryContainer();
+
+ // For each configured attribute index.
+ for (AttributeIndex attrIndex : entryContainer.getAttributeIndexes())
{
- indexEntryLimit = attrIndex.getConfiguration().getIndexEntryLimit();
+ int indexEntryLimit = config.getBackendIndexEntryLimit();
+ if(attrIndex.getConfiguration().getIndexEntryLimit() != null)
+ {
+ indexEntryLimit = attrIndex.getConfiguration().getIndexEntryLimit();
+ }
+
+ if (attrIndex.equalityIndex != null)
+ {
+ Index index = attrIndex.equalityIndex;
+ IndexMergeThread indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig, index,
+ indexEntryLimit);
+ mergers.add(indexMergeThread);
+ }
+ if (attrIndex.presenceIndex != null)
+ {
+ Index index = attrIndex.presenceIndex;
+ IndexMergeThread indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig, index,
+ indexEntryLimit);
+ mergers.add(indexMergeThread);
+ }
+ if (attrIndex.substringIndex != null)
+ {
+ Index index = attrIndex.substringIndex;
+ IndexMergeThread indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig, index,
+ indexEntryLimit);
+ mergers.add(indexMergeThread);
+ }
+ if (attrIndex.orderingIndex != null)
+ {
+ Index index = attrIndex.orderingIndex;
+ IndexMergeThread indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig, index,
+ indexEntryLimit);
+ mergers.add(indexMergeThread);
+ }
+ if (attrIndex.approximateIndex != null)
+ {
+ Index index = attrIndex.approximateIndex;
+ IndexMergeThread indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig, index,
+ indexEntryLimit);
+ mergers.add(indexMergeThread);
+ }
}
- if (attrIndex.equalityIndex != null)
+ // Id2Children index.
+ Index id2Children = entryContainer.getID2Children();
+ IndexMergeThread indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig,
+ id2Children,
+ config.getBackendIndexEntryLimit());
+ mergers.add(indexMergeThread);
+
+ // Id2Subtree index.
+ Index id2Subtree = entryContainer.getID2Subtree();
+ indexMergeThread =
+ new IndexMergeThread(config,
+ ldifImportConfig,
+ id2Subtree,
+ config.getBackendIndexEntryLimit());
+ mergers.add(indexMergeThread);
+ }
+
+ // Run all the merge threads.
+ for (IndexMergeThread imt : mergers)
+ {
+ imt.start();
+ }
+
+ // Wait for the threads to finish.
+ for (IndexMergeThread imt : mergers)
+ {
+ try
{
- Index index = attrIndex.equalityIndex;
- IndexMergeThread indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig, index,
- indexEntryLimit);
- mergers.add(indexMergeThread);
+ imt.join();
}
- if (attrIndex.presenceIndex != null)
+ catch (InterruptedException e)
{
- Index index = attrIndex.presenceIndex;
- IndexMergeThread indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig, index,
- indexEntryLimit);
- mergers.add(indexMergeThread);
- }
- if (attrIndex.substringIndex != null)
- {
- Index index = attrIndex.substringIndex;
- IndexMergeThread indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig, index,
- indexEntryLimit);
- mergers.add(indexMergeThread);
- }
- if (attrIndex.orderingIndex != null)
- {
- Index index = attrIndex.orderingIndex;
- IndexMergeThread indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig, index,
- indexEntryLimit);
- mergers.add(indexMergeThread);
- }
- if (attrIndex.approximateIndex != null)
- {
- Index index = attrIndex.approximateIndex;
- IndexMergeThread indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig, index,
- indexEntryLimit);
- mergers.add(indexMergeThread);
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, e);
+ }
}
}
- // Id2Children index.
- Index id2Children = entryContainer.getID2Children();
- IndexMergeThread indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig,
- id2Children,
- config.getBackendIndexEntryLimit());
- mergers.add(indexMergeThread);
+ long mergeEndTime = System.currentTimeMillis();
- // Id2Subtree index.
- Index id2Subtree = entryContainer.getID2Subtree();
- indexMergeThread =
- new IndexMergeThread(config,
- ldifImportConfig,
- id2Subtree,
- config.getBackendIndexEntryLimit());
- mergers.add(indexMergeThread);
- }
-
- // Run all the merge threads.
- for (IndexMergeThread imt : mergers)
- {
- imt.start();
- }
-
- // Wait for the threads to finish.
- for (IndexMergeThread imt : mergers)
- {
- try
+ if (moreData)
{
- imt.join();
+ int msgID = MSGID_JEB_IMPORT_RESUMING_LDIF_PROCESSING;
+ String message =
+ getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000));
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
}
- catch (InterruptedException e)
+ else
{
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
+ int msgID = MSGID_JEB_IMPORT_FINAL_MERGE_COMPLETED;
+ String message =
+ getMessage(msgID, ((mergeEndTime-mergeStartTime)/1000));
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+ }
+ }
+ finally
+ {
+ if(moreData)
+ {
+ startWorkerThreads();
}
}
}
- /**
- * Create a set of worker threads, one set for each base DN.
- * Read each entry from the LDIF and determine which
- * base DN the entry belongs to. Write the dn2id database, then put the
- * entry on the appropriate queue for the worker threads to consume.
- * Record the entry count for each base DN when all entries have been
- * processed.
- *
- * @return true if thre is more data to be read from the LDIF file (the import
- * pass size was reached), false if the entire LDIF file has been read.
- *
- * @throws JebException If an error occurs in the JE backend.
- * @throws DatabaseException If an error occurs in the JE database.
- * @throws IOException If a problem occurs while opening the LDIF file for
- * reading, or while reading from the LDIF file.
- */
- private boolean processLDIF()
- throws JebException, DatabaseException, IOException
+ private void startWorkerThreads() throws DatabaseException
{
- boolean moreData = false;
-
// Create one set of worker threads for each base DN.
int importThreadCount = config.getBackendImportThreadCount();
for (ImportContext ic : importMap.values())
@@ -457,122 +505,291 @@
}
}
- try
+ // Start a timer for the progress report.
+ timer = new Timer();
+ TimerTask progressTask = new ImportJob.ProgressTask();
+ timer.scheduleAtFixedRate(progressTask, progressInterval,
+ progressInterval);
+ }
+
+ private void stopWorkerThreads()
+ {
+ if(threads.size() > 0)
{
- // Create a counter to use to determine whether we've hit the import
- // pass size.
- int entriesProcessed = 0;
- int importPassSize = config.getBackendImportPassSize();
- if (importPassSize <= 0)
+ // Wait for the queues to be drained.
+ for (ImportContext ic : importMap.values())
{
- importPassSize = Integer.MAX_VALUE;
- }
-
- // Start a timer for the progress report.
- Timer timer = new Timer();
- TimerTask progressTask = new ImportJob.ProgressTask();
- timer.scheduleAtFixedRate(progressTask, progressInterval,
- progressInterval);
-
- try
- {
- do
+ while (ic.getQueue().size() > 0)
{
- if(threads.size() <= 0)
- {
- int msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS;
- String msg = getMessage(msgID);
- throw new JebException(msgID, msg);
- }
-
try
{
- // Read the next entry.
- Entry entry = reader.readEntry();
-
- // Check for end of file.
- if (entry == null)
- {
- break;
- }
-
- // Route it according to base DN.
- ImportContext importContext = getImportConfig(entry.getDN());
-
- processEntry(importContext, entry);
-
- entriesProcessed++;
- if (entriesProcessed >= importPassSize)
- {
- moreData = true;
- break;
- }
- }
- catch (LDIFException e)
+ Thread.sleep(100);
+ } catch (Exception e)
{
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
+ // No action needed.
}
- catch (DirectoryException e)
- {
- if (debugEnabled())
- {
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
- }
- }
- } while (true);
+ }
+ }
+ }
- if(threads.size() > 0)
+ // Order the threads to stop.
+ for (ImportThread t : threads)
+ {
+ t.stopProcessing();
+ }
+
+ // Wait for each thread to stop.
+ for (ImportThread t : threads)
+ {
+ try
+ {
+ t.join();
+ importedCount += t.getImportedCount();
+ }
+ catch (InterruptedException ie)
+ {
+ // No action needed?
+ }
+ }
+
+ timer.cancel();
+ }
+
+ /**
+ * Create a set of worker threads, one set for each base DN.
+ * Read each entry from the LDIF and determine which
+ * base DN the entry belongs to. Write the dn2id database, then put the
+ * entry on the appropriate queue for the worker threads to consume.
+ * Record the entry count for each base DN when all entries have been
+ * processed.
+ *
+ * pass size was reached), false if the entire LDIF file has been read.
+ *
+ * @throws JebException If an error occurs in the JE backend.
+ * @throws DatabaseException If an error occurs in the JE database.
+ * @throws IOException If a problem occurs while opening the LDIF file for
+ * reading, or while reading from the LDIF file.
+ */
+ private void processLDIF()
+ throws JebException, DatabaseException, IOException
+ {
+ int msgID = MSGID_JEB_IMPORT_LDIF_START;
+ String message = getMessage(msgID);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+
+ do
+ {
+ if(threads.size() <= 0)
+ {
+ msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS;
+ message = getMessage(msgID);
+ throw new JebException(msgID, message);
+ }
+ try
+ {
+ // Read the next entry.
+ Entry entry = reader.readEntry();
+
+ // Check for end of file.
+ if (entry == null)
{
- // Wait for the queues to be drained.
- for (ImportContext ic : importMap.values())
+ msgID = MSGID_JEB_IMPORT_LDIF_END;
+ message = getMessage(msgID);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+
+ break;
+ }
+
+ // Route it according to base DN.
+ ImportContext importContext = getImportConfig(entry.getDN());
+
+ processEntry(importContext, entry);
+
+ entriesProcessed++;
+ if (entriesProcessed >= importPassSize)
+ {
+ merge(false);
+ entriesProcessed = 0;
+ }
+ }
+ catch (LDIFException e)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, e);
+ }
+ }
+ catch (DirectoryException e)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, e);
+ }
+ }
+ } while (true);
+ }
+
+ private void migrateExistingEntries()
+ throws JebException, DatabaseException, DirectoryException
+ {
+ for(ImportContext importContext : importMap.values())
+ {
+ EntryContainer srcEntryContainer = importContext.getSrcEntryContainer();
+ if(srcEntryContainer != null &&
+ !importContext.getIncludeBranches().isEmpty())
+ {
+ DatabaseEntry key = new DatabaseEntry();
+ DatabaseEntry data = new DatabaseEntry();
+ LockMode lockMode = LockMode.DEFAULT;
+ OperationStatus status;
+
+ int msgID = MSGID_JEB_IMPORT_MIGRATION_START;
+ String message = getMessage(msgID, "existing",
+ importContext.getBaseDN());
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+
+ Cursor cursor =
+ srcEntryContainer.getDN2ID().openCursor(null,
+ CursorConfig.READ_COMMITTED);
+ try
+ {
+ status = cursor.getFirst(key, data, lockMode);
+
+ while(status == OperationStatus.SUCCESS)
{
- while (ic.getQueue().size() > 0)
+ if(threads.size() <= 0)
{
- try
+ msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS;
+ message = getMessage(msgID);
+ throw new JebException(msgID, message);
+ }
+
+ DN dn = DN.decode(new ASN1OctetString(key.getData()));
+ if(!importContext.getIncludeBranches().contains(dn))
+ {
+ EntryID id = new EntryID(data);
+ Entry entry = srcEntryContainer.getID2Entry().get(null, id);
+ processEntry(importContext, entry);
+
+ entriesProcessed++;
+ migratedCount++;
+ if (entriesProcessed >= importPassSize)
{
- Thread.sleep(100);
- } catch (Exception e)
+ merge(true);
+ entriesProcessed = 0;
+ }
+ status = cursor.getNext(key, data, lockMode);
+ }
+ else
+ {
+ // This is the base entry for a branch that will be included
+ // in the import so we don't want to copy the branch to the new
+ // entry container.
+
+ /**
+ * Advance the cursor to next entry at the same level in the DIT
+ * skipping all the entries in this branch.
+ * Set the next starting value to a value of equal length but
+ * slightly greater than the previous DN. Since keys are compared
+ * in reverse order we must set the first byte (the comma).
+ * No possibility of overflow here.
+ */
+ byte[] begin =
+ StaticUtils.getBytes("," + dn.toNormalizedString());
+ begin[0] = (byte) (begin[0] + 1);
+ key.setData(begin);
+ status = cursor.getSearchKeyRange(key, data, lockMode);
+ }
+ }
+ }
+ finally
+ {
+ cursor.close();
+ }
+ }
+ }
+ }
+
+ private void migrateExcludedEntries()
+ throws JebException, DatabaseException
+ {
+ for(ImportContext importContext : importMap.values())
+ {
+ EntryContainer srcEntryContainer = importContext.getSrcEntryContainer();
+ if(srcEntryContainer != null &&
+ !importContext.getExcludeBranches().isEmpty())
+ {
+ DatabaseEntry key = new DatabaseEntry();
+ DatabaseEntry data = new DatabaseEntry();
+ LockMode lockMode = LockMode.DEFAULT;
+ OperationStatus status;
+
+ int msgID = MSGID_JEB_IMPORT_MIGRATION_START;
+ String message = getMessage(msgID, "excluded",
+ importContext.getBaseDN());
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
+ message, msgID);
+
+ Cursor cursor =
+ srcEntryContainer.getDN2ID().openCursor(null,
+ CursorConfig.READ_COMMITTED);
+ Comparator<byte[]> dn2idComparator =
+ srcEntryContainer.getDN2ID().getComparator();
+ try
+ {
+ for(DN excludedDN : importContext.getExcludeBranches())
+ {
+ byte[] suffix =
+ StaticUtils.getBytes(excludedDN.toNormalizedString());
+ key.setData(suffix);
+ status = cursor.getSearchKeyRange(key, data, lockMode);
+
+ if(status == OperationStatus.SUCCESS &&
+ Arrays.equals(key.getData(), suffix))
+ {
+ // This is the base entry for a branch that was excluded in the
+ // import so we must migrate all entries in this branch over to
+ // the new entry container.
+
+ byte[] end =
+ StaticUtils.getBytes("," + excludedDN.toNormalizedString());
+ end[0] = (byte) (end[0] + 1);
+
+ while(status == OperationStatus.SUCCESS &&
+ dn2idComparator.compare(key.getData(), end) < 0)
{
- // No action needed.
+ if(threads.size() <= 0)
+ {
+ msgID = MSGID_JEB_IMPORT_NO_WORKER_THREADS;
+ message = getMessage(msgID);
+ throw new JebException(msgID, message);
+ }
+
+ EntryID id = new EntryID(data);
+ Entry entry = srcEntryContainer.getID2Entry().get(null, id);
+ processEntry(importContext, entry);
+
+ entriesProcessed++;
+ migratedCount++;
+ if (entriesProcessed >= importPassSize)
+ {
+ merge(true);
+ entriesProcessed = 0;
+ }
+ status = cursor.getNext(key, data, lockMode);
}
}
}
}
-
- }
- finally
- {
- timer.cancel();
- }
- }
- finally
- {
- // Order the threads to stop.
- for (ImportThread t : threads)
- {
- t.stopProcessing();
- }
-
- // Wait for each thread to stop.
- for (ImportThread t : threads)
- {
- try
+ finally
{
- t.join();
- importedCount += t.getImportedCount();
- }
- catch (InterruptedException ie)
- {
- // No action needed?
+ cursor.close();
}
}
}
-
-
- return moreData;
}
/**
@@ -799,6 +1016,133 @@
return importContext;
}
+ private ImportContext getImportContext(EntryContainer entryContainer,
+ long bufferSize)
+ throws DatabaseException, JebException, ConfigException
+ {
+ DN baseDN = entryContainer.getBaseDN();
+ EntryContainer srcEntryContainer = null;
+ List<DN> includeBranches = new ArrayList<DN>();
+ List<DN> excludeBranches = new ArrayList<DN>();
+
+ if(!ldifImportConfig.appendToExistingData() &&
+ !ldifImportConfig.clearBackend())
+ {
+ for(DN dn : ldifImportConfig.getExcludeBranches())
+ {
+ if(baseDN.equals(dn))
+ {
+ // This entire base DN was explicitly excluded. Skip.
+ return null;
+ }
+ if(baseDN.isAncestorOf(dn))
+ {
+ excludeBranches.add(dn);
+ }
+ }
+
+ if(!ldifImportConfig.getIncludeBranches().isEmpty())
+ {
+ for(DN dn : ldifImportConfig.getIncludeBranches())
+ {
+ if(baseDN.isAncestorOf(dn))
+ {
+ includeBranches.add(dn);
+ }
+ }
+
+ if(includeBranches.isEmpty())
+ {
+ // There are no branches in the explicitly defined include list under
+ // this base DN. Skip this base DN alltogether.
+
+ return null;
+ }
+
+ // Remove any overlapping include branches.
+ for(DN includeDN : includeBranches)
+ {
+ boolean keep = true;
+ for(DN dn : includeBranches)
+ {
+ if(!dn.equals(includeDN) && dn.isAncestorOf(includeDN))
+ {
+ keep = false;
+ break;
+ }
+ }
+ if(!keep)
+ {
+ includeBranches.remove(includeDN);
+ }
+ }
+
+ // Remvoe any exclude branches that are not are not under a include
+ // branch since they will be migrated as part of the existing entries
+ // outside of the include branches anyways.
+ for(DN excludeDN : excludeBranches)
+ {
+ boolean keep = false;
+ for(DN includeDN : includeBranches)
+ {
+ if(includeDN.isAncestorOf(excludeDN))
+ {
+ keep = true;
+ break;
+ }
+ }
+ if(!keep)
+ {
+ excludeBranches.remove(excludeDN);
+ }
+ }
+
+ if(includeBranches.size() == 1 && excludeBranches.size() == 0 &&
+ includeBranches.get(0).equals(baseDN))
+ {
+ // This entire base DN is explicitly included in the import with
+ // no exclude branches that we need to migrate. Just clear the entry
+ // container.
+ entryContainer.exclusiveLock.lock();
+ entryContainer.clear();
+ entryContainer.exclusiveLock.unlock();
+ }
+ else
+ {
+ // Create a temp entry container
+ srcEntryContainer = entryContainer;
+ entryContainer =
+ rootContainer.openEntryContainer(baseDN,
+ baseDN.toNormalizedString() +
+ "_importTmp");
+ }
+ }
+ }
+
+ // Create an import context.
+ ImportContext importContext = new ImportContext();
+ importContext.setBufferSize(bufferSize);
+ importContext.setConfig(config);
+ importContext.setLDIFImportConfig(this.ldifImportConfig);
+ importContext.setLDIFReader(reader);
+
+ importContext.setBaseDN(baseDN);
+ importContext.setEntryContainer(entryContainer);
+ importContext.setSrcEntryContainer(srcEntryContainer);
+ importContext.setBufferSize(bufferSize);
+
+ // Create an entry queue.
+ LinkedBlockingQueue<Entry> queue =
+ new LinkedBlockingQueue<Entry>(config.getBackendImportQueueSize());
+ importContext.setQueue(queue);
+
+ // Set the include and exclude branches
+ importContext.setIncludeBranches(includeBranches);
+ importContext.setExcludeBranches(excludeBranches);
+
+ return importContext;
+ }
+
/**
* This class reports progress of the import job at fixed intervals.
*/
@@ -841,7 +1185,7 @@
*/
public void run()
{
- long latestCount = reader.getEntriesRead();
+ long latestCount = reader.getEntriesRead() + migratedCount;
long deltaCount = (latestCount - previousCount);
long latestTime = System.currentTimeMillis();
long deltaTime = latestTime - previousTime;
@@ -858,7 +1202,7 @@
int msgID = MSGID_JEB_IMPORT_PROGRESS_REPORT;
String message = getMessage(msgID, numRead, numIgnored, numRejected,
- rate);
+ migratedCount, rate);
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.NOTICE,
message, msgID);
diff --git a/opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java b/opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java
index 53ac583..13148ec 100644
--- a/opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java
+++ b/opends/src/server/org/opends/server/backends/jeb/IndexMergeThread.java
@@ -196,6 +196,12 @@
String message = getMessage(msgID, index.getName());
TRACER.debugInfo(message);
}
+
+ if(!ldifImportConfig.appendToExistingData())
+ {
+ index.setTrusted(null, true);
+ }
+
return;
}
@@ -307,7 +313,7 @@
{
}
- if(replaceExisting)
+ if(!ldifImportConfig.appendToExistingData())
{
index.setTrusted(txn, true);
}
diff --git a/opends/src/server/org/opends/server/backends/jeb/IndexRebuildThread.java b/opends/src/server/org/opends/server/backends/jeb/IndexRebuildThread.java
index 02bbcbc..2937a90 100644
--- a/opends/src/server/org/opends/server/backends/jeb/IndexRebuildThread.java
+++ b/opends/src/server/org/opends/server/backends/jeb/IndexRebuildThread.java
@@ -76,11 +76,6 @@
Index index = null;
/**
- * Name of the indexType being rebuilt.
- */
- String indexName = null;
-
- /**
* The ID2ENTRY database.
*/
ID2Entry id2entry = null;
@@ -128,12 +123,11 @@
*/
IndexRebuildThread(EntryContainer ec, IndexType index)
{
- super("Index Rebuild Thread " + ec.getContainerName() + "_" +
+ super("Index Rebuild Thread " + ec.getDatabasePrefix() + "_" +
index.toString());
this.ec = ec;
this.indexType = index;
this.id2entry = ec.getID2Entry();
- this.indexName = ec.getContainerName() + "_" + index.toString();
}
/**
@@ -149,7 +143,6 @@
this.indexType = IndexType.INDEX;
this.index = index;
this.id2entry = ec.getID2Entry();
- this.indexName = ec.getContainerName() + "_" + index.toString();
}
/**
@@ -165,7 +158,6 @@
this.indexType = IndexType.ATTRIBUTEINDEX;
this.attrIndex = index;
this.id2entry = ec.getID2Entry();
- this.indexName = ec.getContainerName() + "_" + index.toString();
}
/**
@@ -213,25 +205,25 @@
switch(indexType)
{
case DN2ID :
- ec.clearDatabase(null, ec.getDN2ID());
+ ec.clearDatabase(ec.getDN2ID());
break;
case DN2URI :
- ec.clearDatabase(null, ec.getDN2URI());
+ ec.clearDatabase(ec.getDN2URI());
break;
case ID2CHILDREN :
- ec.clearDatabase(null, ec.getID2Children());
+ ec.clearDatabase(ec.getID2Children());
ec.getID2Children().setRebuildStatus(true);
break;
case ID2SUBTREE :
- ec.clearDatabase(null, ec.getID2Subtree());
+ ec.clearDatabase(ec.getID2Subtree());
ec.getID2Subtree().setRebuildStatus(true);
break;
case ATTRIBUTEINDEX :
- ec.clearAttributeIndex(null, attrIndex);
+ ec.clearAttributeIndex(attrIndex);
attrIndex.setRebuildStatus(true);
break;
case INDEX :
- ec.clearDatabase(null, index);
+ ec.clearDatabase(index);
index.setRebuildStatus(true);
}
}
@@ -278,12 +270,6 @@
try
{
totalEntries = getTotalEntries();
- if(debugEnabled())
- {
- TRACER.debugInfo("Initiating rebuild of the %s indexType/database",
- indexName);
- TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
- }
switch(indexType)
{
@@ -329,6 +315,14 @@
{
DN2ID dn2id = ec.getDN2ID();
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Initiating rebuild of the %s database",
+ dn2id.getName());
+ TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
+ }
+
+
//Iterate through the id2entry database and insert associated dn2id
//records.
Cursor cursor = id2entry.openCursor(null, null);
@@ -379,7 +373,7 @@
skippedEntries++;
int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED;
- String message = getMessage(msgID, indexName,
+ String message = getMessage(msgID, dn2id.getName(),
stackTraceToSingleLineString(e));
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR,
message, msgID);
@@ -406,6 +400,14 @@
{
DN2URI dn2uri = ec.getDN2URI();
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Initiating rebuild of the %s database",
+ dn2uri.getName());
+ TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
+ }
+
+
//Iterate through the id2entry database and insert associated dn2uri
//records.
Cursor cursor = id2entry.openCursor(null, null);
@@ -457,7 +459,7 @@
skippedEntries++;
int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED;
- String message = getMessage(msgID, indexName,
+ String message = getMessage(msgID, dn2uri.getName(),
stackTraceToSingleLineString(e));
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR,
message, msgID);
@@ -485,6 +487,14 @@
{
Index id2children = ec.getID2Children();
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Initiating rebuild of the %s index",
+ id2children.getName());
+ TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
+ }
+
+
DN2ID dn2id = ec.getDN2ID();
DN2URI dn2uri = ec.getDN2URI();
@@ -559,7 +569,7 @@
skippedEntries++;
int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED;
- String message = getMessage(msgID, indexName,
+ String message = getMessage(msgID, id2children.getName(),
stackTraceToSingleLineString(e));
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR,
message, msgID);
@@ -589,6 +599,14 @@
{
Index id2subtree = ec.getID2Subtree();
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Initiating rebuild of the %s index",
+ id2subtree.getName());
+ TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
+ }
+
+
DN2ID dn2id = ec.getDN2ID();
DN2URI dn2uri = ec.getDN2URI();
@@ -693,7 +711,7 @@
skippedEntries++;
int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED;
- String message = getMessage(msgID, indexName,
+ String message = getMessage(msgID, id2subtree.getName(),
stackTraceToSingleLineString(e));
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR,
message, msgID);
@@ -722,6 +740,12 @@
private void rebuildAttributeIndex(AttributeIndex index)
throws DatabaseException
{
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Initiating rebuild of the %s index",
+ index.getName());
+ TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
+ }
//Iterate through the id2entry database and insert associated indexType
//records.
@@ -770,7 +794,7 @@
skippedEntries++;
int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED;
- String message = getMessage(msgID, indexName,
+ String message = getMessage(msgID, index.getName(),
stackTraceToSingleLineString(e));
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR,
message, msgID);
@@ -799,6 +823,12 @@
private void rebuildAttributeIndex(Index index)
throws DatabaseException
{
+ if(debugEnabled())
+ {
+ TRACER.debugInfo("Initiating rebuild of the %s attribute index",
+ index.getName());
+ TRACER.debugVerbose("%d entries will be rebuilt", totalEntries);
+ }
//Iterate through the id2entry database and insert associated indexType
//records.
@@ -847,7 +877,7 @@
skippedEntries++;
int msgID = MSGID_JEB_REBUILD_INSERT_ENTRY_FAILED;
- String message = getMessage(msgID, indexName,
+ String message = getMessage(msgID, index.getName(),
stackTraceToSingleLineString(e));
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.MILD_ERROR,
message, msgID);
diff --git a/opends/src/server/org/opends/server/backends/jeb/RootContainer.java b/opends/src/server/org/opends/server/backends/jeb/RootContainer.java
index 0587ade..c0c8a6a 100644
--- a/opends/src/server/org/opends/server/backends/jeb/RootContainer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/RootContainer.java
@@ -223,7 +223,7 @@
TRACER.debugInfo("Free memory in heap: %d bytes", heapFreeSize);
}
- openEntryContainers(config.getBackendBaseDN());
+ openAndRegisterEntryContainers(config.getBackendBaseDN());
}
/**
@@ -235,28 +235,55 @@
* transactional.
*
* @param baseDN The base DN of the entry container to open.
+ * @param name The name of the entry container or <CODE>NULL</CODE> to open
+ * the default entry container for the given base DN.
* @return The opened entry container.
* @throws DatabaseException If an error occurs while opening the entry
* container.
* @throws ConfigException If an configuration error occurs while opening
* the entry container.
*/
- public EntryContainer openEntryContainer(DN baseDN)
+ public EntryContainer openEntryContainer(DN baseDN, String name)
throws DatabaseException, ConfigException
{
- EntryContainer ec = new EntryContainer(baseDN, backend, config, env, this);
+ String databasePrefix;
+ if(name == null || name.equals(""))
+ {
+ databasePrefix = baseDN.toNormalizedString();
+ }
+ else
+ {
+ databasePrefix = name;
+ }
+
+ EntryContainer ec = new EntryContainer(baseDN, databasePrefix,
+ backend, config, env, this);
+ ec.open();
+ return ec;
+ }
+
+ /**
+ * Registeres the entry container for a base DN.
+ *
+ * @param baseDN The base DN of the entry container to close.
+ * @param entryContainer The entry container to register for the baseDN.
+ * @throws DatabaseException If an error occurs while opening the entry
+ * container.
+ */
+ public void registerEntryContainer(DN baseDN,
+ EntryContainer entryContainer)
+ throws DatabaseException
+ {
EntryContainer ec1=this.entryContainers.get(baseDN);
//If an entry container for this baseDN is already open we don't allow
//another to be opened.
if (ec1 != null)
- throw new DatabaseException("Entry container for baseDN " +
- baseDN.toString() + " already is open.");
+ throw new DatabaseException("An entry container named " +
+ ec1.getDatabasePrefix() + " is alreadly registered for base DN " +
+ baseDN.toString());
- ec.open();
- this.entryContainers.put(baseDN, ec);
-
- return ec;
+ this.entryContainers.put(baseDN, entryContainer);
}
/**
@@ -268,15 +295,16 @@
* @throws ConfigException if a configuration error occurs while opening the
* container.
*/
- private void openEntryContainers(Set<DN> baseDNs)
+ private void openAndRegisterEntryContainers(Set<DN> baseDNs)
throws DatabaseException, ConfigException
{
EntryID id;
EntryID highestID = null;
for(DN baseDN : baseDNs)
{
- EntryContainer ec = openEntryContainer(baseDN);
+ EntryContainer ec = openEntryContainer(baseDN, null);
id = ec.getHighestEntryID();
+ registerEntryContainer(baseDN, ec);
if(highestID == null || id.compareTo(highestID) > 0)
{
highestID = id;
@@ -287,48 +315,15 @@
}
/**
- * Close the entry container for a base DN.
+ * Unregisteres the entry container for a base DN.
*
* @param baseDN The base DN of the entry container to close.
- * @throws DatabaseException If an error occurs while closing the entry
- * container.
+ * @return The entry container that was unregistered or NULL if a entry
+ * container for the base DN was not registered.
*/
- public void closeEntryContainer(DN baseDN) throws DatabaseException
+ public EntryContainer unregisterEntryContainer(DN baseDN)
{
- EntryContainer ec = getEntryContainer(baseDN);
- ec.exclusiveLock.lock();
- try
- {
- ec.close();
- entryContainers.remove(baseDN);
- }
- finally
- {
- ec.exclusiveLock.unlock();
- }
- }
-
- /**
- * Close and remove a entry container for a base DN from disk.
- *
- * @param baseDN The base DN of the entry container to remove.
- * @throws DatabaseException If an error occurs while removing the entry
- * container.
- */
- public void removeEntryContainer(DN baseDN) throws DatabaseException
- {
- EntryContainer ec = getEntryContainer(baseDN);
- ec.exclusiveLock.lock();
- try
- {
- ec.close();
- ec.removeContainer();
- entryContainers.remove(baseDN);
- }
- finally
- {
- ec.exclusiveLock.unlock();
- }
+ return entryContainers.remove(baseDN);
}
@@ -499,7 +494,16 @@
{
for(DN baseDN : entryContainers.keySet())
{
- closeEntryContainer(baseDN);
+ EntryContainer ec = unregisterEntryContainer(baseDN);
+ ec.exclusiveLock.lock();
+ try
+ {
+ ec.close();
+ }
+ finally
+ {
+ ec.exclusiveLock.unlock();
+ }
}
if (env != null)
diff --git a/opends/src/server/org/opends/server/backends/jeb/State.java b/opends/src/server/org/opends/server/backends/jeb/State.java
index 12ae594..fb181b7 100644
--- a/opends/src/server/org/opends/server/backends/jeb/State.java
+++ b/opends/src/server/org/opends/server/backends/jeb/State.java
@@ -117,8 +117,10 @@
public boolean getIndexTrustState(Transaction txn, Index index)
throws DatabaseException
{
+ String sortName =
+ index.getName().replace(entryContainer.getDatabasePrefix(), "");
DatabaseEntry key =
- new DatabaseEntry(StaticUtils.getBytes(index.getName()));
+ new DatabaseEntry(StaticUtils.getBytes(sortName));
DatabaseEntry data = new DatabaseEntry();
OperationStatus status;
@@ -145,8 +147,10 @@
boolean trusted)
throws DatabaseException
{
+ String sortName =
+ index.getName().replace(entryContainer.getDatabasePrefix(), "");
DatabaseEntry key =
- new DatabaseEntry(StaticUtils.getBytes(index.getName()));
+ new DatabaseEntry(StaticUtils.getBytes(sortName));
DatabaseEntry data = new DatabaseEntry();
if(trusted)
@@ -162,4 +166,6 @@
}
return true;
}
+
+ // TODO: Make sure to update the VLV state access methods to use shortname.
}
diff --git a/opends/src/server/org/opends/server/config/ConfigConstants.java b/opends/src/server/org/opends/server/config/ConfigConstants.java
index 818e728..09b7bdd 100644
--- a/opends/src/server/org/opends/server/config/ConfigConstants.java
+++ b/opends/src/server/org/opends/server/config/ConfigConstants.java
@@ -4159,5 +4159,12 @@
*/
public static final String ATTR_REBUILD_MAX_THREADS =
NAME_PREFIX_TASK + "rebuild-max-threads";
+
+ /**
+ * The name of the attribute in an import task definition that specifies
+ * whether the backend should be cleared before the import.
+ */
+ public static final String ATTR_IMPORT_CLEAR_BACKEND =
+ NAME_PREFIX_TASK + "import-clear-backend";
}
diff --git a/opends/src/server/org/opends/server/messages/JebMessages.java b/opends/src/server/org/opends/server/messages/JebMessages.java
index b10809a..75509b7 100644
--- a/opends/src/server/org/opends/server/messages/JebMessages.java
+++ b/opends/src/server/org/opends/server/messages/JebMessages.java
@@ -1215,6 +1215,28 @@
public static final int MSGID_JEB_INVALID_LOGGING_LEVEL =
CATEGORY_MASK_JEB | SEVERITY_MASK_SEVERE_ERROR | 156;
+ /**
+ * The message ID of the message indicating the migration stage has started
+ * during an import process. This takes two arguments, which are the type
+ * of entries being migrated and the base DN of the entry container they
+ * are from.
+ */
+ public static final int MSGID_JEB_IMPORT_MIGRATION_START =
+ CATEGORY_MASK_JEB | SEVERITY_MASK_INFORMATIONAL | 157;
+
+ /**
+ * The message ID of the message indicating the LDIF reading stage has
+ * started during an import process.
+ */
+ public static final int MSGID_JEB_IMPORT_LDIF_START =
+ CATEGORY_MASK_JEB | SEVERITY_MASK_INFORMATIONAL | 158;
+
+ /**
+ * The message ID of the message indicating the LDIF reading stage has
+ * ended during an import process.
+ */
+ public static final int MSGID_JEB_IMPORT_LDIF_END =
+ CATEGORY_MASK_JEB | SEVERITY_MASK_INFORMATIONAL | 159;
/**
* Associates a set of generic messages with the message IDs defined in this
@@ -1426,7 +1448,7 @@
"Exported %d records and skipped %d " +
"(recent rate %.1f/sec)");
registerMessage(MSGID_JEB_IMPORT_THREAD_COUNT,
- "Starting LDIF import (using %d threads)");
+ "Starting import (using %d threads)");
registerMessage(MSGID_JEB_IMPORT_BUFFER_SIZE,
"Buffer size per thread = %,d");
registerMessage(MSGID_JEB_IMPORT_LDIF_PROCESSING_TIME,
@@ -1434,10 +1456,10 @@
registerMessage(MSGID_JEB_IMPORT_INDEX_PROCESSING_TIME,
"Index processing took %d seconds");
registerMessage(MSGID_JEB_IMPORT_BEGINNING_INTERMEDIATE_MERGE,
- "Ending LDIF import pass %d because the pass size has " +
+ "Ending import pass %d because the pass size has " +
"been reached. Beginning the intermediate index merge");
registerMessage(MSGID_JEB_IMPORT_BEGINNING_FINAL_MERGE,
- "End of LDIF reached. Beginning final index merge");
+ "Beginning final index merge");
registerMessage(MSGID_JEB_IMPORT_RESUMING_LDIF_PROCESSING,
"Intermediate index merge processing complete (index " +
"processing time %d seconds). Resuming LDIF processing");
@@ -1446,13 +1468,14 @@
registerMessage(MSGID_JEB_IMPORT_CLOSING_DATABASE,
"Flushing data to disk");
registerMessage(MSGID_JEB_IMPORT_FINAL_STATUS,
- "Processed %d entries, imported %d, skipped %d, and " +
- "rejected %d in %d seconds (average rate %.1f/sec)");
+ "Processed %d entries, imported %d, skipped %d, " +
+ "rejected %d and migrated %d in %d seconds " +
+ "(average rate %.1f/sec)");
registerMessage(MSGID_JEB_IMPORT_ENTRY_LIMIT_EXCEEDED_COUNT,
"Number of index values that exceeded the entry limit: %d");
registerMessage(MSGID_JEB_IMPORT_PROGRESS_REPORT,
- "Processed %d entries, skipped %d, and rejected %d " +
- "(recent rate %.1f/sec)");
+ "Processed %d entries, skipped %d, rejected %d, and " +
+ "migrated %d (recent rate %.1f/sec)");
registerMessage(MSGID_JEB_IMPORT_CACHE_AND_MEMORY_REPORT,
"Free memory = %d MB, Cache miss rate = %.1f/entry");
registerMessage(MSGID_JEB_INDEX_MERGE_NO_DATA,
@@ -1575,5 +1598,11 @@
"imported entries");
registerMessage(MSGID_JEB_IMPORT_CREATE_TMPDIR_ERROR,
"Unable to create the temporary directory %s");
+ registerMessage(MSGID_JEB_IMPORT_MIGRATION_START,
+ "Migrating %s entries for base DN %s");
+ registerMessage(MSGID_JEB_IMPORT_LDIF_START,
+ "Processing LDIF");
+ registerMessage(MSGID_JEB_IMPORT_LDIF_END,
+ "End of LDIF reached");
}
}
diff --git a/opends/src/server/org/opends/server/messages/ToolMessages.java b/opends/src/server/org/opends/server/messages/ToolMessages.java
index ce41e5e..aa7f8c1 100644
--- a/opends/src/server/org/opends/server/messages/ToolMessages.java
+++ b/opends/src/server/org/opends/server/messages/ToolMessages.java
@@ -9240,6 +9240,7 @@
public static final int MSGID_DSCFG_ERROR_MISSING_NON_INTERACTIVE_ARG =
CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1223;
+
/**
* The message ID for the message that will be used if dsconfig cannot
* read input from the console. This takes a single argument, which
@@ -9378,6 +9379,32 @@
CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1239;
/**
+ * The message ID for the message that will be used as the description of the
+ * clearBackend argument. This does not take any arguments.
+ */
+ public static final int MSGID_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND =
+ CATEGORY_MASK_TOOLS | SEVERITY_MASK_INFORMATIONAL | 1240;
+
+ /**
+ * The message ID for the message that will be used if neither the
+ * includeBranchStrings or the backendID arguments was provided.
+ * This takes two arguments, which are the long identifiers for the
+ * includeBranchStrings and backendID options.
+ */
+ public static final int MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT =
+ CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1241;
+
+
+ /**
+ * The message ID for the message that will be used if the clearBackend
+ * argument was not provided when the backendID argument was provided without
+ * an append argument. This takes the long identifier of the clearBackend
+ * option as an argument.
+ */
+ public static final int MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND =
+ CATEGORY_MASK_TOOLS | SEVERITY_MASK_SEVERE_ERROR | 1242;
+
+ /**
* Associates a set of generic messages with the message IDs defined in this
* class.
*/
@@ -9664,11 +9691,12 @@
"Unable to decode include filter string \"%s\" as a " +
"valid search filter: %s");
registerMessage(MSGID_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID,
- "Multiple Directory Server backends are configured with " +
- "backend ID \"%s\"");
+ "Imported branches or backend IDs can not span across " +
+ "multiple Directory Server backends");
registerMessage(MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID,
"None of the Directory Server backends are configured " +
- "with the requested backend ID \"%s\"");
+ "with the requested backend ID or base DNs that include " +
+ "the specified branches");
registerMessage(MSGID_LDIFIMPORT_CANNOT_IMPORT,
"The Directory Server backend with backend ID %s does " +
"not provide a mechanism for performing LDIF imports");
@@ -12404,6 +12432,18 @@
"Invalid response. Please enter a value between 1 and %s");
registerMessage(MSGID_DSCFG_ERROR_GENERAL_CONFIRM,
"Invalid response. Please enter \"%s\" or \"%s\"");
+ registerMessage(MSGID_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND,
+ "Remove all entries for all base DNs in the backend " +
+ "before importing");
+ registerMessage(MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT,
+ "Neither the %s or the %s argument was provided. One " +
+ "of these arguments must be given to specify the backend " +
+ "for the LDIF data to be imported to");
+ registerMessage(MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND,
+ "Importing to a backend without the append argument will " +
+ "remove all entries for all base DNs (%s) in the " +
+ "backend. The %s argument must be given to continue with " +
+ "import");
}
}
diff --git a/opends/src/server/org/opends/server/tasks/ImportTask.java b/opends/src/server/org/opends/server/tasks/ImportTask.java
index 217677b..a0b2901 100644
--- a/opends/src/server/org/opends/server/tasks/ImportTask.java
+++ b/opends/src/server/org/opends/server/tasks/ImportTask.java
@@ -80,6 +80,7 @@
boolean overwrite = false;
boolean replaceExisting = false;
boolean skipSchemaValidation = false;
+ boolean clearBackend = false;
String backendID = null;
String rejectFile = null;
String skipFile = null;
@@ -132,6 +133,7 @@
AttributeType typeSkipSchemaValidation;
AttributeType typeIsCompressed;
AttributeType typeIsEncrypted;
+ AttributeType typeClearBackend;
typeLdifFile =
getAttributeType(ATTR_IMPORT_LDIF_FILE, true);
@@ -165,6 +167,8 @@
getAttributeType(ATTR_IMPORT_IS_COMPRESSED, true);
typeIsEncrypted =
getAttributeType(ATTR_IMPORT_IS_ENCRYPTED, true);
+ typeClearBackend =
+ getAttributeType(ATTR_IMPORT_CLEAR_BACKEND, true);
List<Attribute> attrList;
@@ -216,6 +220,113 @@
attrList = taskEntry.getAttribute(typeIsEncrypted);
isEncrypted = TaskUtils.getBoolean(attrList, false);
+ attrList = taskEntry.getAttribute(typeClearBackend);
+ clearBackend = TaskUtils.getBoolean(attrList, false);
+
+ // Make sure that either the "includeBranchStrings" argument or the
+ // "backendID" argument was provided, but not both.
+ if(!includeBranchStrings.isEmpty())
+ {
+ if(backendID != null)
+ {
+ int msgID = MSGID_LDIFIMPORT_CONFLICTING_OPTIONS;
+ String message = getMessage(msgID, typeIncludeBranch.getNameOrOID(),
+ typeBackendID.getNameOrOID());
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+ }
+ else if(backendID == null)
+ {
+ int msgID = MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT;
+ String message = getMessage(msgID, typeIncludeBranch.getNameOrOID(),
+ typeBackendID.getNameOrOID());
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+
+ if(backendID != null)
+ {
+ Backend backend = DirectoryServer.getBackend(backendID);
+ if (backend == null)
+ {
+ int msgID = MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID;
+ String message = getMessage(msgID, backendID);
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+ else if (! backend.supportsLDIFImport())
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_IMPORT;
+ String message = getMessage(msgID, backendID);
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+ // Make sure that if the "backendID" argument was provided and the
+ // "append" option was not provided, the "clearBackend" argument was also
+ // provided if there are more then one baseDNs for the backend being
+ // imported.
+ else if(!append && backend.getBaseDNs().length > 1 && !clearBackend)
+ {
+ StringBuilder builder = new StringBuilder();
+ for(DN dn : backend.getBaseDNs())
+ {
+ builder.append(dn.toNormalizedString());
+ builder.append(" ");
+ }
+ int msgID = MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND;
+ String message = getMessage(msgID, builder.toString(),
+ typeClearBackend.getNameOrOID());
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+ }
+ else
+ {
+ // Find the backend that includes all the branches.
+ Backend backend = null;
+ for (String s : includeBranchStrings)
+ {
+ DN includeBranch;
+ try
+ {
+ includeBranch = DN.decode(s);
+ }
+ catch (DirectoryException de)
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
+ String message = getMessage(msgID, s, de.getErrorMessage());
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+ catch (Exception e)
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
+ String message = getMessage(msgID, s, getExceptionMessage(e));
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, message,
+ msgID);
+ }
+
+ Backend locatedBackend = DirectoryServer.getBackend(includeBranch);
+ if(locatedBackend != null)
+ {
+ if(backend == null)
+ {
+ backend = locatedBackend;
+ }
+ else if(backend != locatedBackend)
+ {
+ // The include branches span across multiple backends.
+ int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE;
+ String message = getMessage(msgID, s, backend.getBackendID());
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM,
+ message, msgID);
+ }
+ }
+ }
+ }
}
@@ -295,27 +406,106 @@
// Get the backend into which the LDIF should be imported.
- Backend backend;
+ Backend backend = null;
ArrayList<DN> defaultIncludeBranches;
ArrayList<DN> excludeBranches = new ArrayList<DN>();
+ ArrayList<DN> includeBranches = new ArrayList<DN>();
- backend = DirectoryServer.getBackend(backendID);
- if (backend == null)
+ includeBranches = new ArrayList<DN>(includeBranchStrings.size());
+ for (String s : includeBranchStrings)
{
- int msgID = MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID;
- String message = getMessage(msgID, backendID);
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message,
- msgID);
- return TaskState.STOPPED_BY_ERROR;
+ DN includeBranch;
+ try
+ {
+ includeBranch = DN.decode(s);
+ }
+ catch (DirectoryException de)
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
+ String message = getMessage(msgID, s, de.getErrorMessage());
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return TaskState.STOPPED_BY_ERROR;
+ }
+ catch (Exception e)
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
+ String message = getMessage(msgID, s, getExceptionMessage(e));
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return TaskState.STOPPED_BY_ERROR;
+ }
+
+ includeBranches.add(includeBranch);
}
- else if (! backend.supportsLDIFImport())
+
+ if(backendID != null)
{
- int msgID = MSGID_LDIFIMPORT_CANNOT_IMPORT;
- String message = getMessage(msgID, backendID);
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR, message,
- msgID);
- return TaskState.STOPPED_BY_ERROR;
+ backend = DirectoryServer.getBackend(backendID);
+
+ if (backend == null)
+ {
+ int msgID = MSGID_LDIFIMPORT_NO_BACKENDS_FOR_ID;
+ String message = getMessage(msgID, backendID);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return TaskState.STOPPED_BY_ERROR;
+ }
+ else if (! backend.supportsLDIFImport())
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_IMPORT;
+ String message = getMessage(msgID, backendID);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return TaskState.STOPPED_BY_ERROR;
+ }
+ // Make sure that if the "backendID" argument was provided and the
+ // "append" option was not provided, the "clearBackend" argument was also
+ // provided if there are more then one baseDNs for the backend being
+ // imported.
+ else if(!append && backend.getBaseDNs().length > 1 && !clearBackend)
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append(backend.getBaseDNs()[0].toNormalizedString());
+ for(int i = 1; i < backend.getBaseDNs().length; i++)
+ {
+ builder.append(" / ");
+ builder.append(backend.getBaseDNs()[i].toNormalizedString());
+ }
+ int msgID = MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND;
+ String message = getMessage(msgID, builder.toString(),
+ ATTR_IMPORT_CLEAR_BACKEND);
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return TaskState.STOPPED_BY_ERROR;
+ }
+ }
+ else
+ {
+ // Find the backend that includes all the branches.
+ for(DN includeBranch : includeBranches)
+ {
+ Backend locatedBackend = DirectoryServer.getBackend(includeBranch);
+ if(locatedBackend != null)
+ {
+ if(backend == null)
+ {
+ backend = locatedBackend;
+ }
+ else if(backend != locatedBackend)
+ {
+ // The include branches span across multiple backends.
+ int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE;
+ String message = getMessage(msgID,
+ includeBranch.toNormalizedString(),
+ backend.getBackendID());
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return TaskState.STOPPED_BY_ERROR;
+ }
+ }
+ }
}
// Find backends with subordinate base DNs that should be excluded from the
@@ -380,53 +570,10 @@
}
}
-
- ArrayList<DN> includeBranches;
if (includeBranchStrings.isEmpty())
{
includeBranches = defaultIncludeBranches;
}
- else
- {
- includeBranches = new ArrayList<DN>(includeBranchStrings.size());
- for (String s : includeBranchStrings)
- {
- DN includeBranch;
- try
- {
- includeBranch = DN.decode(s);
- }
- catch (DirectoryException de)
- {
- int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
- String message = getMessage(msgID, s, de.getErrorMessage());
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
- message, msgID);
- return TaskState.STOPPED_BY_ERROR;
- }
- catch (Exception e)
- {
- int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
- String message = getMessage(msgID, s, getExceptionMessage(e));
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
- message, msgID);
- return TaskState.STOPPED_BY_ERROR;
- }
-
- if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches,
- excludeBranches))
- {
- int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE;
- String message = getMessage(msgID, s, backendID);
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
- message, msgID);
- return TaskState.STOPPED_BY_ERROR;
- }
-
- includeBranches.add(includeBranch);
- }
- }
-
// Create the LDIF import configuration to use when reading the LDIF.
ArrayList<String> fileList = new ArrayList<String>(ldifFiles);
@@ -435,6 +582,7 @@
importConfig.setReplaceExistingEntries(replaceExisting);
importConfig.setCompressed(isCompressed);
importConfig.setEncrypted(isEncrypted);
+ importConfig.setClearBackend(clearBackend);
importConfig.setExcludeAttributes(excludeAttributes);
importConfig.setExcludeBranches(excludeBranches);
importConfig.setExcludeFilters(excludeFilters);
@@ -511,7 +659,7 @@
// Disable the backend.
try
{
- TaskUtils.disableBackend(backendID);
+ TaskUtils.disableBackend(backend.getBackendID());
}
catch (DirectoryException e)
{
@@ -631,11 +779,11 @@
// Enable the backend.
try
{
- TaskUtils.enableBackend(backendID);
+ TaskUtils.enableBackend(backend.getBackendID());
// It is necessary to retrieve the backend structure again
// because disabling and enabling it again may have resulted
// in a new backend being registered to the server.
- backend = DirectoryServer.getBackend(backendID);
+ backend = DirectoryServer.getBackend(backend.getBackendID());
}
catch (DirectoryException e)
{
diff --git a/opends/src/server/org/opends/server/tools/ImportLDIF.java b/opends/src/server/org/opends/server/tools/ImportLDIF.java
index 164b998..8f3b9dd 100644
--- a/opends/src/server/org/opends/server/tools/ImportLDIF.java
+++ b/opends/src/server/org/opends/server/tools/ImportLDIF.java
@@ -172,6 +172,7 @@
BooleanArgument quietMode = null;
BooleanArgument replaceExisting = null;
BooleanArgument skipSchemaValidation = null;
+ BooleanArgument clearBackend = null;
IntegerArgument randomSeed = null;
StringArgument backendID = null;
StringArgument configClass = null;
@@ -245,11 +246,16 @@
backendID =
- new StringArgument("backendid", 'n', "backendID", true, false, true,
+ new StringArgument("backendid", 'n', "backendID", false, false, true,
"{backendID}", null, null,
MSGID_LDIFIMPORT_DESCRIPTION_BACKEND_ID);
argParser.addArgument(backendID);
+ clearBackend =
+ new BooleanArgument("clearbackend", 'F', "clearBackend",
+ MSGID_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND);
+ argParser.addArgument(clearBackend);
+
includeBranchStrings =
new StringArgument("includebranch", 'b', "includeBranch", false,
@@ -413,6 +419,29 @@
return 1;
}
+ // Make sure that either the "includeBranchStrings" argument or the
+ // "backendID" argument was provided, but not both.
+ if(includeBranchStrings.isPresent())
+ {
+ if(backendID.isPresent())
+ {
+ int msgID = MSGID_LDIFIMPORT_CONFLICTING_OPTIONS;
+ String message = getMessage(msgID,
+ includeBranchStrings.getLongIdentifier(),
+ backendID.getLongIdentifier());
+ err.println(wrapText(message, MAX_LINE_WIDTH));
+ return 1;
+ }
+ }
+ else if(! backendID.isPresent())
+ {
+ int msgID = MSGID_LDIFIMPORT_MISSING_BACKEND_ARGUMENT;
+ String message = getMessage(msgID,
+ includeBranchStrings.getLongIdentifier(),
+ backendID.getLongIdentifier());
+ err.println(wrapText(message, MAX_LINE_WIDTH));
+ return 1;
+ }
// Perform the initial bootstrap of the Directory Server and process the
// configuration.
@@ -766,6 +795,38 @@
Backend backend = null;
List<DN> defaultIncludeBranches = null;
List<DN> excludeBranches = new ArrayList<DN>();
+ List<DN> includeBranches = new ArrayList<DN>();
+
+ if (includeBranchStrings.isPresent())
+ {
+ includeBranches = new ArrayList<DN>();
+ for (String s : includeBranchStrings.getValues())
+ {
+ DN includeBranch;
+ try
+ {
+ includeBranch = DN.decode(s);
+ }
+ catch (DirectoryException de)
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
+ String message = getMessage(msgID, s, de.getErrorMessage());
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return 1;
+ }
+ catch (Exception e)
+ {
+ int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
+ String message = getMessage(msgID, s, getExceptionMessage(e));
+ logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
+ message, msgID);
+ return 1;
+ }
+
+ includeBranches.add(includeBranch);
+ }
+ }
ArrayList<Backend> backendList = new ArrayList<Backend>();
ArrayList<BackendCfg> entryList = new ArrayList<BackendCfg>();
@@ -780,9 +841,36 @@
for (int i=0; i < numBackends; i++)
{
Backend b = backendList.get(i);
- if (! backendID.getValue().equals(b.getBackendID()))
+
+ if(backendID.isPresent())
{
- continue;
+ if (! backendID.getValue().equals(b.getBackendID()))
+ {
+ continue;
+ }
+ }
+ else
+ {
+ boolean useBackend = false;
+ for(DN baseDN : dnList.get(i))
+ {
+ for(DN includeDN : includeBranches)
+ {
+ if(baseDN.isAncestorOf(includeDN))
+ {
+ useBackend = true;
+ break;
+ }
+ }
+ if(useBackend)
+ {
+ break;
+ }
+ }
+ if(!useBackend)
+ {
+ continue;
+ }
}
if (backend == null)
@@ -793,7 +881,7 @@
else
{
int msgID = MSGID_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID;
- String message = getMessage(msgID, backendID.getValue());
+ String message = getMessage(msgID);
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
message, msgID);
return 1;
@@ -837,6 +925,27 @@
}
}
+ // Make sure that if the "backendID" argument was provided and the "append"
+ // option was not provided, the "clearBackend" argument was also
+ // provided if there are more then one baseDNs for the backend being
+ // imported.
+ if(backendID.isPresent() && !append.isPresent() &&
+ defaultIncludeBranches.size() > 1 && !clearBackend.isPresent())
+ {
+ StringBuilder builder = new StringBuilder();
+ builder.append(backend.getBaseDNs()[0].toNormalizedString());
+ for(int i = 1; i < backend.getBaseDNs().length; i++)
+ {
+ builder.append(" / ");
+ builder.append(backend.getBaseDNs()[i].toNormalizedString());
+ }
+ int msgID = MSGID_LDIFIMPORT_MISSING_CLEAR_BACKEND;
+ String message = getMessage(msgID, builder.toString(),
+ clearBackend.getLongIdentifier());
+ err.println(wrapText(message, MAX_LINE_WIDTH));
+ return 1;
+ }
+
for (String s : excludeBranchStrings.getValues())
{
DN excludeBranch;
@@ -867,50 +976,25 @@
}
}
-
- List<DN> includeBranches;
if (! includeBranchStrings.isPresent())
{
includeBranches = defaultIncludeBranches;
}
else
{
- includeBranches = new ArrayList<DN>();
- for (String s : includeBranchStrings.getValues())
+ // Make sure the selected backend will handle all the include branches
+ for(DN includeBranch : includeBranches)
{
- DN includeBranch;
- try
- {
- includeBranch = DN.decode(s);
- }
- catch (DirectoryException de)
- {
- int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
- String message = getMessage(msgID, s, de.getErrorMessage());
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
- message, msgID);
- return 1;
- }
- catch (Exception e)
- {
- int msgID = MSGID_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE;
- String message = getMessage(msgID, s, getExceptionMessage(e));
- logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
- message, msgID);
- return 1;
- }
-
if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches,
excludeBranches))
{
int msgID = MSGID_LDIFIMPORT_INVALID_INCLUDE_BASE;
- String message = getMessage(msgID, s, backendID.getValue());
+ String message = getMessage(msgID, includeBranch.toNormalizedString(),
+ backendID.getValue());
logError(ErrorLogCategory.BACKEND, ErrorLogSeverity.SEVERE_ERROR,
message, msgID);
return 1;
}
-
- includeBranches.add(includeBranch);
}
}
@@ -983,6 +1067,7 @@
importConfig.setAppendToExistingData(append.isPresent());
importConfig.setReplaceExistingEntries(replaceExisting.isPresent());
importConfig.setCompressed(isCompressed.isPresent());
+ importConfig.setClearBackend(clearBackend.isPresent());
importConfig.setEncrypted(isEncrypted.isPresent());
importConfig.setExcludeAttributes(excludeAttributes);
importConfig.setExcludeBranches(excludeBranches);
diff --git a/opends/src/server/org/opends/server/types/LDIFImportConfig.java b/opends/src/server/org/opends/server/types/LDIFImportConfig.java
index a5976cd..cd72546 100644
--- a/opends/src/server/org/opends/server/types/LDIFImportConfig.java
+++ b/opends/src/server/org/opends/server/types/LDIFImportConfig.java
@@ -95,6 +95,9 @@
// Indicates whether the import is encrypted.
private boolean isEncrypted;
+ // Indicates whether to clear all base DNs in a backend.
+ private boolean clearBackend;
+
// Indicates whether to replace existing entries when appending
// data.
private boolean replaceExistingEntries;
@@ -184,6 +187,7 @@
invokeImportPlugins = false;
isCompressed = false;
isEncrypted = false;
+ clearBackend = false;
validateSchema = true;
reader = null;
rejectWriter = null;
@@ -795,6 +799,34 @@
/**
+ * Indicates whether to clear the entire backend if importing to a
+ * backend with more than one base DNs.
+ *
+ * @return <CODE>true</code> if the entire backend should be
+ * cleared or <CODE>false</CODE> if not.
+ */
+ public boolean clearBackend()
+ {
+ return clearBackend;
+ }
+
+
+
+ /**
+ * Specifies whether to clear the entire backend if importing to a
+ * backend.
+ *
+ * @param clearBackend Indicates whether to clear the entire
+ * backend.
+ */
+ public void setClearBackend(boolean clearBackend)
+ {
+ this.clearBackend = clearBackend;
+ }
+
+
+
+ /**
* Indicates whether to perform schema validation on entries as they
* are read.
*
diff --git a/opends/src/server/org/opends/server/util/StaticUtils.java b/opends/src/server/org/opends/server/util/StaticUtils.java
index 768eefb..e2ea4e5 100644
--- a/opends/src/server/org/opends/server/util/StaticUtils.java
+++ b/opends/src/server/org/opends/server/util/StaticUtils.java
@@ -1601,7 +1601,7 @@
buffer.append("(");
for (StackTraceElement e : t.getStackTrace())
{
- if (i > 4)
+ if (i > 20)
{
buffer.append(" ...");
break;
--
Gitblit v1.10.0