From 3e7410b2ec4ceeef110ed4864d86cd56993512fe Mon Sep 17 00:00:00 2001
From: dugan <dugan@localhost>
Date: Sat, 26 Sep 2009 18:33:49 +0000
Subject: [PATCH] Rebuild index using new import engine.
---
opends/resource/schema/02-config.ldif | 6
opends/src/server/org/opends/server/tasks/RebuildTask.java | 68 +-
opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java | 83 +-
opends/tests/unit-tests-testng/src/server/org/opends/server/tasks/TestRebuildTask.java | 9
opends/src/server/org/opends/server/backends/jeb/BackendImpl.java | 72 ++
opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java | 30
opends/src/server/org/opends/server/util/LDIFReader.java | 66 +-
opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java | 1138 ++++++++++++++++++++++++++++++++++++++---
opends/src/server/org/opends/server/tools/RebuildIndex.java | 50 +
opends/src/server/org/opends/server/backends/jeb/importLDIF/Suffix.java | 30
opends/src/server/org/opends/server/config/ConfigConstants.java | 11
11 files changed, 1,330 insertions(+), 233 deletions(-)
diff --git a/opends/resource/schema/02-config.ldif b/opends/resource/schema/02-config.ldif
index d88de33..d1010ec 100644
--- a/opends/resource/schema/02-config.ldif
+++ b/opends/resource/schema/02-config.ldif
@@ -1423,8 +1423,8 @@
SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.291
- NAME 'ds-task-rebuild-max-threads'
- SYNTAX 1.3.6.1.4.1.1466.115.121.1.27
+ NAME 'ds-task-rebuild-tmp-directory'
+ SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
SINGLE-VALUE
X-ORIGIN 'OpenDS Directory Server' )
attributeTypes: ( 1.3.6.1.4.1.26027.1.1.292
@@ -3398,7 +3398,7 @@
STRUCTURAL
MUST ( ds-task-rebuild-base-dn $
ds-task-rebuild-index )
- MAY ds-task-rebuild-max-threads
+ MAY ds-task-rebuild-tmp-directory
X-ORIGIN 'OpenDS Directory Server' )
objectClasses: ( 1.3.6.1.4.1.26027.1.2.98
NAME 'ds-virtual-static-group'
diff --git a/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java b/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
index 4408b56..67652d2 100644
--- a/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
+++ b/opends/src/server/org/opends/server/backends/jeb/BackendImpl.java
@@ -1136,8 +1136,7 @@
envConfig.setConfigParam(EnvironmentConfig.ENV_RUN_CLEANER, "false");
envConfig.setConfigParam(EnvironmentConfig.EVICTOR_LRU_ONLY, "false");
envConfig.setConfigParam(EnvironmentConfig.EVICTOR_NODES_PER_SCAN, "128");
- Importer importer = new Importer(importConfig, cfg);
- importer.initialize(envConfig);
+ Importer importer = Importer.getInstance(importConfig, cfg, envConfig);
rootContainer = initializeRootContainer(envConfig);
return importer.processImport(rootContainer);
}
@@ -1337,35 +1336,73 @@
* @throws DirectoryException If a Directory Server error occurs.
*/
public void rebuildBackend(RebuildConfig rebuildConfig)
- throws InitializationException, ConfigException, DirectoryException
+ throws InitializationException, ConfigException, DirectoryException
{
// If the backend already has the root container open, we must use the same
// underlying root container
boolean openRootContainer = rootContainer == null;
- // If the rootContainer is open, the backend is initialized by something
- // else.
- // We can't do any rebuild of system indexes while others are using this
- // backend. Throw error. TODO: Need to make baseDNs disablable.
+ /*
+ If the rootContainer is open, the backend is initialized by something
+ else. We can't do any rebuild of system indexes while others are using
+ this backend.
+ */
if(!openRootContainer && rebuildConfig.includesSystemIndex())
{
Message message = ERR_JEB_REBUILD_BACKEND_ONLINE.get();
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
- message);
+ message);
}
-
+ Importer importer;
try
{
+ EnvironmentConfig envConfig =
+ ConfigurableEnvironment.parseConfigEntry(cfg);
+ importer = Importer.getInstance(rebuildConfig, cfg, envConfig);
if (openRootContainer)
{
- EnvironmentConfig envConfig =
- ConfigurableEnvironment.parseConfigEntry(cfg);
-
rootContainer = initializeRootContainer(envConfig);
}
-
- RebuildJob rebuildJob = new RebuildJob(rebuildConfig);
- rebuildJob.rebuildBackend(rootContainer);
+ importer.rebuildIndexes(rootContainer);
+ }
+ catch (ExecutionException execEx)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, execEx);
+ }
+ Message message = ERR_EXECUTION_ERROR.get(execEx.getMessage());
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message);
+ }
+ catch (InterruptedException intEx)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, intEx);
+ }
+ Message message = ERR_INTERRUPTED_ERROR.get(intEx.getMessage());
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message);
+ }
+ catch (IOException ioe)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, ioe);
+ }
+ Message message = ERR_JEB_IO_ERROR.get(ioe.getMessage());
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ message);
+ }
+ catch (ConfigException ce)
+ {
+ if (debugEnabled())
+ {
+ TRACER.debugCaught(DebugLogLevel.ERROR, ce);
+ }
+ throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
+ ce.getMessageObject());
}
catch (DatabaseException e)
{
@@ -1382,7 +1419,7 @@
TRACER.debugCaught(DebugLogLevel.ERROR, e);
}
throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
- e.getMessageObject());
+ e.getMessageObject());
}
finally
{
@@ -1407,7 +1444,6 @@
}
-
/**
* {@inheritDoc}
*/
@@ -1652,7 +1688,7 @@
*/
DirectoryException createDirectoryException(DatabaseException e) {
ResultCode resultCode = DirectoryServer.getServerErrorResultCode();
- Message message = null;
+ Message message;
if (e instanceof RunRecoveryException) {
message = NOTE_BACKEND_ENVIRONMENT_UNUSABLE.get(getBackendID());
logError(message);
diff --git a/opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java b/opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java
index c5d8130..0c6f332 100644
--- a/opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java
+++ b/opends/src/server/org/opends/server/backends/jeb/RebuildConfig.java
@@ -22,7 +22,7 @@
* CDDL HEADER END
*
*
- * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Copyright 2006-2009 Sun Microsystems, Inc.
*/
package org.opends.server.backends.jeb;
@@ -45,11 +45,9 @@
*/
private ArrayList<String> rebuildList;
- /**
- * The maximum number of rebuild threads to use at one time. An negative
- * number indicates unlimited max number of threads.
- */
- private int maxRebuildThreads = -1;
+ private boolean rebuildAll = false;
+
+ private String tmpDirectory;
/**
* Create a new rebuild configuraiton.
@@ -173,28 +171,6 @@
}
/**
- * Get the maximum number of rebuild threads to use for the rebuild job
- * at one time.
- *
- * @return The maximum number of rebuild threads.
- */
- public int getMaxRebuildThreads()
- {
- return maxRebuildThreads;
- }
-
- /**
- * Set the maximum number of rebuild threads to use for the rebuild
- * job at one time.
- *
- * @param maxRebuildThreads The maximum number of rebuild threads.
- */
- public void setMaxRebuildThreads(int maxRebuildThreads)
- {
- this.maxRebuildThreads = maxRebuildThreads;
- }
-
- /**
* Test if this rebuild config includes any system indexes to rebuild.
*
* @return True if rebuilding of system indexes are included. False otherwise.
@@ -215,17 +191,52 @@
{
return true;
}
- if(index.equalsIgnoreCase("id2children"))
- {
- return true;
- }
- if(index.equalsIgnoreCase("id2subtree"))
- {
- return true;
- }
}
return false;
}
+
+ /**
+ * Set the temporary directory to the specified path.
+ *
+ * @param path The path to set the temporary directory to.
+ */
+ public void setTmpDirectory(String path)
+ {
+ tmpDirectory = path;
+ }
+
+ /**
+ * Return the temporary directory path.
+ *
+ * @return The temporary directory string.
+ */
+ public String getTmpDirectory()
+ {
+ return tmpDirectory;
+ }
+
+
+ /**
+ * Set the rebuild all boolean to the specified value.
+ *
+ * @param v The value to set the rebuild all boolean to.
+ */
+ public void setRebuildAll(boolean v)
+ {
+ rebuildAll = v;
+ }
+
+ /**
+ * Return the rebuild all boolean value.
+ *
+ * @return Return the rebuild all boolean value.
+ */
+ public boolean isRebuildAll()
+ {
+ return rebuildAll;
+ }
+
+
}
diff --git a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
index 8b2a4c0..4f51731 100644
--- a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
@@ -45,6 +45,8 @@
import org.opends.messages.Category;
import org.opends.messages.Severity;
import org.opends.server.admin.std.server.LocalDBBackendCfg;
+import org.opends.server.admin.std.server.LocalDBIndexCfg;
+import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
import org.opends.server.backends.jeb.*;
import org.opends.server.config.ConfigException;
import org.opends.server.core.DirectoryServer;
@@ -54,7 +56,7 @@
/**
- * Performs a LDIF import.
+ * Performs LDIF import and rebuild of indexes.
*/
public class Importer
{
@@ -134,8 +136,9 @@
new ConcurrentHashMap<Integer, EntryContainer>();
private final Object synObj = new Object();
+ private final RebuildManager rebuildManager;
- static
+ static
{
if ((dnType = DirectoryServer.getAttributeType("dn")) == null)
{
@@ -143,18 +146,79 @@
}
}
+ private void initialize()
+ {
+
+ }
+
+ private
+ Importer(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg,
+ EnvironmentConfig envConfig) throws IOException,
+ InitializationException, JebException, ConfigException
+ {
+ this.importConfiguration = null;
+ this.threadCount = 1;
+ this.rebuildManager = new RebuildManager(rebuildConfig, cfg);
+ indexCount = rebuildManager.getIndexCount();
+ indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
+ indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
+ File parentDir;
+ if(rebuildConfig.getTmpDirectory() == null)
+ {
+ parentDir = getFileForPath("import-tmp");
+ }
+ else
+ {
+ parentDir = getFileForPath(rebuildConfig.getTmpDirectory());
+ }
+ tempDir = new File(parentDir, cfg.getBackendId());
+ if(!tempDir.exists() && !tempDir.mkdirs())
+ {
+ Message message =
+ ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String.valueOf(tempDir));
+ throw new IOException(message.toString());
+ }
+ if (tempDir.listFiles() != null)
+ {
+ for (File f : tempDir.listFiles())
+ {
+ f.delete();
+ }
+ }
+ skipDNValidation = true;
+ String propString = System.getProperty(DIRECT_PROPERTY);
+ if(propString != null)
+ {
+ int directSize = Integer.valueOf(propString);
+ directBuffer = ByteBuffer.allocateDirect(directSize);
+ }
+ else
+ {
+ directBuffer = null;
+ }
+ if(envConfig != null)
+ {
+ initializeDBEnv(envConfig);
+ }
+ }
+
+
/**
* Create a new import job with the specified ldif import config.
*
* @param importConfiguration The LDIF import configuration.
- * @param dbCfg The local DB back-end configuration.
+ * @param localDBBackendCfg The local DB back-end configuration.
+ * @param envConfig The JEB environment config.
* @throws IOException If a problem occurs while opening the LDIF file for
* reading.
* @throws InitializationException If a problem occurs during initialization.
*/
- public Importer(LDIFImportConfig importConfiguration, LocalDBBackendCfg dbCfg)
- throws IOException, InitializationException
+ private Importer(LDIFImportConfig importConfiguration,
+ LocalDBBackendCfg localDBBackendCfg,
+ EnvironmentConfig envConfig) throws IOException,
+ InitializationException
{
+ this.rebuildManager = null;
this.importConfiguration = importConfiguration;
if(importConfiguration.getThreadCount() == 0)
{
@@ -164,7 +228,9 @@
{
threadCount = importConfiguration.getThreadCount();
}
- indexCount = dbCfg.listLocalDBIndexes().length + 2;
+ indexCount = localDBBackendCfg.listLocalDBIndexes().length + 2;
+
+
indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
File parentDir;
@@ -177,7 +243,7 @@
parentDir = getFileForPath(importConfiguration.getTmpDirectory());
}
- tempDir = new File(parentDir, dbCfg.getBackendId());
+ tempDir = new File(parentDir, localDBBackendCfg.getBackendId());
if(!tempDir.exists() && !tempDir.mkdirs())
{
Message message =
@@ -202,9 +268,53 @@
{
directBuffer = null;
}
+ initializeDBEnv(envConfig);
}
- private void getBufferSizes(long availMem, int buffers)
+ /**
+ * Return and import LDIF instance using the specified arguments.
+ *
+ * @param importCfg The import config to use.
+ * @param localDBBackendCfg The local DB backend config to use.
+ * @param envCfg The JEB environment config to use.
+ * @return A import LDIF instance.
+ *
+ * @throws IOException If an I/O error occurs.
+ * @throws InitializationException If the instance cannot be initialized.
+ */
+ public static
+ Importer getInstance(LDIFImportConfig importCfg,
+ LocalDBBackendCfg localDBBackendCfg,
+ EnvironmentConfig envCfg)
+ throws IOException, InitializationException
+ {
+ return new Importer(importCfg, localDBBackendCfg, envCfg);
+ }
+
+
+ /**
+ * Return an import rebuild index instance using the specified arguments.
+ *
+ * @param rebuildCfg The rebuild config to use.
+ * @param localDBBackendCfg The local DB backend config to use.
+ * @param envCfg The JEB environment config to use.
+ * @return An import rebuild index instance.
+ *
+ * @throws IOException If an I/O error occurs.
+ * @throws InitializationException If the instance cannot be initialized.
+ * @throws JebException If a JEB exception occurs.
+ * @throws ConfigException If the instance cannot be configured.
+ */
+ public static synchronized
+ Importer getInstance(RebuildConfig rebuildCfg,
+ LocalDBBackendCfg localDBBackendCfg,
+ EnvironmentConfig envCfg)
+ throws IOException, InitializationException, JebException, ConfigException
+ {
+ return new Importer(rebuildCfg, localDBBackendCfg, envCfg);
+ }
+
+ private void getBufferSizes(long availMem, int buffers)
{
long memory = availMem - (MAX_DB_CACHE_SIZE + MAX_DB_LOG_BUFFER_BYTES);
bufferSize = (int) (memory/buffers);
@@ -240,7 +350,6 @@
}
}
-
/**
* Return the suffix instance in the specified map that matches the specified
* DN.
@@ -272,7 +381,7 @@
*
* @throws InitializationException If a problem occurs during calculation.
*/
- public void initialize(EnvironmentConfig envConfig)
+ private void initializeDBEnv(EnvironmentConfig envConfig)
throws InitializationException
{
Message message;
@@ -447,6 +556,33 @@
}
+ /**
+ * Rebuild the indexes using the specified rootcontainer.
+ *
+ * @param rootContainer The rootcontainer to rebuild indexes in.
+ *
+ * @throws ConfigException If a configuration error occurred.
+ * @throws InitializationException If an initialization error occurred.
+ * @throws IOException If an IO error occurred.
+ * @throws JebException If the JEB database had an error.
+ * @throws DatabaseException If a database error occurred.
+ * @throws InterruptedException If an interrupted error occurred.
+ * @throws ExecutionException If an execution error occurred.
+ */
+ public void
+ rebuildIndexes(RootContainer rootContainer) throws ConfigException,
+ InitializationException, IOException, JebException, DatabaseException,
+ InterruptedException, ExecutionException
+ {
+ this.rootContainer = rootContainer;
+ long startTime = System.currentTimeMillis();
+ rebuildManager.initialize();
+ rebuildManager.printStartMessage();
+ rebuildManager.rebuldIndexes();
+ tempDir.delete();
+ rebuildManager.printStopMessage(startTime);
+ }
+
/**
* Import a LDIF using the specified root container.
@@ -493,9 +629,10 @@
float rate = 0;
if (importTime > 0)
rate = 1000f * reader.getEntriesRead() / importTime;
- message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
- importCount.get(), reader.getEntriesIgnored(), reader
- .getEntriesRejected(), migratedCount, importTime / 1000, rate);
+ message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
+ importCount.get(), reader.getEntriesIgnored(),
+ reader.getEntriesRejected(), migratedCount,
+ importTime / 1000, rate);
logError(message);
}
finally
@@ -566,8 +703,11 @@
tasks.add(new MigrateExistingTask());
List<Future<Void>> results = execService.invokeAll(tasks);
- for (Future<Void> result : results)
- assert result.isDone();
+ for (Future<Void> result : results) {
+ if(!result.isDone()) {
+ result.get();
+ }
+ }
tasks.clear();
results.clear();
@@ -588,21 +728,23 @@
}
results = execService.invokeAll(tasks);
for (Future<Void> result : results)
- assert result.isDone();
-
-
+ if(!result.isDone()) {
+ result.get();
+ }
tasks.clear();
results.clear();
tasks.add(new MigrateExcludedTask());
results = execService.invokeAll(tasks);
for (Future<Void> result : results)
- assert result.isDone();
-
-
+ if(!result.isDone()) {
+ result.get();
+ }
stopIndexWriterTasks();
for (Future<?> result : indexWriterFutures)
{
- result.get();
+ if(!result.isDone()) {
+ result.get();
+ }
}
indexWriterList.clear();
indexWriterFutures.clear();
@@ -618,7 +760,7 @@
private void processPhaseTwo() throws InterruptedException
{
SecondPhaseProgressTask progress2Task =
- new SecondPhaseProgressTask(indexMgrList);
+ new SecondPhaseProgressTask(indexMgrList, reader.getEntriesRead());
Timer timer2 = new Timer();
timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
processIndexFiles();
@@ -975,24 +1117,23 @@
Index index;
if((index=attributeIndex.getEqualityIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.EQUALITY));
+ new IndexKey(attributeType, ImportIndexType.EQUALITY));
}
if((index=attributeIndex.getPresenceIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.PRESENCE));
+ new IndexKey(attributeType, ImportIndexType.PRESENCE));
}
if((index=attributeIndex.getSubstringIndex()) != null) {
- int subLen = ((SubstringIndexer)index.indexer).getSubStringLen();
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.SUBSTRING, subLen));
+ new IndexKey(attributeType, ImportIndexType.SUBSTRING));
}
if((index=attributeIndex.getOrderingIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.ORDERING));
+ new IndexKey(attributeType, ImportIndexType.ORDERING));
}
if((index=attributeIndex.getApproximateIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.APPROXIMATE));
+ new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
}
for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
Transaction transaction = null;
@@ -1007,7 +1148,7 @@
if(subIndexes != null) {
for(Index subIndex: subIndexes) {
processAttribute(subIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SUBSTRING));
+ new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
}
}
Collection<Index> sharedIndexes =
@@ -1016,7 +1157,7 @@
if(sharedIndexes !=null) {
for(Index sharedIndex:sharedIndexes) {
processAttribute(sharedIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SHARED));
+ new IndexKey(attributeType, ImportIndexType.EX_SHARED));
}
}
}
@@ -1274,24 +1415,23 @@
Index index;
if((index=attributeIndex.getEqualityIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.EQUALITY));
+ new IndexKey(attributeType, ImportIndexType.EQUALITY));
}
if((index=attributeIndex.getPresenceIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.PRESENCE));
+ new IndexKey(attributeType, ImportIndexType.PRESENCE));
}
if((index=attributeIndex.getSubstringIndex()) != null) {
- int subLen = ((SubstringIndexer)index.indexer).getSubStringLen();
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.SUBSTRING, subLen));
+ new IndexKey(attributeType, ImportIndexType.SUBSTRING));
}
if((index=attributeIndex.getOrderingIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.ORDERING));
+ new IndexKey(attributeType, ImportIndexType.ORDERING));
}
if((index=attributeIndex.getApproximateIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.APPROXIMATE));
+ new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
}
for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
Transaction transaction = null;
@@ -1306,7 +1446,7 @@
if(subIndexes != null) {
for(Index subIndex: subIndexes) {
processAttribute(subIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SUBSTRING));
+ new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
}
}
Collection<Index> sharedIndexes =
@@ -1315,7 +1455,7 @@
if(sharedIndexes !=null) {
for(Index sharedIndex:sharedIndexes) {
processAttribute(sharedIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SHARED));
+ new IndexKey(attributeType, ImportIndexType.EX_SHARED));
}
}
}
@@ -1346,8 +1486,8 @@
{
IndexKey indexKey = e.getKey();
IndexBuffer indexBuffer = e.getValue();
- IndexType indexType = indexKey.getIndexType();
- if(indexType.equals(IndexType.DN))
+ ImportIndexType indexType = indexKey.getIndexType();
+ if(indexType.equals(ImportIndexType.DN))
{
indexBuffer.setComparator(dnComparator);
}
@@ -1413,7 +1553,7 @@
DatabaseContainer dn2id = suffix.getDN2ID();
byte[] dnBytes = StaticUtils.getBytes(dn.toNormalizedString());
int id = processKey(dn2id, dnBytes, entryID, dnComparator,
- new IndexKey(dnType, IndexType.DN), true);
+ new IndexKey(dnType, ImportIndexType.DN), true);
idECMap.putIfAbsent(id, suffix.getEntryContainer());
}
@@ -2070,7 +2210,8 @@
*/
public Void call() throws Exception
{
- if (importConfiguration.isCancelled())
+ if (importConfiguration != null &&
+ importConfiguration.isCancelled())
{
return null;
}
@@ -2100,7 +2241,7 @@
{
return;
}
- if(indexKey.getIndexType().equals(IndexType.DN))
+ if(indexKey.getIndexType().equals(ImportIndexType.DN))
{
isDN = true;
}
@@ -2522,6 +2663,837 @@
}
}
+
+ /**
+ * The rebuild manager handles all rebuild index related tasks.
+ */
+ class RebuildManager extends ImportTask {
+
+ private final RebuildConfig rebuildConfig;
+ private final LocalDBBackendCfg cfg;
+ private final Map<IndexKey, Index> indexMap =
+ new LinkedHashMap<IndexKey, Index>();
+ private final Map<IndexKey, Collection<Index>> extensibleIndexMap =
+ new LinkedHashMap<IndexKey, Collection<Index>>();
+ private DN2ID dn2id = null;
+ private DN2URI dn2uri = null;
+ private long totalEntries =0;
+ private final AtomicLong entriesProcessed = new AtomicLong(0);
+ private Suffix suffix = null;
+ private final boolean rebuildAll;
+ private EntryContainer ec;
+
+
+ /**
+ * Create an instance of the rebuild index manager using the specified
+ * parameters.
+ *
+ * @param rebuildConfig The rebuild configuration to use.
+ * @param cfg The local DB configuration to use.
+ */
+ public RebuildManager(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg)
+ {
+ this.rebuildConfig = rebuildConfig;
+ this.cfg = cfg;
+ this.rebuildAll = rebuildConfig.isRebuildAll();
+ }
+
+ /**
+ * Initialize a rebuild manager to start rebuilding indexes.
+ *
+ * @throws ConfigException If an configuration error occurred.
+ * @throws InitializationException If an initialization error occurred.
+ */
+ public void initialize() throws ConfigException, InitializationException
+ {
+ ec = rootContainer.getEntryContainer(rebuildConfig.getBaseDN());
+ suffix = Suffix.createSuffixContext(ec, null, null, null);
+ if(suffix == null)
+ {
+ Message msg = ERR_JEB_REBUILD_SUFFIX_ERROR.get(rebuildConfig.
+ getBaseDN().toString());
+ throw new InitializationException(msg);
+ }
+ }
+
+ /**
+ * Print start message.
+ *
+ * @throws DatabaseException If an database error occurred.
+ */
+ public void printStartMessage() throws DatabaseException
+ {
+ StringBuilder sb = new StringBuilder();
+ List<String> rebuildList = rebuildConfig.getRebuildList();
+ for(String index : rebuildList)
+ {
+ if(sb.length() > 0)
+ {
+ sb.append(", ");
+ }
+ sb.append(index);
+ }
+ totalEntries = suffix.getID2Entry().getRecordCount();
+ Message message = NOTE_JEB_REBUILD_START.get(sb.toString(), totalEntries);
+ if(rebuildAll) {
+ message = NOTE_JEB_REBUILD_ALL_START.get(totalEntries);
+ }
+ logError(message);
+ }
+
+ /**
+ * Print stop message.
+ *
+ * @param startTime The time the rebuild started.
+ */
+ public void printStopMessage(long startTime)
+ {
+ long finishTime = System.currentTimeMillis();
+ long totalTime = (finishTime - startTime);
+ float rate = 0;
+ if (totalTime > 0)
+ {
+ rate = 1000f* entriesProcessed.get() / totalTime;
+ }
+ Message message =
+ NOTE_JEB_REBUILD_FINAL_STATUS.get(entriesProcessed.get(),
+ totalTime/1000, rate);
+ logError(message);
+ }
+
+
+ /**
+ * {@inheritDoc}
+ */
+ public Void call() throws Exception
+ {
+ ID2Entry id2entry = ec.getID2Entry();
+ Cursor cursor = id2entry.openCursor(null, CursorConfig.READ_COMMITTED);
+ DatabaseEntry key = new DatabaseEntry();
+ DatabaseEntry data = new DatabaseEntry();
+ LockMode lockMode = LockMode.DEFAULT;
+ OperationStatus status;
+ try {
+ for (status = cursor.getFirst(key, data, lockMode);
+ status == OperationStatus.SUCCESS;
+ status = cursor.getNext(key, data, lockMode))
+ {
+ EntryID entryID = new EntryID(key);
+ Entry entry = ID2Entry.entryFromDatabase(
+ ByteString.wrap(data.getData()),
+ ec.getRootContainer().getCompressedSchema());
+ processEntry(entry, entryID);
+ entriesProcessed.getAndIncrement();
+ }
+ flushIndexBuffers();
+ cursor.close();
+ } catch (Exception e) {
+ System.out.println("here");
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * Perform the index rebuild.
+ *
+ * @throws DatabaseException If an database error occurred.
+ * @throws InterruptedException If an interrupted error occurred.
+ * @throws ExecutionException If an Excecution error occurred.
+ * @throws JebException If an JEB error occurred.
+ */
+ public void rebuldIndexes() throws DatabaseException, InterruptedException,
+ ExecutionException, JebException
+ {
+ processPhaseOne();
+ processPhaseTwo();
+ setIndexesTrusted();
+ }
+
+ private void setIndexesTrusted() throws JebException
+ {
+ try {
+ suffix.setIndexesTrusted();
+ }
+ catch (DatabaseException ex)
+ {
+ Message message =
+ NOTE_JEB_IMPORT_LDIF_TRUSTED_FAILED.get(ex.getMessage());
+ throw new JebException(message);
+ }
+ }
+
+ private void processPhaseOne() throws DatabaseException,
+ InterruptedException, ExecutionException {
+ if(rebuildAll)
+ {
+ clearAllIndexes();
+ }
+ else
+ {
+ clearRebuildListIndexes();
+ }
+ initializeIndexBuffers(threadCount);
+ RBFirstPhaseProgressTask progressTask = new RBFirstPhaseProgressTask();
+ Timer timer = new Timer();
+ timer.scheduleAtFixedRate(progressTask, TIMER_INTERVAL, TIMER_INTERVAL);
+ indexProcessService = Executors.newFixedThreadPool(2 * indexCount);
+ sortService = Executors.newFixedThreadPool(threadCount);
+ ExecutorService execService = Executors.newFixedThreadPool(threadCount);
+ List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(threadCount);
+ for (int i = 0; i < threadCount; i++)
+ {
+ tasks.add(this);
+ }
+ List<Future<Void>> results = execService.invokeAll(tasks);
+ for (Future<Void> result : results) {
+ if(!result.isDone()) {
+ result.get();
+ }
+ }
+ stopIndexWriterTasks();
+ for (Future<?> result : indexWriterFutures)
+ {
+ if(!result.isDone()) {
+ result.get();
+ }
+ }
+ tasks.clear();
+ results.clear();
+ execService.shutdown();
+ freeBufferQueue.clear();
+ sortService.shutdown();
+ timer.cancel();
+ }
+
+
+ private void processPhaseTwo() throws InterruptedException
+ {
+ SecondPhaseProgressTask progress2Task =
+ new SecondPhaseProgressTask(indexMgrList, entriesProcessed.get());
+ Timer timer2 = new Timer();
+ timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
+ processIndexFiles();
+ timer2.cancel();
+ }
+
+ private int getIndexCount() throws ConfigException, JebException
+ {
+ int indexCount;
+ if(!rebuildAll)
+ {
+ indexCount = getRebuildListIndexCount(cfg);
+ }
+ else
+ {
+ indexCount = getAllIndexesCount(cfg);
+ }
+ return indexCount;
+ }
+
+ private int getAllIndexesCount(LocalDBBackendCfg cfg)
+ {
+ int indexCount = cfg.listLocalDBIndexes().length;
+ indexCount += cfg.listLocalDBVLVIndexes().length;
+ indexCount += 4;
+ return indexCount;
+ }
+
+ private int getRebuildListIndexCount(LocalDBBackendCfg cfg)
+ throws JebException, ConfigException
+ {
+ int indexCount = 0;
+ List<String> rebuildList = rebuildConfig.getRebuildList();
+ if(!rebuildList.isEmpty())
+ {
+ for (String index : rebuildList)
+ {
+ String lowerName = index.toLowerCase();
+ if (lowerName.equals("dn2id"))
+ {
+ indexCount += 3;
+ }
+ else if (lowerName.equals("dn2uri"))
+ {
+ indexCount++;
+ }
+ else if (lowerName.startsWith("vlv."))
+ {
+ if(lowerName.length() < 5)
+ {
+ Message msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
+ throw new JebException(msg);
+ }
+ indexCount++;
+ } else if(lowerName.equals("id2subtree") ||
+ lowerName.equals("id2children"))
+ {
+ Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ else
+ {
+ String[] attrIndexParts = lowerName.split("\\.");
+ if((attrIndexParts.length <= 0) || (attrIndexParts.length > 3))
+ {
+ Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(attrIndexParts[0]);
+ if (attrType == null)
+ {
+ Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ if(attrIndexParts.length != 1)
+ {
+ if(attrIndexParts.length == 2)
+ {
+ if(attrIndexParts[1].equals("presence"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("equality"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("substring"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("ordering"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("approximate"))
+ {
+ indexCount++;
+ } else {
+ Message msg =
+ ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ }
+ else
+ {
+ boolean found = false;
+ String s = attrIndexParts[1] + "." + attrIndexParts[2];
+ for (String idx : cfg.listLocalDBIndexes())
+ {
+ LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
+ if (indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
+ {
+ Set<String> extensibleRules =
+ indexCfg.getIndexExtensibleMatchingRule();
+ for(String exRule : extensibleRules)
+ {
+ if(exRule.equalsIgnoreCase(s))
+ {
+ found = true;
+ break;
+ }
+ }
+ }
+ if(found)
+ {
+ break;
+ }
+ }
+ if(!found) {
+ Message msg =
+ ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ indexCount++;
+ }
+ }
+ else
+ {
+ for (String idx : cfg.listLocalDBIndexes())
+ {
+ if(!idx.equalsIgnoreCase(index))
+ {
+ continue;
+ }
+ LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.EQUALITY))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.ORDERING))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.PRESENCE))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE))
+ {
+ indexCount++;
+ }
+ if (indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
+ {
+ Set<String> extensibleRules =
+ indexCfg.getIndexExtensibleMatchingRule();
+ boolean shared = false;
+ for(String exRule : extensibleRules)
+ {
+ if(exRule.endsWith(".sub"))
+ {
+ indexCount++;
+ }
+ else
+ {
+ if(!shared)
+ {
+ shared=true;
+ indexCount++;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return indexCount;
+ }
+
+ private void clearRebuildListIndexes() throws DatabaseException
+ {
+ List<String> rebuildList = rebuildConfig.getRebuildList();
+ if(!rebuildList.isEmpty())
+ {
+ for (String index : rebuildList)
+ {
+ String lowerName = index.toLowerCase();
+ if (lowerName.equals("dn2id"))
+ {
+ clearDN2IDIndexes(ec);
+ }
+ else if (lowerName.equals("dn2uri"))
+ {
+ clearDN2URI(ec);
+ }
+ else if (lowerName.startsWith("vlv."))
+ {
+ clearVLVIndex(lowerName.substring(4), ec);
+ }
+ else
+ {
+ String[] attrIndexParts = lowerName.split("\\.");
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(attrIndexParts[0]);
+ AttributeIndex attrIndex = ec.getAttributeIndex(attrType);
+
+ if(attrIndexParts.length != 1)
+ {
+ Index partialAttrIndex;
+ if(attrIndexParts[1].equals("presence"))
+ {
+ partialAttrIndex = attrIndex.getPresenceIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.PRESENCE);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("equality"))
+ {
+ partialAttrIndex = attrIndex.getEqualityIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.EQUALITY);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("substring"))
+ {
+ partialAttrIndex = attrIndex.getSubstringIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.SUBSTRING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("ordering"))
+ {
+ partialAttrIndex = attrIndex.getOrderingIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.ORDERING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("approximate"))
+ {
+ partialAttrIndex = attrIndex.getApproximateIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.APPROXIMATE);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else
+ {
+ String dbPart = "shared";
+ if(attrIndexParts[2].startsWith("sub"))
+ {
+ dbPart = "substring";
+ }
+ StringBuilder nameBldr = new StringBuilder();
+ nameBldr.append(ec.getDatabasePrefix());
+ nameBldr.append("_");
+ nameBldr.append(attrIndexParts[0]);
+ nameBldr.append(".");
+ nameBldr.append(attrIndexParts[1]);
+ nameBldr.append(".");
+ nameBldr.append(dbPart);
+ String indexName = nameBldr.toString();
+ Map<String,Collection<Index>> extensibleMap =
+ attrIndex.getExtensibleIndexes();
+ if(!extensibleMap.isEmpty()) {
+ Collection<Index> subIndexes =
+ attrIndex.getExtensibleIndexes().get(
+ EXTENSIBLE_INDEXER_ID_SUBSTRING);
+ if(subIndexes != null) {
+ for(Index subIndex : subIndexes) {
+ String name = subIndex.getName();
+ if(name.equalsIgnoreCase(indexName))
+ {
+ ec.clearDatabase(subIndex);
+ Collection<Index> substring = new ArrayList<Index>();
+ substring.add(subIndex);
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SUBSTRING),substring);
+ break;
+ }
+ }
+ Collection<Index> sharedIndexes = attrIndex.
+ getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
+ if(sharedIndexes !=null) {
+ for(Index sharedIndex : sharedIndexes) {
+ String name = sharedIndex.getName();
+ if(name.equalsIgnoreCase(indexName))
+ {
+ ec.clearDatabase(sharedIndex);
+ Collection<Index> shared = new ArrayList<Index>();
+ shared.add(sharedIndex);
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SHARED), shared);
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ clearAttributeIndexes(attrIndex, attrType, ec);
+ }
+ }
+ }
+ }
+ }
+
+
+ private void clearAllIndexes() throws DatabaseException
+ {
+ for(Map.Entry<AttributeType, AttributeIndex> mapEntry :
+ suffix.getAttrIndexMap().entrySet()) {
+ AttributeType attributeType = mapEntry.getKey();
+ AttributeIndex attributeIndex = mapEntry.getValue();
+ clearAttributeIndexes(attributeIndex, attributeType, ec);
+ }
+ for(VLVIndex vlvIndex : suffix.getEntryContainer().getVLVIndexes()) {
+ ec.clearDatabase(vlvIndex);
+ }
+ clearDN2IDIndexes(ec);
+ if(ec.getDN2URI() != null)
+ {
+ clearDN2URI(ec);
+ }
+ }
+
+ private void clearVLVIndex(String name, EntryContainer ec)
+ throws DatabaseException
+ {
+ VLVIndex vlvIndex = ec.getVLVIndex(name);
+ ec.clearDatabase(vlvIndex);
+ }
+
+ private void clearDN2URI(EntryContainer ec) throws DatabaseException
+ {
+ ec.clearDatabase(ec.getDN2URI());
+ dn2uri = ec.getDN2URI();
+ }
+
+ private void clearDN2IDIndexes(EntryContainer ec) throws DatabaseException
+ {
+ ec.clearDatabase(ec.getDN2ID());
+ ec.clearDatabase(ec.getID2Children());
+ ec.clearDatabase(ec.getID2Subtree());
+ dn2id = ec.getDN2ID();
+ }
+
+ private void clearAttributeIndexes(AttributeIndex attrIndex,
+ AttributeType attrType, EntryContainer ec)
+ throws DatabaseException
+ {
+ Index partialAttrIndex;
+ if(attrIndex.getSubstringIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getSubstringIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.SUBSTRING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ if(attrIndex.getOrderingIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getOrderingIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.ORDERING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ if(attrIndex.getEqualityIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getEqualityIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.EQUALITY);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ if(attrIndex.getPresenceIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getPresenceIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.PRESENCE);
+ indexMap.put(indexKey, partialAttrIndex);
+
+ }
+ if(attrIndex.getApproximateIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getApproximateIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.APPROXIMATE);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ Map<String,Collection<Index>> extensibleMap =
+ attrIndex.getExtensibleIndexes();
+ if(!extensibleMap.isEmpty()) {
+ Collection<Index> subIndexes =
+ attrIndex.getExtensibleIndexes().get(
+ EXTENSIBLE_INDEXER_ID_SUBSTRING);
+ if(subIndexes != null) {
+ for(Index subIndex : subIndexes) {
+ ec.clearDatabase(subIndex);
+ }
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SUBSTRING), subIndexes);
+ }
+ Collection<Index> sharedIndexes =
+ attrIndex.getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
+ if(sharedIndexes !=null) {
+ for(Index sharedIndex : sharedIndexes) {
+ ec.clearDatabase(sharedIndex);
+ }
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SHARED), sharedIndexes);
+ }
+ }
+ }
+
+
+ private
+ void processEntry(Entry entry, EntryID entryID) throws DatabaseException,
+ ConfigException, DirectoryException, JebException
+ {
+ if(dn2id != null)
+ {
+ processDN2ID(suffix, entry.getDN(), entryID);
+ }
+ if(dn2uri != null)
+ {
+ processDN2URI(suffix, null, entry);
+ }
+ processIndexes(entry, entryID);
+ processExtensibleIndexes(entry, entryID);
+ processVLVIndexes(entry, entryID);
+ }
+
+ private void processVLVIndexes(Entry entry, EntryID entryID)
+ throws DatabaseException, JebException, DirectoryException
+ {
+ for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
+ Transaction transaction = null;
+ vlvIdx.addEntry(transaction, entryID, entry);
+ }
+ }
+
+
+ private void processExtensibleIndexes(Entry entry, EntryID entryID) throws
+ DatabaseException, DirectoryException, JebException, ConfigException
+ {
+ for(Map.Entry<IndexKey, Collection<Index>> mapEntry :
+ this.extensibleIndexMap.entrySet()) {
+ IndexKey key = mapEntry.getKey();
+ AttributeType attrType = key.getType();
+ if(entry.hasAttribute(attrType)) {
+ Collection<Index> indexes = mapEntry.getValue();
+ for(Index index : indexes) {
+ processAttribute(index, entry, entryID, key);
+ }
+ }
+ }
+ }
+
+ private void
+ processIndexes(Entry entry, EntryID entryID) throws
+ DatabaseException, DirectoryException, JebException, ConfigException
+ {
+
+ for(Map.Entry<IndexKey, Index> mapEntry :
+ indexMap.entrySet()) {
+ IndexKey key = mapEntry.getKey();
+ AttributeType attrType = key.getType();
+ if(entry.hasAttribute(attrType)) {
+ ImportIndexType indexType = key.getIndexType();
+ Index index = mapEntry.getValue();
+ if(indexType == ImportIndexType.SUBSTRING)
+ {
+ processAttribute(index, entry, entryID,
+ new IndexKey(attrType, ImportIndexType.SUBSTRING));
+ }
+ else
+ {
+ processAttribute(index, entry, entryID,
+ new IndexKey(attrType, indexType));
+ }
+ }
+ }
+ }
+
+ /**
+ * Return the number of entries processed by the rebuild manager.
+ *
+ * @return The number of entries processed.
+ */
+ public long getEntriesProcess()
+ {
+ return this.entriesProcessed.get();
+ }
+
+ /**
+ * Return the total number of entries to process by the rebuild manager.
+ *
+ * @return The total number for entries to process.
+ */
+ public long getTotEntries()
+ {
+ return this.totalEntries;
+ }
+ }
+
+ /**
+ * This class reports progress of the rebuild job at fixed intervals.
+ */
+ class RBFirstPhaseProgressTask extends TimerTask
+ {
+ /**
+ * The number of records that had been processed at the time of the
+ * previous progress report.
+ */
+ private long previousProcessed = 0;
+
+ /**
+ * The time in milliseconds of the previous progress report.
+ */
+ private long previousTime;
+
+ /**
+ * The environment statistics at the time of the previous report.
+ */
+ private EnvironmentStats prevEnvStats;
+
+ /**
+ * Create a new verify progress task.
+ * @throws DatabaseException An error occurred while accessing the JE
+ * database.
+ */
+ public RBFirstPhaseProgressTask() throws DatabaseException
+ {
+ previousTime = System.currentTimeMillis();
+ prevEnvStats =
+ rootContainer.getEnvironmentStats(new StatsConfig());
+ }
+
+ /**
+ * The action to be performed by this timer task.
+ */
+ public void run()
+ {
+ long latestTime = System.currentTimeMillis();
+ long deltaTime = latestTime - previousTime;
+
+ if (deltaTime == 0)
+ {
+ return;
+ }
+ long currentRBProcessed = rebuildManager.getEntriesProcess();
+ long deltaCount = (currentRBProcessed - previousProcessed);
+ float rate = 1000f*deltaCount / deltaTime;
+ float completed = 0;
+ if(rebuildManager.getTotEntries() > 0)
+ {
+ completed = 100f*currentRBProcessed / rebuildManager.getTotEntries();
+ }
+ Message message = NOTE_JEB_REBUILD_PROGRESS_REPORT.get(
+ completed, currentRBProcessed, rebuildManager.getTotEntries(), rate);
+ logError(message);
+ try
+ {
+ Runtime runtime = Runtime.getRuntime();
+ long freeMemory = runtime.freeMemory() / MB;
+ EnvironmentStats envStats =
+ rootContainer.getEnvironmentStats(new StatsConfig());
+ long nCacheMiss =
+ envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
+
+ float cacheMissRate = 0;
+ if (deltaCount > 0)
+ {
+ cacheMissRate = nCacheMiss/(float)deltaCount;
+ }
+ message = NOTE_JEB_REBUILD_CACHE_AND_MEMORY_REPORT.get(
+ freeMemory, cacheMissRate);
+ logError(message);
+ prevEnvStats = envStats;
+ }
+ catch (DatabaseException e)
+ {
+
+ }
+ previousProcessed = currentRBProcessed;
+ previousTime = latestTime;
+ }
+ }
+
+
/**
* This class reports progress of the import job at fixed intervals.
*/
@@ -2665,7 +3637,7 @@
/**
* This class reports progress of the import job at fixed intervals.
*/
- private final class SecondPhaseProgressTask extends TimerTask
+ class SecondPhaseProgressTask extends TimerTask
{
/**
* The number of entries that had been read at the time of the
@@ -2687,16 +3659,19 @@
private boolean evicting = false;
private final List<IndexManager> indexMgrList;
-
+ private long latestCount;
/**
* Create a new import progress task.
* @param indexMgrList List of index managers.
+ * @param latestCount The latest count of entries processed in phase one.
*/
- public SecondPhaseProgressTask (List<IndexManager> indexMgrList)
+ public SecondPhaseProgressTask (List<IndexManager> indexMgrList,
+ long latestCount)
{
previousTime = System.currentTimeMillis();
this.indexMgrList = indexMgrList;
+ this.latestCount = latestCount;
try
{
previousStats =
@@ -2715,7 +3690,6 @@
@Override
public void run()
{
- long latestCount = reader.getEntriesRead() + 0;
long deltaCount = (latestCount - previousCount);
long latestTime = System.currentTimeMillis();
long deltaTime = latestTime - previousTime;
@@ -2847,7 +3821,7 @@
* This class defines the individual index type available.
*
*/
- public enum IndexType {
+ public enum ImportIndexType {
/**
* The DN index type.
**/
@@ -2886,9 +3860,13 @@
/**
* The extensible shared index type.
**/
- EX_SHARED
- }
+ EX_SHARED,
+ /**
+ * The vlv index type.
+ */
+ VLV
+ }
/**
* This class is used as an index key for hash maps that need to
@@ -2901,22 +3879,8 @@
public class IndexKey {
private final AttributeType type;
- private final IndexType indexType;
- private byte[] keyBytes = null;
+ private final ImportIndexType indexType;
- /**
- * Create index key instance using the specified attribute type, index type
- * and sub-string length. Used only for sub-string indexes.
- *
- * @param type The attribute type.
- * @param indexType The index type.
- * @param subLen The sub-string length.
- */
- IndexKey(AttributeType type, IndexType indexType, int subLen)
- {
- this(type, indexType);
- keyBytes = new byte[subLen];
- }
/**
* Create index key instance using the specified attribute type, index type.
@@ -2924,31 +3888,31 @@
* @param type The attribute type.
* @param indexType The index type.
*/
- IndexKey(AttributeType type, IndexType indexType)
+ IndexKey(AttributeType type, ImportIndexType indexType)
{
this.type = type;
this.indexType = indexType;
}
- /**
- * An equals method that uses both the attribute type and the index type.
- *
- * @param obj the object to compare.
- * @return <CODE>true</CODE> if the objects are equal.
- */
- public boolean equals(Object obj)
- {
- boolean returnCode = false;
- if (obj instanceof IndexKey) {
- IndexKey oKey = (IndexKey) obj;
- if(type.equals(oKey.getType()) &&
- indexType.equals(oKey.getIndexType()))
- {
- returnCode = true;
- }
- }
- return returnCode;
+ /**
+ * An equals method that uses both the attribute type and the index type.
+ *
+ * @param obj the object to compare.
+ * @return <CODE>true</CODE> if the objects are equal.
+ */
+ public boolean equals(Object obj)
+ {
+ boolean returnCode = false;
+ if (obj instanceof IndexKey) {
+ IndexKey oKey = (IndexKey) obj;
+ if(type.equals(oKey.getType()) &&
+ indexType.equals(oKey.getIndexType()))
+ {
+ returnCode = true;
+ }
}
+ return returnCode;
+ }
/**
* A hash code method that adds the hash codes of the attribute type and
@@ -2975,7 +3939,7 @@
* Return the index type.
* @return The index type.
*/
- public IndexType getIndexType()
+ public ImportIndexType getIndexType()
{
return indexType;
}
@@ -2990,7 +3954,7 @@
public String getName()
{
return type.getPrimaryName() + "." +
- StaticUtils.toLowerCase(indexType.name());
+ StaticUtils.toLowerCase(indexType.name());
}
}
}
diff --git a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Suffix.java b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Suffix.java
index 1ecb201..e5ce065 100644
--- a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Suffix.java
+++ b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Suffix.java
@@ -27,16 +27,13 @@
package org.opends.server.backends.jeb.importLDIF;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import org.opends.server.backends.jeb.*;
import org.opends.server.config.ConfigException;
import org.opends.server.types.*;
import static org.opends.server.loggers.ErrorLogger.logError;
+import static org.opends.server.util.ServerConstants.*;
import org.opends.messages.Message;
import org.opends.messages.Category;
import org.opends.messages.Severity;
@@ -277,7 +274,7 @@
entryContainer.getID2Children().setTrusted(null,true);
entryContainer.getID2Subtree().setTrusted(null, true);
for(AttributeIndex attributeIndex :
- entryContainer.getAttributeIndexes()) {
+ entryContainer.getAttributeIndexes()) {
Index index;
if((index = attributeIndex.getEqualityIndex()) != null) {
index.setTrusted(null, true);
@@ -294,9 +291,28 @@
if((index=attributeIndex.getApproximateIndex()) != null) {
index.setTrusted(null, true);
}
+ Map<String,Collection<Index>> exIndexes =
+ attributeIndex.getExtensibleIndexes();
+ if(!exIndexes.isEmpty())
+ {
+ Collection<Index> subIndexes = attributeIndex.getExtensibleIndexes().
+ get(EXTENSIBLE_INDEXER_ID_SUBSTRING);
+ if(subIndexes != null) {
+ for(Index subIndex : subIndexes) {
+ subIndex.setTrusted(null, true);
+ }
+ }
+ Collection<Index> sharedIndexes = attributeIndex.
+ getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
+ if(sharedIndexes !=null) {
+ for(Index sharedIndex : sharedIndexes) {
+ sharedIndex.setTrusted(null, true);
+ }
+ }
+ }
}
for(VLVIndex vlvIdx : entryContainer.getVLVIndexes()) {
- vlvIdx.setTrusted(null, true);
+ vlvIdx.setTrusted(null, true);
}
}
diff --git a/opends/src/server/org/opends/server/config/ConfigConstants.java b/opends/src/server/org/opends/server/config/ConfigConstants.java
index 4975cca..dded712 100644
--- a/opends/src/server/org/opends/server/config/ConfigConstants.java
+++ b/opends/src/server/org/opends/server/config/ConfigConstants.java
@@ -4332,10 +4332,15 @@
/**
* The name of the attribute in an rebuild task definition that specifies the
- * maximum number of threads.
+ * temporary scratch file.
*/
- public static final String ATTR_REBUILD_MAX_THREADS =
- NAME_PREFIX_TASK + "rebuild-max-threads";
+ public static final String ATTR_REBUILD_TMP_DIRECTORY =
+ NAME_PREFIX_TASK + "rebuild-tmp-directory";
+
+ /**
+ * Used to specify that the rebuild all boolean should be set.
+ */
+ public static final String REBUILD_ALL = "rebuildall";
/**
* The name of the objectclass that will be used for a Directory Server
diff --git a/opends/src/server/org/opends/server/tasks/RebuildTask.java b/opends/src/server/org/opends/server/tasks/RebuildTask.java
index a8fa88a..e7a800b 100644
--- a/opends/src/server/org/opends/server/tasks/RebuildTask.java
+++ b/opends/src/server/org/opends/server/tasks/RebuildTask.java
@@ -52,26 +52,10 @@
import static org.opends.server.util.StaticUtils.*;
import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
-import static org.opends.messages.TaskMessages.
- ERR_TASK_INDEXREBUILD_INSUFFICIENT_PRIVILEGES;
-import static org.opends.messages.ToolMessages.
- ERR_REBUILDINDEX_ERROR_DURING_REBUILD;
-import static org.opends.messages.ToolMessages.
- ERR_REBUILDINDEX_WRONG_BACKEND_TYPE;
-import static org.opends.messages.ToolMessages.
- ERR_NO_BACKENDS_FOR_BASE;
-import static org.opends.messages.ToolMessages.
- ERR_CANNOT_DECODE_BASE_DN;
-import static org.opends.messages.ToolMessages.
- ERR_REBUILDINDEX_CANNOT_EXCLUSIVE_LOCK_BACKEND;
-import static org.opends.messages.ToolMessages.
- WARN_REBUILDINDEX_CANNOT_UNLOCK_BACKEND;
-import static org.opends.server.config.ConfigConstants.
- ATTR_REBUILD_BASE_DN;
-import static org.opends.server.config.ConfigConstants.
- ATTR_REBUILD_INDEX;
-import static org.opends.server.config.ConfigConstants.
- ATTR_REBUILD_MAX_THREADS;
+import static org.opends.messages.TaskMessages.*;
+import static org.opends.messages.ToolMessages.*;
+import static org.opends.server.config.ConfigConstants.*;
+
import java.util.List;
import java.util.ArrayList;
@@ -89,7 +73,8 @@
String baseDN = null;
ArrayList<String> indexes = null;
- int maxThreads = -1;
+ private String tmpDirectory = null;
+ private boolean rebuildAll = false;
/**
* {@inheritDoc}
@@ -123,14 +108,14 @@
AttributeType typeBaseDN;
AttributeType typeIndex;
- AttributeType typeMaxThreads;
+ AttributeType typeTmpDirectory;
typeBaseDN =
getAttributeType(ATTR_REBUILD_BASE_DN, true);
typeIndex =
getAttributeType(ATTR_REBUILD_INDEX, true);
- typeMaxThreads =
- getAttributeType(ATTR_REBUILD_MAX_THREADS, true);
+ typeTmpDirectory =
+ getAttributeType(ATTR_REBUILD_TMP_DIRECTORY, true);
List<Attribute> attrList;
@@ -140,9 +125,32 @@
attrList = taskEntry.getAttribute(typeIndex);
indexes = TaskUtils.getMultiValueString(attrList);
+ if(isRebuildAll(indexes))
+ {
+ if(indexes.size() != 1)
+ {
+ Message msg = ERR_TASK_INDEXREBUILD_ALL_ERROR.get();
+ throw new DirectoryException(ResultCode.UNWILLING_TO_PERFORM, msg);
+ }
+ rebuildAll = true;
+ indexes.clear();
+ }
- attrList = taskEntry.getAttribute(typeMaxThreads);
- maxThreads = TaskUtils.getSingleValueInteger(attrList, -1);
+ attrList = taskEntry.getAttribute(typeTmpDirectory);
+ tmpDirectory = TaskUtils.getSingleValueString(attrList);
+
+ }
+
+ private boolean isRebuildAll(List<String> indexList)
+ {
+ for(String s : indexList)
+ {
+ if(s.equalsIgnoreCase(REBUILD_ALL))
+ {
+ return true;
+ }
+ }
+ return false;
}
/**
@@ -169,7 +177,6 @@
rebuildConfig.addRebuildIndex(index);
}
- rebuildConfig.setMaxRebuildThreads(maxThreads);
Backend backend =
DirectoryServer.getBackendWithBaseDN(rebuildConfig.getBaseDN());
@@ -228,7 +235,12 @@
return TaskState.STOPPED_BY_ERROR;
}
-
+ if(tmpDirectory == null)
+ {
+ tmpDirectory = "import-tmp";
+ }
+ rebuildConfig.setTmpDirectory(tmpDirectory);
+ rebuildConfig.setRebuildAll(rebuildAll);
TaskState returnCode = TaskState.COMPLETED_SUCCESSFULLY;
// Launch the rebuild process.
try
diff --git a/opends/src/server/org/opends/server/tools/RebuildIndex.java b/opends/src/server/org/opends/server/tools/RebuildIndex.java
index a408db0..c26b30f 100644
--- a/opends/src/server/org/opends/server/tools/RebuildIndex.java
+++ b/opends/src/server/org/opends/server/tools/RebuildIndex.java
@@ -80,6 +80,8 @@
private StringArgument configFile = null;
private StringArgument baseDNString = null;
private StringArgument indexList = null;
+ private StringArgument tmpDirectory = null;
+ private BooleanArgument rebuildAll = null;
/**
* Processes the command-line arguments and invokes the rebuild process.
@@ -140,7 +142,7 @@
}
// Define the command-line arguments that may be used with this program.
- BooleanArgument displayUsage = null;
+ BooleanArgument displayUsage ;
// Create the command-line argument parser for use with this program.
@@ -187,6 +189,19 @@
argParser.addArgument(indexList);
+ rebuildAll =
+ new BooleanArgument("rebuildAll", null, "rebuildAll",
+ INFO_REBUILDINDEX_DESCRIPTION_REBUILD_ALL.get());
+ argParser.addArgument(rebuildAll);
+
+
+ tmpDirectory =
+ new StringArgument("tmpdirectory", null, "tmpdirectory", false,
+ false, true, INFO_REBUILDINDEX_TEMP_DIR_PLACEHOLDER.get(),
+ "import-tmp",
+ null, INFO_REBUILDINDEX_DESCRIPTION_TEMP_DIRECTORY.get());
+ argParser.addArgument(tmpDirectory);
+
displayUsage =
new BooleanArgument("help", 'H', "help",
INFO_DESCRIPTION_USAGE.get());
@@ -236,7 +251,7 @@
}
- if (indexList.getValues().size() <= 0)
+ if (indexList.getValues().size() <= 0 && !rebuildAll.isPresent())
{
Message message = ERR_REBUILDINDEX_REQUIRES_AT_LEAST_ONE_INDEX.get();
@@ -245,6 +260,13 @@
return 1;
}
+ if(rebuildAll.isPresent() && indexList.isPresent())
+ {
+ Message msg = ERR_REBUILDINDEX_REBUILD_ALL_ERROR.get();
+ err.println(wrapText(msg, MAX_LINE_WIDTH));
+ out.println(argParser.getUsage());
+ return 1;
+ }
return process(argParser, initializeServer, out, err);
}
@@ -424,7 +446,7 @@
ArrayList<Backend> backendList = new ArrayList<Backend>();
ArrayList<BackendCfg> entryList = new ArrayList<BackendCfg>();
ArrayList<List<DN>> dnList = new ArrayList<List<DN>>();
- int code = BackendToolUtils.getBackends(backendList, entryList, dnList);
+ BackendToolUtils.getBackends(backendList, entryList, dnList);
int numBackends = backendList.size();
for (int i=0; i < numBackends; i++)
@@ -498,6 +520,9 @@
return 1;
}
+ rebuildConfig.setRebuildAll(rebuildAll.isPresent());
+ rebuildConfig.setTmpDirectory(tmpDirectory.getValue());
+
// Launch the rebuild process.
int returnCode = 0;
try
@@ -567,6 +592,25 @@
values.add(ByteString.valueOf(s));
}
attributes.add(new LDAPAttribute(ATTR_REBUILD_INDEX, values));
+
+
+ if (tmpDirectory.getValue() != null &&
+ !tmpDirectory.getValue().equals(
+ tmpDirectory.getDefaultValue())) {
+ values = new ArrayList<ByteString>(1);
+ values.add(ByteString.valueOf(tmpDirectory.getValue()));
+ attributes.add(new LDAPAttribute(ATTR_REBUILD_TMP_DIRECTORY, values));
+ }
+
+
+ if (rebuildAll.getValue() != null &&
+ !rebuildAll.getValue().equals(
+ rebuildAll.getDefaultValue())) {
+ values = new ArrayList<ByteString>(1);
+ values.add(ByteString.valueOf(REBUILD_ALL));
+ attributes.add(
+ new LDAPAttribute(ATTR_REBUILD_INDEX, values));
+ }
}
/**
diff --git a/opends/src/server/org/opends/server/util/LDIFReader.java b/opends/src/server/org/opends/server/util/LDIFReader.java
index 48a32ba..e52a697 100644
--- a/opends/src/server/org/opends/server/util/LDIFReader.java
+++ b/opends/src/server/org/opends/server/util/LDIFReader.java
@@ -236,7 +236,7 @@
- private final Entry readEntry(boolean checkSchema, Map<DN, Suffix> map,
+ private Entry readEntry(boolean checkSchema, Map<DN, Suffix> map,
Importer.EntryInformation entryInfo)
throws IOException, LDIFException
{
@@ -244,8 +244,8 @@
{
LinkedList<StringBuilder> lines;
DN entryDN;
- EntryID entryID = null;
- Suffix suffix = null;
+ EntryID entryID;
+ Suffix suffix;
synchronized (this)
{
// Read the set of lines that make up the next entry.
@@ -260,7 +260,14 @@
// Read the DN of the entry and see if it is one that should be included
// in the import.
- entryDN = readDN(lines);
+
+ try
+ {
+ entryDN = readDN(lines);
+ } catch (LDIFException le) {
+ entriesIgnored.incrementAndGet();
+ continue;
+ }
if (entryDN == null)
{
// This should only happen if the LDIF starts with the "version:" line
@@ -904,7 +911,8 @@
// The value did not have a valid base64-encoding.
if (debugEnabled())
{
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
+ TRACER.debugInfo("Base64 decode failed for dn: ",
+ line.substring(pos));
}
Message message =
@@ -925,7 +933,7 @@
{
if (debugEnabled())
{
- TRACER.debugCaught(DebugLogLevel.ERROR, de);
+ TRACER.debugInfo("DN decode failed for: ", dnStr);
}
Message message = ERR_LDIF_INVALID_DN.get(
@@ -940,9 +948,8 @@
{
if (debugEnabled())
{
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
+ TRACER.debugInfo("DN decode failed for: ", dnStr);
}
-
Message message = ERR_LDIF_INVALID_DN.get(
lastEntryLineNumber, line.toString(),
String.valueOf(e));
@@ -972,9 +979,8 @@
{
if (debugEnabled())
{
- TRACER.debugCaught(DebugLogLevel.ERROR, de);
+ TRACER.debugInfo("DN decode failed for: ", line.substring(pos));
}
-
Message message = ERR_LDIF_INVALID_DN.get(
lastEntryLineNumber, line.toString(), de.getMessageObject());
@@ -986,7 +992,7 @@
{
if (debugEnabled())
{
- TRACER.debugCaught(DebugLogLevel.ERROR, e);
+ TRACER.debugInfo("DN decode failed for: ", line.substring(pos));
}
Message message = ERR_LDIF_INVALID_DN.get(
@@ -1103,9 +1109,7 @@
pos++;
}
- String changeTypeString = line.substring(pos);
-
- return changeTypeString;
+ return line.substring(pos);
}
}
@@ -1269,51 +1273,41 @@
// Check to see if any of the attributes in the list have the same set of
// options. If so, then try to add a value to that attribute.
- for (int i = 0; i < attrList.size(); i++) {
- AttributeBuilder a = attrList.get(i);
-
- if (a.optionsEqual(attribute.getOptions()))
- {
- if (a.contains(attributeValue))
- {
- if (! checkSchema)
- {
+ for (AttributeBuilder a : attrList) {
+ if (a.optionsEqual(attribute.getOptions())) {
+ if (a.contains(attributeValue)) {
+ if (!checkSchema) {
// If we're not doing schema checking, then it is possible that
// the attribute type should use case-sensitive matching and the
// values differ in capitalization. Only reject the proposed
// value if we find another value that is exactly the same as the
// one that was provided.
- for (AttributeValue v : a)
- {
- if (v.getValue().equals(attributeValue.getValue()))
- {
+ for (AttributeValue v : a) {
+ if (v.getValue().equals(attributeValue.getValue())) {
Message message = WARN_LDIF_DUPLICATE_ATTR.get(
String.valueOf(entryDN),
lastEntryLineNumber, attrName,
value.toString());
logToRejectWriter(lines, message);
throw new LDIFException(message, lastEntryLineNumber,
- true);
+ true);
}
}
- }
- else
- {
+ } else {
Message message = WARN_LDIF_DUPLICATE_ATTR.get(
String.valueOf(entryDN),
lastEntryLineNumber, attrName,
value.toString());
logToRejectWriter(lines, message);
throw new LDIFException(message, lastEntryLineNumber,
- true);
+ true);
}
}
- if (attrType.isSingleValue() && !a.isEmpty() && checkSchema)
- {
+ if (attrType.isSingleValue() && !a.isEmpty() && checkSchema) {
Message message = ERR_LDIF_MULTIPLE_VALUES_FOR_SINGLE_VALUED_ATTR
.get(String.valueOf(entryDN),
- lastEntryLineNumber, attrName);
+ lastEntryLineNumber, attrName);
logToRejectWriter(lines, message);
throw new LDIFException(message, lastEntryLineNumber, true);
}
@@ -1328,7 +1322,6 @@
AttributeBuilder builder = new AttributeBuilder(attribute, true);
builder.add(attributeValue);
attrList.add(builder);
- return;
}
}
@@ -2356,7 +2349,6 @@
AttributeBuilder builder = new AttributeBuilder(attribute, true);
builder.add(attributeValue);
attrList.add(builder.toAttribute());
- return;
}
}
}
diff --git a/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java b/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java
index 353a67e..8c3107e 100644
--- a/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java
+++ b/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestRebuildJob.java
@@ -22,7 +22,7 @@
* CDDL HEADER END
*
*
- * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Copyright 2006-2009 Sun Microsystems, Inc.
*/
package org.opends.server.backends.jeb;
@@ -32,6 +32,7 @@
import org.testng.annotations.Test;
import static org.testng.Assert.*;
import org.opends.server.TestCaseUtils;
+import org.opends.server.config.ConfigConstants;
import org.opends.server.tasks.TaskUtils;
import static org.opends.server.util.ServerConstants.OC_TOP;
import static org.opends.server.util.ServerConstants.OC_EXTENSIBLE_OBJECT;
@@ -58,8 +59,6 @@
@DataProvider(name = "systemIndexes")
public Object[][] systemIndexes() {
return new Object[][] {
- { "id2subtree" },
- { "id2children" },
{ "dn2id" },
{ "dn2uri" }
};
@@ -83,6 +82,8 @@
return new Object[][] {
{ "id2entry" },
{ "nonindex" },
+ { "id2subtree" },
+ { "id2children" },
{ "mail.nonindex" }
};
}
@@ -221,13 +222,32 @@
}
@Test
- public void testRebuildDependentIndexes() throws Exception
+ public void testRebuildAll() throws Exception
+ {
+ cleanAndLoad(10);
+ RebuildConfig rebuildConfig = new RebuildConfig();
+ rebuildConfig.setBaseDN(baseDNs[0]);
+ rebuildConfig.setRebuildAll(true);
+
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+
+ TaskUtils.disableBackend(beID);
+
+ be.rebuildBackend(rebuildConfig);
+
+ assertEquals(verifyBackend(null), 0);
+
+ TaskUtils.enableBackend(beID);
+ }
+
+
+ @Test
+ public void testRebuildDN2ID() throws Exception
{
cleanAndLoad(10);
RebuildConfig rebuildConfig = new RebuildConfig();
rebuildConfig.setBaseDN(baseDNs[0]);
rebuildConfig.addRebuildIndex("dn2id");
- rebuildConfig.addRebuildIndex("id2children");
be=(BackendImpl) DirectoryServer.getBackend(beID);
diff --git a/opends/tests/unit-tests-testng/src/server/org/opends/server/tasks/TestRebuildTask.java b/opends/tests/unit-tests-testng/src/server/org/opends/server/tasks/TestRebuildTask.java
index d3763b4..6e7e480 100644
--- a/opends/tests/unit-tests-testng/src/server/org/opends/server/tasks/TestRebuildTask.java
+++ b/opends/tests/unit-tests-testng/src/server/org/opends/server/tasks/TestRebuildTask.java
@@ -22,7 +22,7 @@
* CDDL HEADER END
*
*
- * Copyright 2006-2008 Sun Microsystems, Inc.
+ * Copyright 2006-2009 Sun Microsystems, Inc.
*/
package org.opends.server.tasks;
@@ -122,10 +122,7 @@
"ds-task-rebuild-base-dn: " + suffix,
"ds-task-rebuild-index: dn2id",
"ds-task-rebuild-index: dn2uri",
- "ds-task-rebuild-index: id2children",
- "ds-task-rebuild-index: id2subtree",
- "ds-task-rebuild-index: mail",
- "ds-task-rebuild-max-threads: 3"
+ "ds-task-rebuild-index: mail"
),
TaskState.COMPLETED_SUCCESSFULLY
},
@@ -150,6 +147,6 @@
public void testRebuildTask(Entry taskEntry, TaskState expectedState)
throws Exception
{
- testTask(taskEntry, expectedState, 60);
+ testTask(taskEntry, expectedState, 60);
}
}
--
Gitblit v1.10.0