From 3e7410b2ec4ceeef110ed4864d86cd56993512fe Mon Sep 17 00:00:00 2001
From: dugan <dugan@localhost>
Date: Sat, 26 Sep 2009 18:33:49 +0000
Subject: [PATCH] Rebuild index using new import engine.
---
opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java | 1138 +++++++++++++++++++++++++++++++++++++++++++++++++++++++----
1 files changed, 1,051 insertions(+), 87 deletions(-)
diff --git a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
index 8b2a4c0..4f51731 100644
--- a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
@@ -45,6 +45,8 @@
import org.opends.messages.Category;
import org.opends.messages.Severity;
import org.opends.server.admin.std.server.LocalDBBackendCfg;
+import org.opends.server.admin.std.server.LocalDBIndexCfg;
+import org.opends.server.admin.std.meta.LocalDBIndexCfgDefn;
import org.opends.server.backends.jeb.*;
import org.opends.server.config.ConfigException;
import org.opends.server.core.DirectoryServer;
@@ -54,7 +56,7 @@
/**
- * Performs a LDIF import.
+ * Performs LDIF import and rebuild of indexes.
*/
public class Importer
{
@@ -134,8 +136,9 @@
new ConcurrentHashMap<Integer, EntryContainer>();
private final Object synObj = new Object();
+ private final RebuildManager rebuildManager;
- static
+ static
{
if ((dnType = DirectoryServer.getAttributeType("dn")) == null)
{
@@ -143,18 +146,79 @@
}
}
+ private void initialize()
+ {
+
+ }
+
+ private
+ Importer(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg,
+ EnvironmentConfig envConfig) throws IOException,
+ InitializationException, JebException, ConfigException
+ {
+ this.importConfiguration = null;
+ this.threadCount = 1;
+ this.rebuildManager = new RebuildManager(rebuildConfig, cfg);
+ indexCount = rebuildManager.getIndexCount();
+ indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
+ indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
+ File parentDir;
+ if(rebuildConfig.getTmpDirectory() == null)
+ {
+ parentDir = getFileForPath("import-tmp");
+ }
+ else
+ {
+ parentDir = getFileForPath(rebuildConfig.getTmpDirectory());
+ }
+ tempDir = new File(parentDir, cfg.getBackendId());
+ if(!tempDir.exists() && !tempDir.mkdirs())
+ {
+ Message message =
+ ERR_JEB_IMPORT_CREATE_TMPDIR_ERROR.get(String.valueOf(tempDir));
+ throw new IOException(message.toString());
+ }
+ if (tempDir.listFiles() != null)
+ {
+ for (File f : tempDir.listFiles())
+ {
+ f.delete();
+ }
+ }
+ skipDNValidation = true;
+ String propString = System.getProperty(DIRECT_PROPERTY);
+ if(propString != null)
+ {
+ int directSize = Integer.valueOf(propString);
+ directBuffer = ByteBuffer.allocateDirect(directSize);
+ }
+ else
+ {
+ directBuffer = null;
+ }
+ if(envConfig != null)
+ {
+ initializeDBEnv(envConfig);
+ }
+ }
+
+
/**
* Create a new import job with the specified ldif import config.
*
* @param importConfiguration The LDIF import configuration.
- * @param dbCfg The local DB back-end configuration.
+ * @param localDBBackendCfg The local DB back-end configuration.
+ * @param envConfig The JEB environment config.
* @throws IOException If a problem occurs while opening the LDIF file for
* reading.
* @throws InitializationException If a problem occurs during initialization.
*/
- public Importer(LDIFImportConfig importConfiguration, LocalDBBackendCfg dbCfg)
- throws IOException, InitializationException
+ private Importer(LDIFImportConfig importConfiguration,
+ LocalDBBackendCfg localDBBackendCfg,
+ EnvironmentConfig envConfig) throws IOException,
+ InitializationException
{
+ this.rebuildManager = null;
this.importConfiguration = importConfiguration;
if(importConfiguration.getThreadCount() == 0)
{
@@ -164,7 +228,9 @@
{
threadCount = importConfiguration.getThreadCount();
}
- indexCount = dbCfg.listLocalDBIndexes().length + 2;
+ indexCount = localDBBackendCfg.listLocalDBIndexes().length + 2;
+
+
indexWriterList = new ArrayList<IndexFileWriterTask>(indexCount);
indexWriterFutures = new CopyOnWriteArrayList<Future<?>>();
File parentDir;
@@ -177,7 +243,7 @@
parentDir = getFileForPath(importConfiguration.getTmpDirectory());
}
- tempDir = new File(parentDir, dbCfg.getBackendId());
+ tempDir = new File(parentDir, localDBBackendCfg.getBackendId());
if(!tempDir.exists() && !tempDir.mkdirs())
{
Message message =
@@ -202,9 +268,53 @@
{
directBuffer = null;
}
+ initializeDBEnv(envConfig);
}
- private void getBufferSizes(long availMem, int buffers)
+ /**
+ * Return and import LDIF instance using the specified arguments.
+ *
+ * @param importCfg The import config to use.
+ * @param localDBBackendCfg The local DB backend config to use.
+ * @param envCfg The JEB environment config to use.
+ * @return A import LDIF instance.
+ *
+ * @throws IOException If an I/O error occurs.
+ * @throws InitializationException If the instance cannot be initialized.
+ */
+ public static
+ Importer getInstance(LDIFImportConfig importCfg,
+ LocalDBBackendCfg localDBBackendCfg,
+ EnvironmentConfig envCfg)
+ throws IOException, InitializationException
+ {
+ return new Importer(importCfg, localDBBackendCfg, envCfg);
+ }
+
+
+ /**
+ * Return an import rebuild index instance using the specified arguments.
+ *
+ * @param rebuildCfg The rebuild config to use.
+ * @param localDBBackendCfg The local DB backend config to use.
+ * @param envCfg The JEB environment config to use.
+ * @return An import rebuild index instance.
+ *
+ * @throws IOException If an I/O error occurs.
+ * @throws InitializationException If the instance cannot be initialized.
+ * @throws JebException If a JEB exception occurs.
+ * @throws ConfigException If the instance cannot be configured.
+ */
+ public static synchronized
+ Importer getInstance(RebuildConfig rebuildCfg,
+ LocalDBBackendCfg localDBBackendCfg,
+ EnvironmentConfig envCfg)
+ throws IOException, InitializationException, JebException, ConfigException
+ {
+ return new Importer(rebuildCfg, localDBBackendCfg, envCfg);
+ }
+
+ private void getBufferSizes(long availMem, int buffers)
{
long memory = availMem - (MAX_DB_CACHE_SIZE + MAX_DB_LOG_BUFFER_BYTES);
bufferSize = (int) (memory/buffers);
@@ -240,7 +350,6 @@
}
}
-
/**
* Return the suffix instance in the specified map that matches the specified
* DN.
@@ -272,7 +381,7 @@
*
* @throws InitializationException If a problem occurs during calculation.
*/
- public void initialize(EnvironmentConfig envConfig)
+ private void initializeDBEnv(EnvironmentConfig envConfig)
throws InitializationException
{
Message message;
@@ -447,6 +556,33 @@
}
+ /**
+ * Rebuild the indexes using the specified rootcontainer.
+ *
+ * @param rootContainer The rootcontainer to rebuild indexes in.
+ *
+ * @throws ConfigException If a configuration error occurred.
+ * @throws InitializationException If an initialization error occurred.
+ * @throws IOException If an IO error occurred.
+ * @throws JebException If the JEB database had an error.
+ * @throws DatabaseException If a database error occurred.
+ * @throws InterruptedException If an interrupted error occurred.
+ * @throws ExecutionException If an execution error occurred.
+ */
+ public void
+ rebuildIndexes(RootContainer rootContainer) throws ConfigException,
+ InitializationException, IOException, JebException, DatabaseException,
+ InterruptedException, ExecutionException
+ {
+ this.rootContainer = rootContainer;
+ long startTime = System.currentTimeMillis();
+ rebuildManager.initialize();
+ rebuildManager.printStartMessage();
+ rebuildManager.rebuldIndexes();
+ tempDir.delete();
+ rebuildManager.printStopMessage(startTime);
+ }
+
/**
* Import a LDIF using the specified root container.
@@ -493,9 +629,10 @@
float rate = 0;
if (importTime > 0)
rate = 1000f * reader.getEntriesRead() / importTime;
- message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
- importCount.get(), reader.getEntriesIgnored(), reader
- .getEntriesRejected(), migratedCount, importTime / 1000, rate);
+ message = NOTE_JEB_IMPORT_FINAL_STATUS.get(reader.getEntriesRead(),
+ importCount.get(), reader.getEntriesIgnored(),
+ reader.getEntriesRejected(), migratedCount,
+ importTime / 1000, rate);
logError(message);
}
finally
@@ -566,8 +703,11 @@
tasks.add(new MigrateExistingTask());
List<Future<Void>> results = execService.invokeAll(tasks);
- for (Future<Void> result : results)
- assert result.isDone();
+ for (Future<Void> result : results) {
+ if(!result.isDone()) {
+ result.get();
+ }
+ }
tasks.clear();
results.clear();
@@ -588,21 +728,23 @@
}
results = execService.invokeAll(tasks);
for (Future<Void> result : results)
- assert result.isDone();
-
-
+ if(!result.isDone()) {
+ result.get();
+ }
tasks.clear();
results.clear();
tasks.add(new MigrateExcludedTask());
results = execService.invokeAll(tasks);
for (Future<Void> result : results)
- assert result.isDone();
-
-
+ if(!result.isDone()) {
+ result.get();
+ }
stopIndexWriterTasks();
for (Future<?> result : indexWriterFutures)
{
- result.get();
+ if(!result.isDone()) {
+ result.get();
+ }
}
indexWriterList.clear();
indexWriterFutures.clear();
@@ -618,7 +760,7 @@
private void processPhaseTwo() throws InterruptedException
{
SecondPhaseProgressTask progress2Task =
- new SecondPhaseProgressTask(indexMgrList);
+ new SecondPhaseProgressTask(indexMgrList, reader.getEntriesRead());
Timer timer2 = new Timer();
timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
processIndexFiles();
@@ -975,24 +1117,23 @@
Index index;
if((index=attributeIndex.getEqualityIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.EQUALITY));
+ new IndexKey(attributeType, ImportIndexType.EQUALITY));
}
if((index=attributeIndex.getPresenceIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.PRESENCE));
+ new IndexKey(attributeType, ImportIndexType.PRESENCE));
}
if((index=attributeIndex.getSubstringIndex()) != null) {
- int subLen = ((SubstringIndexer)index.indexer).getSubStringLen();
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.SUBSTRING, subLen));
+ new IndexKey(attributeType, ImportIndexType.SUBSTRING));
}
if((index=attributeIndex.getOrderingIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.ORDERING));
+ new IndexKey(attributeType, ImportIndexType.ORDERING));
}
if((index=attributeIndex.getApproximateIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.APPROXIMATE));
+ new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
}
for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
Transaction transaction = null;
@@ -1007,7 +1148,7 @@
if(subIndexes != null) {
for(Index subIndex: subIndexes) {
processAttribute(subIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SUBSTRING));
+ new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
}
}
Collection<Index> sharedIndexes =
@@ -1016,7 +1157,7 @@
if(sharedIndexes !=null) {
for(Index sharedIndex:sharedIndexes) {
processAttribute(sharedIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SHARED));
+ new IndexKey(attributeType, ImportIndexType.EX_SHARED));
}
}
}
@@ -1274,24 +1415,23 @@
Index index;
if((index=attributeIndex.getEqualityIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.EQUALITY));
+ new IndexKey(attributeType, ImportIndexType.EQUALITY));
}
if((index=attributeIndex.getPresenceIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.PRESENCE));
+ new IndexKey(attributeType, ImportIndexType.PRESENCE));
}
if((index=attributeIndex.getSubstringIndex()) != null) {
- int subLen = ((SubstringIndexer)index.indexer).getSubStringLen();
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.SUBSTRING, subLen));
+ new IndexKey(attributeType, ImportIndexType.SUBSTRING));
}
if((index=attributeIndex.getOrderingIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType, IndexType.ORDERING));
+ new IndexKey(attributeType, ImportIndexType.ORDERING));
}
if((index=attributeIndex.getApproximateIndex()) != null) {
processAttribute(index, entry, entryID,
- new IndexKey(attributeType,IndexType.APPROXIMATE));
+ new IndexKey(attributeType, ImportIndexType.APPROXIMATE));
}
for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
Transaction transaction = null;
@@ -1306,7 +1446,7 @@
if(subIndexes != null) {
for(Index subIndex: subIndexes) {
processAttribute(subIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SUBSTRING));
+ new IndexKey(attributeType, ImportIndexType.EX_SUBSTRING));
}
}
Collection<Index> sharedIndexes =
@@ -1315,7 +1455,7 @@
if(sharedIndexes !=null) {
for(Index sharedIndex:sharedIndexes) {
processAttribute(sharedIndex, entry, entryID,
- new IndexKey(attributeType, IndexType.EX_SHARED));
+ new IndexKey(attributeType, ImportIndexType.EX_SHARED));
}
}
}
@@ -1346,8 +1486,8 @@
{
IndexKey indexKey = e.getKey();
IndexBuffer indexBuffer = e.getValue();
- IndexType indexType = indexKey.getIndexType();
- if(indexType.equals(IndexType.DN))
+ ImportIndexType indexType = indexKey.getIndexType();
+ if(indexType.equals(ImportIndexType.DN))
{
indexBuffer.setComparator(dnComparator);
}
@@ -1413,7 +1553,7 @@
DatabaseContainer dn2id = suffix.getDN2ID();
byte[] dnBytes = StaticUtils.getBytes(dn.toNormalizedString());
int id = processKey(dn2id, dnBytes, entryID, dnComparator,
- new IndexKey(dnType, IndexType.DN), true);
+ new IndexKey(dnType, ImportIndexType.DN), true);
idECMap.putIfAbsent(id, suffix.getEntryContainer());
}
@@ -2070,7 +2210,8 @@
*/
public Void call() throws Exception
{
- if (importConfiguration.isCancelled())
+ if (importConfiguration != null &&
+ importConfiguration.isCancelled())
{
return null;
}
@@ -2100,7 +2241,7 @@
{
return;
}
- if(indexKey.getIndexType().equals(IndexType.DN))
+ if(indexKey.getIndexType().equals(ImportIndexType.DN))
{
isDN = true;
}
@@ -2522,6 +2663,837 @@
}
}
+
+ /**
+ * The rebuild manager handles all rebuild index related tasks.
+ */
+ class RebuildManager extends ImportTask {
+
+ private final RebuildConfig rebuildConfig;
+ private final LocalDBBackendCfg cfg;
+ private final Map<IndexKey, Index> indexMap =
+ new LinkedHashMap<IndexKey, Index>();
+ private final Map<IndexKey, Collection<Index>> extensibleIndexMap =
+ new LinkedHashMap<IndexKey, Collection<Index>>();
+ private DN2ID dn2id = null;
+ private DN2URI dn2uri = null;
+ private long totalEntries =0;
+ private final AtomicLong entriesProcessed = new AtomicLong(0);
+ private Suffix suffix = null;
+ private final boolean rebuildAll;
+ private EntryContainer ec;
+
+
+ /**
+ * Create an instance of the rebuild index manager using the specified
+ * parameters.
+ *
+ * @param rebuildConfig The rebuild configuration to use.
+ * @param cfg The local DB configuration to use.
+ */
+ public RebuildManager(RebuildConfig rebuildConfig, LocalDBBackendCfg cfg)
+ {
+ this.rebuildConfig = rebuildConfig;
+ this.cfg = cfg;
+ this.rebuildAll = rebuildConfig.isRebuildAll();
+ }
+
+ /**
+ * Initialize a rebuild manager to start rebuilding indexes.
+ *
+ * @throws ConfigException If an configuration error occurred.
+ * @throws InitializationException If an initialization error occurred.
+ */
+ public void initialize() throws ConfigException, InitializationException
+ {
+ ec = rootContainer.getEntryContainer(rebuildConfig.getBaseDN());
+ suffix = Suffix.createSuffixContext(ec, null, null, null);
+ if(suffix == null)
+ {
+ Message msg = ERR_JEB_REBUILD_SUFFIX_ERROR.get(rebuildConfig.
+ getBaseDN().toString());
+ throw new InitializationException(msg);
+ }
+ }
+
+ /**
+ * Print start message.
+ *
+ * @throws DatabaseException If an database error occurred.
+ */
+ public void printStartMessage() throws DatabaseException
+ {
+ StringBuilder sb = new StringBuilder();
+ List<String> rebuildList = rebuildConfig.getRebuildList();
+ for(String index : rebuildList)
+ {
+ if(sb.length() > 0)
+ {
+ sb.append(", ");
+ }
+ sb.append(index);
+ }
+ totalEntries = suffix.getID2Entry().getRecordCount();
+ Message message = NOTE_JEB_REBUILD_START.get(sb.toString(), totalEntries);
+ if(rebuildAll) {
+ message = NOTE_JEB_REBUILD_ALL_START.get(totalEntries);
+ }
+ logError(message);
+ }
+
+ /**
+ * Print stop message.
+ *
+ * @param startTime The time the rebuild started.
+ */
+ public void printStopMessage(long startTime)
+ {
+ long finishTime = System.currentTimeMillis();
+ long totalTime = (finishTime - startTime);
+ float rate = 0;
+ if (totalTime > 0)
+ {
+ rate = 1000f* entriesProcessed.get() / totalTime;
+ }
+ Message message =
+ NOTE_JEB_REBUILD_FINAL_STATUS.get(entriesProcessed.get(),
+ totalTime/1000, rate);
+ logError(message);
+ }
+
+
+ /**
+ * {@inheritDoc}
+ */
+ public Void call() throws Exception
+ {
+ ID2Entry id2entry = ec.getID2Entry();
+ Cursor cursor = id2entry.openCursor(null, CursorConfig.READ_COMMITTED);
+ DatabaseEntry key = new DatabaseEntry();
+ DatabaseEntry data = new DatabaseEntry();
+ LockMode lockMode = LockMode.DEFAULT;
+ OperationStatus status;
+ try {
+ for (status = cursor.getFirst(key, data, lockMode);
+ status == OperationStatus.SUCCESS;
+ status = cursor.getNext(key, data, lockMode))
+ {
+ EntryID entryID = new EntryID(key);
+ Entry entry = ID2Entry.entryFromDatabase(
+ ByteString.wrap(data.getData()),
+ ec.getRootContainer().getCompressedSchema());
+ processEntry(entry, entryID);
+ entriesProcessed.getAndIncrement();
+ }
+ flushIndexBuffers();
+ cursor.close();
+ } catch (Exception e) {
+ System.out.println("here");
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * Perform the index rebuild.
+ *
+ * @throws DatabaseException If an database error occurred.
+ * @throws InterruptedException If an interrupted error occurred.
+ * @throws ExecutionException If an Excecution error occurred.
+ * @throws JebException If an JEB error occurred.
+ */
+ public void rebuldIndexes() throws DatabaseException, InterruptedException,
+ ExecutionException, JebException
+ {
+ processPhaseOne();
+ processPhaseTwo();
+ setIndexesTrusted();
+ }
+
+ private void setIndexesTrusted() throws JebException
+ {
+ try {
+ suffix.setIndexesTrusted();
+ }
+ catch (DatabaseException ex)
+ {
+ Message message =
+ NOTE_JEB_IMPORT_LDIF_TRUSTED_FAILED.get(ex.getMessage());
+ throw new JebException(message);
+ }
+ }
+
+ private void processPhaseOne() throws DatabaseException,
+ InterruptedException, ExecutionException {
+ if(rebuildAll)
+ {
+ clearAllIndexes();
+ }
+ else
+ {
+ clearRebuildListIndexes();
+ }
+ initializeIndexBuffers(threadCount);
+ RBFirstPhaseProgressTask progressTask = new RBFirstPhaseProgressTask();
+ Timer timer = new Timer();
+ timer.scheduleAtFixedRate(progressTask, TIMER_INTERVAL, TIMER_INTERVAL);
+ indexProcessService = Executors.newFixedThreadPool(2 * indexCount);
+ sortService = Executors.newFixedThreadPool(threadCount);
+ ExecutorService execService = Executors.newFixedThreadPool(threadCount);
+ List<Callable<Void>> tasks = new ArrayList<Callable<Void>>(threadCount);
+ for (int i = 0; i < threadCount; i++)
+ {
+ tasks.add(this);
+ }
+ List<Future<Void>> results = execService.invokeAll(tasks);
+ for (Future<Void> result : results) {
+ if(!result.isDone()) {
+ result.get();
+ }
+ }
+ stopIndexWriterTasks();
+ for (Future<?> result : indexWriterFutures)
+ {
+ if(!result.isDone()) {
+ result.get();
+ }
+ }
+ tasks.clear();
+ results.clear();
+ execService.shutdown();
+ freeBufferQueue.clear();
+ sortService.shutdown();
+ timer.cancel();
+ }
+
+
+ private void processPhaseTwo() throws InterruptedException
+ {
+ SecondPhaseProgressTask progress2Task =
+ new SecondPhaseProgressTask(indexMgrList, entriesProcessed.get());
+ Timer timer2 = new Timer();
+ timer2.scheduleAtFixedRate(progress2Task, TIMER_INTERVAL, TIMER_INTERVAL);
+ processIndexFiles();
+ timer2.cancel();
+ }
+
+ private int getIndexCount() throws ConfigException, JebException
+ {
+ int indexCount;
+ if(!rebuildAll)
+ {
+ indexCount = getRebuildListIndexCount(cfg);
+ }
+ else
+ {
+ indexCount = getAllIndexesCount(cfg);
+ }
+ return indexCount;
+ }
+
+ private int getAllIndexesCount(LocalDBBackendCfg cfg)
+ {
+ int indexCount = cfg.listLocalDBIndexes().length;
+ indexCount += cfg.listLocalDBVLVIndexes().length;
+ indexCount += 4;
+ return indexCount;
+ }
+
+ private int getRebuildListIndexCount(LocalDBBackendCfg cfg)
+ throws JebException, ConfigException
+ {
+ int indexCount = 0;
+ List<String> rebuildList = rebuildConfig.getRebuildList();
+ if(!rebuildList.isEmpty())
+ {
+ for (String index : rebuildList)
+ {
+ String lowerName = index.toLowerCase();
+ if (lowerName.equals("dn2id"))
+ {
+ indexCount += 3;
+ }
+ else if (lowerName.equals("dn2uri"))
+ {
+ indexCount++;
+ }
+ else if (lowerName.startsWith("vlv."))
+ {
+ if(lowerName.length() < 5)
+ {
+ Message msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
+ throw new JebException(msg);
+ }
+ indexCount++;
+ } else if(lowerName.equals("id2subtree") ||
+ lowerName.equals("id2children"))
+ {
+ Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ else
+ {
+ String[] attrIndexParts = lowerName.split("\\.");
+ if((attrIndexParts.length <= 0) || (attrIndexParts.length > 3))
+ {
+ Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(attrIndexParts[0]);
+ if (attrType == null)
+ {
+ Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ if(attrIndexParts.length != 1)
+ {
+ if(attrIndexParts.length == 2)
+ {
+ if(attrIndexParts[1].equals("presence"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("equality"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("substring"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("ordering"))
+ {
+ indexCount++;
+ }
+ else if(attrIndexParts[1].equals("approximate"))
+ {
+ indexCount++;
+ } else {
+ Message msg =
+ ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ }
+ else
+ {
+ boolean found = false;
+ String s = attrIndexParts[1] + "." + attrIndexParts[2];
+ for (String idx : cfg.listLocalDBIndexes())
+ {
+ LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
+ if (indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
+ {
+ Set<String> extensibleRules =
+ indexCfg.getIndexExtensibleMatchingRule();
+ for(String exRule : extensibleRules)
+ {
+ if(exRule.equalsIgnoreCase(s))
+ {
+ found = true;
+ break;
+ }
+ }
+ }
+ if(found)
+ {
+ break;
+ }
+ }
+ if(!found) {
+ Message msg =
+ ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
+ throw new JebException(msg);
+ }
+ indexCount++;
+ }
+ }
+ else
+ {
+ for (String idx : cfg.listLocalDBIndexes())
+ {
+ if(!idx.equalsIgnoreCase(index))
+ {
+ continue;
+ }
+ LocalDBIndexCfg indexCfg = cfg.getLocalDBIndex(idx);
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.EQUALITY))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.ORDERING))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.PRESENCE))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.SUBSTRING))
+ {
+ indexCount++;
+ }
+ if(indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.APPROXIMATE))
+ {
+ indexCount++;
+ }
+ if (indexCfg.getIndexType().
+ contains(LocalDBIndexCfgDefn.IndexType.EXTENSIBLE))
+ {
+ Set<String> extensibleRules =
+ indexCfg.getIndexExtensibleMatchingRule();
+ boolean shared = false;
+ for(String exRule : extensibleRules)
+ {
+ if(exRule.endsWith(".sub"))
+ {
+ indexCount++;
+ }
+ else
+ {
+ if(!shared)
+ {
+ shared=true;
+ indexCount++;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return indexCount;
+ }
+
+ private void clearRebuildListIndexes() throws DatabaseException
+ {
+ List<String> rebuildList = rebuildConfig.getRebuildList();
+ if(!rebuildList.isEmpty())
+ {
+ for (String index : rebuildList)
+ {
+ String lowerName = index.toLowerCase();
+ if (lowerName.equals("dn2id"))
+ {
+ clearDN2IDIndexes(ec);
+ }
+ else if (lowerName.equals("dn2uri"))
+ {
+ clearDN2URI(ec);
+ }
+ else if (lowerName.startsWith("vlv."))
+ {
+ clearVLVIndex(lowerName.substring(4), ec);
+ }
+ else
+ {
+ String[] attrIndexParts = lowerName.split("\\.");
+ AttributeType attrType =
+ DirectoryServer.getAttributeType(attrIndexParts[0]);
+ AttributeIndex attrIndex = ec.getAttributeIndex(attrType);
+
+ if(attrIndexParts.length != 1)
+ {
+ Index partialAttrIndex;
+ if(attrIndexParts[1].equals("presence"))
+ {
+ partialAttrIndex = attrIndex.getPresenceIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.PRESENCE);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("equality"))
+ {
+ partialAttrIndex = attrIndex.getEqualityIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.EQUALITY);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("substring"))
+ {
+ partialAttrIndex = attrIndex.getSubstringIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.SUBSTRING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("ordering"))
+ {
+ partialAttrIndex = attrIndex.getOrderingIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.ORDERING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else if(attrIndexParts[1].equals("approximate"))
+ {
+ partialAttrIndex = attrIndex.getApproximateIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.APPROXIMATE);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ else
+ {
+ String dbPart = "shared";
+ if(attrIndexParts[2].startsWith("sub"))
+ {
+ dbPart = "substring";
+ }
+ StringBuilder nameBldr = new StringBuilder();
+ nameBldr.append(ec.getDatabasePrefix());
+ nameBldr.append("_");
+ nameBldr.append(attrIndexParts[0]);
+ nameBldr.append(".");
+ nameBldr.append(attrIndexParts[1]);
+ nameBldr.append(".");
+ nameBldr.append(dbPart);
+ String indexName = nameBldr.toString();
+ Map<String,Collection<Index>> extensibleMap =
+ attrIndex.getExtensibleIndexes();
+ if(!extensibleMap.isEmpty()) {
+ Collection<Index> subIndexes =
+ attrIndex.getExtensibleIndexes().get(
+ EXTENSIBLE_INDEXER_ID_SUBSTRING);
+ if(subIndexes != null) {
+ for(Index subIndex : subIndexes) {
+ String name = subIndex.getName();
+ if(name.equalsIgnoreCase(indexName))
+ {
+ ec.clearDatabase(subIndex);
+ Collection<Index> substring = new ArrayList<Index>();
+ substring.add(subIndex);
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SUBSTRING),substring);
+ break;
+ }
+ }
+ Collection<Index> sharedIndexes = attrIndex.
+ getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
+ if(sharedIndexes !=null) {
+ for(Index sharedIndex : sharedIndexes) {
+ String name = sharedIndex.getName();
+ if(name.equalsIgnoreCase(indexName))
+ {
+ ec.clearDatabase(sharedIndex);
+ Collection<Index> shared = new ArrayList<Index>();
+ shared.add(sharedIndex);
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SHARED), shared);
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ clearAttributeIndexes(attrIndex, attrType, ec);
+ }
+ }
+ }
+ }
+ }
+
+
+ private void clearAllIndexes() throws DatabaseException
+ {
+ for(Map.Entry<AttributeType, AttributeIndex> mapEntry :
+ suffix.getAttrIndexMap().entrySet()) {
+ AttributeType attributeType = mapEntry.getKey();
+ AttributeIndex attributeIndex = mapEntry.getValue();
+ clearAttributeIndexes(attributeIndex, attributeType, ec);
+ }
+ for(VLVIndex vlvIndex : suffix.getEntryContainer().getVLVIndexes()) {
+ ec.clearDatabase(vlvIndex);
+ }
+ clearDN2IDIndexes(ec);
+ if(ec.getDN2URI() != null)
+ {
+ clearDN2URI(ec);
+ }
+ }
+
+ private void clearVLVIndex(String name, EntryContainer ec)
+ throws DatabaseException
+ {
+ VLVIndex vlvIndex = ec.getVLVIndex(name);
+ ec.clearDatabase(vlvIndex);
+ }
+
+ private void clearDN2URI(EntryContainer ec) throws DatabaseException
+ {
+ ec.clearDatabase(ec.getDN2URI());
+ dn2uri = ec.getDN2URI();
+ }
+
+ private void clearDN2IDIndexes(EntryContainer ec) throws DatabaseException
+ {
+ ec.clearDatabase(ec.getDN2ID());
+ ec.clearDatabase(ec.getID2Children());
+ ec.clearDatabase(ec.getID2Subtree());
+ dn2id = ec.getDN2ID();
+ }
+
+ private void clearAttributeIndexes(AttributeIndex attrIndex,
+ AttributeType attrType, EntryContainer ec)
+ throws DatabaseException
+ {
+ Index partialAttrIndex;
+ if(attrIndex.getSubstringIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getSubstringIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.SUBSTRING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ if(attrIndex.getOrderingIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getOrderingIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.ORDERING);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ if(attrIndex.getEqualityIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getEqualityIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.EQUALITY);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ if(attrIndex.getPresenceIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getPresenceIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.PRESENCE);
+ indexMap.put(indexKey, partialAttrIndex);
+
+ }
+ if(attrIndex.getApproximateIndex() != null)
+ {
+ partialAttrIndex = attrIndex.getApproximateIndex();
+ ec.clearDatabase(partialAttrIndex);
+ IndexKey indexKey =
+ new IndexKey(attrType, ImportIndexType.APPROXIMATE);
+ indexMap.put(indexKey, partialAttrIndex);
+ }
+ Map<String,Collection<Index>> extensibleMap =
+ attrIndex.getExtensibleIndexes();
+ if(!extensibleMap.isEmpty()) {
+ Collection<Index> subIndexes =
+ attrIndex.getExtensibleIndexes().get(
+ EXTENSIBLE_INDEXER_ID_SUBSTRING);
+ if(subIndexes != null) {
+ for(Index subIndex : subIndexes) {
+ ec.clearDatabase(subIndex);
+ }
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SUBSTRING), subIndexes);
+ }
+ Collection<Index> sharedIndexes =
+ attrIndex.getExtensibleIndexes().get(EXTENSIBLE_INDEXER_ID_SHARED);
+ if(sharedIndexes !=null) {
+ for(Index sharedIndex : sharedIndexes) {
+ ec.clearDatabase(sharedIndex);
+ }
+ extensibleIndexMap.put(new IndexKey(attrType,
+ ImportIndexType.EX_SHARED), sharedIndexes);
+ }
+ }
+ }
+
+
+ private
+ void processEntry(Entry entry, EntryID entryID) throws DatabaseException,
+ ConfigException, DirectoryException, JebException
+ {
+ if(dn2id != null)
+ {
+ processDN2ID(suffix, entry.getDN(), entryID);
+ }
+ if(dn2uri != null)
+ {
+ processDN2URI(suffix, null, entry);
+ }
+ processIndexes(entry, entryID);
+ processExtensibleIndexes(entry, entryID);
+ processVLVIndexes(entry, entryID);
+ }
+
+ private void processVLVIndexes(Entry entry, EntryID entryID)
+ throws DatabaseException, JebException, DirectoryException
+ {
+ for(VLVIndex vlvIdx : suffix.getEntryContainer().getVLVIndexes()) {
+ Transaction transaction = null;
+ vlvIdx.addEntry(transaction, entryID, entry);
+ }
+ }
+
+
+ private void processExtensibleIndexes(Entry entry, EntryID entryID) throws
+ DatabaseException, DirectoryException, JebException, ConfigException
+ {
+ for(Map.Entry<IndexKey, Collection<Index>> mapEntry :
+ this.extensibleIndexMap.entrySet()) {
+ IndexKey key = mapEntry.getKey();
+ AttributeType attrType = key.getType();
+ if(entry.hasAttribute(attrType)) {
+ Collection<Index> indexes = mapEntry.getValue();
+ for(Index index : indexes) {
+ processAttribute(index, entry, entryID, key);
+ }
+ }
+ }
+ }
+
+ private void
+ processIndexes(Entry entry, EntryID entryID) throws
+ DatabaseException, DirectoryException, JebException, ConfigException
+ {
+
+ for(Map.Entry<IndexKey, Index> mapEntry :
+ indexMap.entrySet()) {
+ IndexKey key = mapEntry.getKey();
+ AttributeType attrType = key.getType();
+ if(entry.hasAttribute(attrType)) {
+ ImportIndexType indexType = key.getIndexType();
+ Index index = mapEntry.getValue();
+ if(indexType == ImportIndexType.SUBSTRING)
+ {
+ processAttribute(index, entry, entryID,
+ new IndexKey(attrType, ImportIndexType.SUBSTRING));
+ }
+ else
+ {
+ processAttribute(index, entry, entryID,
+ new IndexKey(attrType, indexType));
+ }
+ }
+ }
+ }
+
+ /**
+ * Return the number of entries processed by the rebuild manager.
+ *
+ * @return The number of entries processed.
+ */
+ public long getEntriesProcess()
+ {
+ return this.entriesProcessed.get();
+ }
+
+ /**
+ * Return the total number of entries to process by the rebuild manager.
+ *
+ * @return The total number for entries to process.
+ */
+ public long getTotEntries()
+ {
+ return this.totalEntries;
+ }
+ }
+
+ /**
+ * This class reports progress of the rebuild job at fixed intervals.
+ */
+ class RBFirstPhaseProgressTask extends TimerTask
+ {
+ /**
+ * The number of records that had been processed at the time of the
+ * previous progress report.
+ */
+ private long previousProcessed = 0;
+
+ /**
+ * The time in milliseconds of the previous progress report.
+ */
+ private long previousTime;
+
+ /**
+ * The environment statistics at the time of the previous report.
+ */
+ private EnvironmentStats prevEnvStats;
+
+ /**
+ * Create a new verify progress task.
+ * @throws DatabaseException An error occurred while accessing the JE
+ * database.
+ */
+ public RBFirstPhaseProgressTask() throws DatabaseException
+ {
+ previousTime = System.currentTimeMillis();
+ prevEnvStats =
+ rootContainer.getEnvironmentStats(new StatsConfig());
+ }
+
+ /**
+ * The action to be performed by this timer task.
+ */
+ public void run()
+ {
+ long latestTime = System.currentTimeMillis();
+ long deltaTime = latestTime - previousTime;
+
+ if (deltaTime == 0)
+ {
+ return;
+ }
+ long currentRBProcessed = rebuildManager.getEntriesProcess();
+ long deltaCount = (currentRBProcessed - previousProcessed);
+ float rate = 1000f*deltaCount / deltaTime;
+ float completed = 0;
+ if(rebuildManager.getTotEntries() > 0)
+ {
+ completed = 100f*currentRBProcessed / rebuildManager.getTotEntries();
+ }
+ Message message = NOTE_JEB_REBUILD_PROGRESS_REPORT.get(
+ completed, currentRBProcessed, rebuildManager.getTotEntries(), rate);
+ logError(message);
+ try
+ {
+ Runtime runtime = Runtime.getRuntime();
+ long freeMemory = runtime.freeMemory() / MB;
+ EnvironmentStats envStats =
+ rootContainer.getEnvironmentStats(new StatsConfig());
+ long nCacheMiss =
+ envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
+
+ float cacheMissRate = 0;
+ if (deltaCount > 0)
+ {
+ cacheMissRate = nCacheMiss/(float)deltaCount;
+ }
+ message = NOTE_JEB_REBUILD_CACHE_AND_MEMORY_REPORT.get(
+ freeMemory, cacheMissRate);
+ logError(message);
+ prevEnvStats = envStats;
+ }
+ catch (DatabaseException e)
+ {
+
+ }
+ previousProcessed = currentRBProcessed;
+ previousTime = latestTime;
+ }
+ }
+
+
/**
* This class reports progress of the import job at fixed intervals.
*/
@@ -2665,7 +3637,7 @@
/**
* This class reports progress of the import job at fixed intervals.
*/
- private final class SecondPhaseProgressTask extends TimerTask
+ class SecondPhaseProgressTask extends TimerTask
{
/**
* The number of entries that had been read at the time of the
@@ -2687,16 +3659,19 @@
private boolean evicting = false;
private final List<IndexManager> indexMgrList;
-
+ private long latestCount;
/**
* Create a new import progress task.
* @param indexMgrList List of index managers.
+ * @param latestCount The latest count of entries processed in phase one.
*/
- public SecondPhaseProgressTask (List<IndexManager> indexMgrList)
+ public SecondPhaseProgressTask (List<IndexManager> indexMgrList,
+ long latestCount)
{
previousTime = System.currentTimeMillis();
this.indexMgrList = indexMgrList;
+ this.latestCount = latestCount;
try
{
previousStats =
@@ -2715,7 +3690,6 @@
@Override
public void run()
{
- long latestCount = reader.getEntriesRead() + 0;
long deltaCount = (latestCount - previousCount);
long latestTime = System.currentTimeMillis();
long deltaTime = latestTime - previousTime;
@@ -2847,7 +3821,7 @@
* This class defines the individual index type available.
*
*/
- public enum IndexType {
+ public enum ImportIndexType {
/**
* The DN index type.
**/
@@ -2886,9 +3860,13 @@
/**
* The extensible shared index type.
**/
- EX_SHARED
- }
+ EX_SHARED,
+ /**
+ * The vlv index type.
+ */
+ VLV
+ }
/**
* This class is used as an index key for hash maps that need to
@@ -2901,22 +3879,8 @@
public class IndexKey {
private final AttributeType type;
- private final IndexType indexType;
- private byte[] keyBytes = null;
+ private final ImportIndexType indexType;
- /**
- * Create index key instance using the specified attribute type, index type
- * and sub-string length. Used only for sub-string indexes.
- *
- * @param type The attribute type.
- * @param indexType The index type.
- * @param subLen The sub-string length.
- */
- IndexKey(AttributeType type, IndexType indexType, int subLen)
- {
- this(type, indexType);
- keyBytes = new byte[subLen];
- }
/**
* Create index key instance using the specified attribute type, index type.
@@ -2924,31 +3888,31 @@
* @param type The attribute type.
* @param indexType The index type.
*/
- IndexKey(AttributeType type, IndexType indexType)
+ IndexKey(AttributeType type, ImportIndexType indexType)
{
this.type = type;
this.indexType = indexType;
}
- /**
- * An equals method that uses both the attribute type and the index type.
- *
- * @param obj the object to compare.
- * @return <CODE>true</CODE> if the objects are equal.
- */
- public boolean equals(Object obj)
- {
- boolean returnCode = false;
- if (obj instanceof IndexKey) {
- IndexKey oKey = (IndexKey) obj;
- if(type.equals(oKey.getType()) &&
- indexType.equals(oKey.getIndexType()))
- {
- returnCode = true;
- }
- }
- return returnCode;
+ /**
+ * An equals method that uses both the attribute type and the index type.
+ *
+ * @param obj the object to compare.
+ * @return <CODE>true</CODE> if the objects are equal.
+ */
+ public boolean equals(Object obj)
+ {
+ boolean returnCode = false;
+ if (obj instanceof IndexKey) {
+ IndexKey oKey = (IndexKey) obj;
+ if(type.equals(oKey.getType()) &&
+ indexType.equals(oKey.getIndexType()))
+ {
+ returnCode = true;
+ }
}
+ return returnCode;
+ }
/**
* A hash code method that adds the hash codes of the attribute type and
@@ -2975,7 +3939,7 @@
* Return the index type.
* @return The index type.
*/
- public IndexType getIndexType()
+ public ImportIndexType getIndexType()
{
return indexType;
}
@@ -2990,7 +3954,7 @@
public String getName()
{
return type.getPrimaryName() + "." +
- StaticUtils.toLowerCase(indexType.name());
+ StaticUtils.toLowerCase(indexType.name());
}
}
}
--
Gitblit v1.10.0