From a465378dd21226dd26665300c5aadb1eedef5caa Mon Sep 17 00:00:00 2001
From: Jean-Noel Rouvignac <jean-noel.rouvignac@forgerock.com>
Date: Wed, 11 Mar 2015 13:37:19 +0000
Subject: [PATCH] OPENDJ-1708 Persistit: no rebuild-index support
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ImportIDSet.java | 89 ++++++---
opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/Importer.java | 45 +---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/Importer.java | 64 +++---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexOutputBuffer.java | 178 ++++++++++---------
opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/IndexOutputBuffer.java | 20 +-
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexInputBuffer.java | 35 +--
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryIDSet.java | 36 +--
opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/ImportIDSet.java | 25 +-
8 files changed, 236 insertions(+), 256 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/ImportIDSet.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/ImportIDSet.java
index f3591f8..f909237 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/ImportIDSet.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/ImportIDSet.java
@@ -67,13 +67,6 @@
this.doCount = doCount;
}
- /** Create an empty import instance. */
- public ImportIDSet()
- {
- this.limit = -1;
- this.doCount = false;
- }
-
/**
* Clear the set so it can be reused again. The boolean indexParam specifies
* if the index parameters should be cleared also.
@@ -113,16 +106,16 @@
/**
* Add the specified long value to an import ID set.
*
- * @param l The long value to add to an import ID set.
+ * @param entryID The long value to add to an import ID set.
*/
- void addEntryID(long l) {
+ void addEntryID(long entryID) {
if(!isDefined()) {
if(doCount) {
undefinedSize++;
}
return;
}
- if (l < 0 || (isDefined() && count + 1 > limit))
+ if (entryID < 0 || (isDefined() && count + 1 > limit))
{
setUndefined();
if(doCount) {
@@ -132,7 +125,7 @@
}
count = 0;
} else {
- add(l);
+ add(entryID);
}
}
@@ -346,17 +339,17 @@
return count;
}
- private boolean add(long v)
+ private boolean add(long entryID)
{
resize(count+1);
- if (count == 0 || v > array[count-1])
+ if (count == 0 || entryID > array[count-1])
{
- array[count++] = v;
+ array[count++] = entryID;
return true;
}
- int pos = binarySearch(array, count, v);
+ int pos = binarySearch(array, count, entryID);
if (pos >=0)
{
return false;
@@ -368,7 +361,7 @@
pos = -(pos+1);
System.arraycopy(array, pos, array, pos+1, count-pos);
- array[pos] = v;
+ array[pos] = entryID;
count++;
return true;
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/Importer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/Importer.java
index a7d502d..13a3901 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/Importer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/Importer.java
@@ -1796,7 +1796,6 @@
private final DatabaseEntry dbKey, dbValue;
private final int cacheSize;
private final Map<Integer, DNState> dnStateMap = new HashMap<Integer, DNState>();
- private final Map<Integer, Index> indexMap = new HashMap<Integer, Index>();
private final Semaphore permits;
private final int maxPermits;
private final AtomicLong bytesRead = new AtomicLong();
@@ -2041,7 +2040,7 @@
}
else if (b.compare(key, indexID) != 0)
{
- addToDB(insertIDSet, deleteIDSet, indexID);
+ addToDB(indexID, insertIDSet, deleteIDSet);
keyCount.incrementAndGet();
indexID = b.getIndexID();
@@ -2088,7 +2087,7 @@
if (key != null)
{
- addToDB(insertIDSet, deleteIDSet, indexID);
+ addToDB(indexID, insertIDSet, deleteIDSet);
}
}
return null;
@@ -2104,40 +2103,30 @@
}
}
- private void addToDB(ImportIDSet insertSet, ImportIDSet deleteSet,
- int indexID) throws DirectoryException
+ private void addToDB(int indexID, ImportIDSet insertSet, ImportIDSet deleteSet) throws DirectoryException
{
- if (!indexMgr.isDN2ID())
+ if (indexMgr.isDN2ID())
+ {
+ addDN2ID(indexID, insertSet);
+ }
+ else
{
if (deleteSet.size() > 0 || !deleteSet.isDefined())
{
dbKey.setData(deleteSet.getKey().array(), 0, deleteSet.getKey().limit());
final Index index = idContainerMap.get(indexID);
index.delete(dbKey, deleteSet, dbValue);
- if (!indexMap.containsKey(indexID))
- {
- indexMap.put(indexID, index);
- }
}
if (insertSet.size() > 0 || !insertSet.isDefined())
{
dbKey.setData(insertSet.getKey().array(), 0, insertSet.getKey().limit());
final Index index = idContainerMap.get(indexID);
index.insert(dbKey, insertSet, dbValue);
- if (!indexMap.containsKey(indexID))
- {
- indexMap.put(indexID, index);
- }
}
}
- else
- {
- addDN2ID(insertSet, indexID);
- }
}
- private void addDN2ID(ImportIDSet record, Integer indexID)
- throws DirectoryException
+ private void addDN2ID(int indexID, ImportIDSet record) throws DirectoryException
{
DNState dnState;
if (!dnStateMap.containsKey(indexID))
@@ -2635,32 +2624,32 @@
deleteKeyCount = 0;
}
- private void insertOrDeleteKey(IndexOutputBuffer indexBuffer, int i)
+ private void insertOrDeleteKey(IndexOutputBuffer indexBuffer, int position)
{
- if (indexBuffer.isInsertRecord(i))
+ if (indexBuffer.isInsertRecord(position))
{
- indexBuffer.writeID(insertByteStream, i);
+ indexBuffer.writeEntryID(insertByteStream, position);
insertKeyCount++;
}
else
{
- indexBuffer.writeID(deleteByteStream, i);
+ indexBuffer.writeEntryID(deleteByteStream, position);
deleteKeyCount++;
}
}
- private void insertOrDeleteKeyCheckEntryLimit(IndexOutputBuffer indexBuffer, int i)
+ private void insertOrDeleteKeyCheckEntryLimit(IndexOutputBuffer indexBuffer, int position)
{
- if (indexBuffer.isInsertRecord(i))
+ if (indexBuffer.isInsertRecord(position))
{
if (insertKeyCount++ <= indexMgr.getLimit())
{
- indexBuffer.writeID(insertByteStream, i);
+ indexBuffer.writeEntryID(insertByteStream, position);
}
}
else
{
- indexBuffer.writeID(deleteByteStream, i);
+ indexBuffer.writeEntryID(deleteByteStream, position);
deleteKeyCount++;
}
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/IndexOutputBuffer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/IndexOutputBuffer.java
index 0f3bace..c1a7c0a 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/IndexOutputBuffer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/IndexOutputBuffer.java
@@ -339,7 +339,7 @@
* @param stream The stream to write the record at the index to.
* @param position The position of the record to write.
*/
- public void writeID(ByteArrayOutputStream stream, int position)
+ public void writeEntryID(ByteArrayOutputStream stream, int position)
{
int offSet = getOffset(position);
int len = PackedInteger.getReadLongLength(buffer, offSet + REC_OVERHEAD);
@@ -369,7 +369,7 @@
public int getKeySize()
{
int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ offSet += PackedInteger.getReadLongLength(buffer, offSet);
return PackedInteger.readInt(buffer, offSet);
}
@@ -389,7 +389,7 @@
{
keyBuffer.clear();
int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ offSet += PackedInteger.getReadLongLength(buffer, offSet);
int keyLen = PackedInteger.readInt(buffer, offSet);
offSet += PackedInteger.getReadIntLength(buffer, offSet);
//Re-allocate if the key is bigger than the capacity.
@@ -412,7 +412,7 @@
private byte[] getKey(int position)
{
int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ offSet += PackedInteger.getReadLongLength(buffer, offSet);
int keyLen = PackedInteger.readInt(buffer, offSet);
offSet += PackedInteger.getReadIntLength(buffer, offSet);
byte[] key = new byte[keyLen];
@@ -450,14 +450,14 @@
int xoffSet = getOffset(xPosition);
int xIndexID = getIndexIDFromOffset(xoffSet);
xoffSet += REC_OVERHEAD;
- xoffSet += PackedInteger.getReadIntLength(buffer, xoffSet);
+ xoffSet += PackedInteger.getReadLongLength(buffer, xoffSet);
int xKeyLen = PackedInteger.readInt(buffer, xoffSet);
int xKey = PackedInteger.getReadIntLength(buffer, xoffSet) + xoffSet;
int yoffSet = getOffset(yPosition);
int yIndexID = getIndexIDFromOffset(yoffSet);
yoffSet += REC_OVERHEAD;
- yoffSet += PackedInteger.getReadIntLength(buffer, yoffSet);
+ yoffSet += PackedInteger.getReadLongLength(buffer, yoffSet);
int yKeyLen = PackedInteger.readInt(buffer, yoffSet);
int yKey = PackedInteger.getReadIntLength(buffer, yoffSet) + yoffSet;
@@ -470,7 +470,7 @@
int xoffSet = getOffset(xPosition);
int xIndexID = getIndexIDFromOffset(xoffSet);
xoffSet += REC_OVERHEAD;
- xoffSet += PackedInteger.getReadIntLength(buffer, xoffSet);
+ xoffSet += PackedInteger.getReadLongLength(buffer, xoffSet);
int xKeyLen = PackedInteger.readInt(buffer, xoffSet);
int xKey = PackedInteger.getReadIntLength(buffer, xoffSet) + xoffSet;
@@ -494,7 +494,7 @@
int offset = getOffset(position);
int indexID = getIndexIDFromOffset(offset);
offset += REC_OVERHEAD;
- offset += PackedInteger.getReadIntLength(buffer, offset);
+ offset += PackedInteger.getReadLongLength(buffer, offset);
int keyLen = PackedInteger.readInt(buffer, offset);
int key = PackedInteger.getReadIntLength(buffer, offset) + offset;
return comparator.compare(buffer, key, keyLen, b, b.length) == 0
@@ -519,7 +519,7 @@
int offset = getOffset(position);
int indexID = getIndexIDFromOffset(offset);
offset += REC_OVERHEAD;
- offset += PackedInteger.getReadIntLength(buffer, offset);
+ offset += PackedInteger.getReadLongLength(buffer, offset);
int keyLen = PackedInteger.readInt(buffer, offset);
int key = PackedInteger.getReadIntLength(buffer, offset) + offset;
@@ -573,7 +573,7 @@
public void writeKey(DataOutputStream dataStream) throws IOException
{
int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ offSet += PackedInteger.getReadLongLength(buffer, offSet);
int keyLen = PackedInteger.readInt(buffer, offSet);
offSet += PackedInteger.getReadIntLength(buffer, offSet);
dataStream.write(buffer, offSet, keyLen);
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryIDSet.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryIDSet.java
index 4070123..9c7d64d 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryIDSet.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryIDSet.java
@@ -267,37 +267,27 @@
/**
* Convert to a short string to aid with debugging.
*
- * @param buffer The string is appended to this string builder.
+ * @param sb The string is appended to this string builder.
*/
- void toString(StringBuilder buffer)
+ void toString(StringBuilder sb)
{
- if (!isDefined())
+ if (isDefined())
{
- if (key != null)
+ sb.append("[COUNT:").append(size()).append("]");
+ }
+ else if (key != null)
+ {
+ // The index entry limit was exceeded
+ sb.append("[LIMIT-EXCEEDED");
+ if (undefinedSize == Long.MAX_VALUE)
{
- // The index entry limit was exceeded
- if(undefinedSize == Long.MAX_VALUE)
- {
- buffer.append("[LIMIT-EXCEEDED]");
- }
- else
- {
- buffer.append("[LIMIT-EXCEEDED:");
- buffer.append(undefinedSize);
- buffer.append("]");
- }
+ sb.append(":").append(undefinedSize);
}
- else
- {
- // Not indexed
- buffer.append("[NOT-INDEXED]");
- }
+ sb.append("]");
}
else
{
- buffer.append("[COUNT:");
- buffer.append(size());
- buffer.append("]");
+ sb.append("[NOT-INDEXED]");
}
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ImportIDSet.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ImportIDSet.java
index 2ff042b..8d51830 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ImportIDSet.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ImportIDSet.java
@@ -46,14 +46,23 @@
private int count;
/** Boolean to keep track if the instance is defined or not. */
private boolean isDefined = true;
- /** Size of the undefined if count is kept. */
- private long undefinedSize;
/** Key related to an ID set. */
private ByteBuffer key;
- /** The entry limit size. */
- private final int limit;
- /** Set to true if a count of ids above the entry limit should be kept. */
- private final boolean doCount;
+ /** The index entry limit size. */
+ private final int indexEntryLimit;
+ /**
+ * Set to true if a count of ids above the index entry limit should be kept, a.k.a
+ * {@link #undefinedSize}.
+ *
+ * @see #undefinedSize
+ */
+ private final boolean maintainCount;
+ /**
+ * Size of the undefined id set, if count is maintained.
+ *
+ * @see #maintainCount
+ */
+ private long undefinedSize;
/**
* Create an import ID set of the specified size, index limit and index
@@ -61,21 +70,14 @@
*
* @param size The size of the the underlying array, plus some extra space.
* @param limit The index entry limit.
- * @param doCount The index maintain count.
+ * @param maintainCount whether to maintain the count when size is undefined.
*/
- public ImportIDSet(int size, int limit, boolean doCount)
+ public ImportIDSet(int size, int limit, boolean maintainCount)
{
this.array = new long[size + 128];
// A limit of 0 means unlimited.
- this.limit = limit == 0 ? Integer.MAX_VALUE : limit;
- this.doCount = doCount;
- }
-
- /** Create an empty import instance. */
- public ImportIDSet()
- {
- this.limit = -1;
- this.doCount = false;
+ this.indexEntryLimit = limit == 0 ? Integer.MAX_VALUE : limit;
+ this.maintainCount = maintainCount;
}
/**
@@ -117,26 +119,26 @@
/**
* Add the specified long value to an import ID set.
*
- * @param l The long value to add to an import ID set.
+ * @param entryID The {@link EntryID} to add to an import ID set.
*/
- void addEntryID(long l) {
+ void addEntryID(long entryID) {
if(!isDefined()) {
- if(doCount) {
+ if (maintainCount) {
undefinedSize++;
}
return;
}
- if (l < 0 || (isDefined() && count + 1 > limit))
+ if (entryID < 0 || (isDefined() && count + 1 > indexEntryLimit))
{
setUndefined();
- if(doCount) {
+ if (maintainCount) {
undefinedSize = count + 1;
} else {
undefinedSize = Long.MAX_VALUE;
}
count = 0;
} else {
- add(l);
+ add(entryID);
}
}
@@ -157,7 +159,7 @@
incrementLimitCount = true;
} else {
array = EntryIDSet.decodeEntryIDList(dBbytes);
- if(array.length + importIdSet.size() > limit) {
+ if (array.length + importIdSet.size() > indexEntryLimit) {
undefinedSize = array.length + importIdSet.size();
isDefined=false;
incrementLimitCount=true;
@@ -186,7 +188,7 @@
undefinedSize = Long.MAX_VALUE;
} else {
array = EntryIDSet.decodeEntryIDList(bytes);
- if(array.length - importIdSet.size() > limit) {
+ if (array.length - importIdSet.size() > indexEntryLimit) {
isDefined=false;
count = 0;
importIdSet.setUndefined();
@@ -211,7 +213,7 @@
public boolean merge(ByteString bytes, ImportIDSet importIdSet)
{
boolean incrementLimitCount=false;
- if(doCount) {
+ if (maintainCount) {
incrementLimitCount = mergeCount(bytes, importIdSet);
} else if (isDBUndefined(bytes)) {
isDefined = false;
@@ -225,7 +227,7 @@
count = 0;
} else {
array = EntryIDSet.decodeEntryIDList(bytes);
- if (array.length + importIdSet.size() > limit) {
+ if (array.length + importIdSet.size() > indexEntryLimit) {
isDefined = false;
incrementLimitCount = true;
count = 0;
@@ -348,17 +350,17 @@
return count;
}
- private boolean add(long v)
+ private boolean add(long entryID)
{
resize(count+1);
- if (count == 0 || v > array[count-1])
+ if (count == 0 || entryID > array[count-1])
{
- array[count++] = v;
+ array[count++] = entryID;
return true;
}
- int pos = binarySearch(array, count, v);
+ int pos = binarySearch(array, count, entryID);
if (pos >=0)
{
return false;
@@ -370,7 +372,7 @@
pos = -(pos+1);
System.arraycopy(array, pos, array, pos+1, count-pos);
- array[pos] = v;
+ array[pos] = entryID;
count++;
return true;
}
@@ -429,7 +431,7 @@
*/
ByteString keyToByteString()
{
- return ByteString.wrap(getKey().array(), 0, getKey().limit());
+ return ByteString.wrap(key.array(), 0, key.limit());
}
/**
@@ -475,4 +477,25 @@
{
return key;
}
+
+ /** {@inheritDoc} */
+ @Override
+ public String toString()
+ {
+ final StringBuilder sb = new StringBuilder();
+ if (isDefined())
+ {
+ sb.append("[COUNT:").append(size()).append("]");
+ }
+ else
+ {
+ sb.append("[LIMIT-EXCEEDED");
+ if (undefinedSize < Long.MAX_VALUE)
+ {
+ sb.append(":").append(undefinedSize);
+ }
+ sb.append("]");
+ }
+ return sb.toString();
+ }
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/Importer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/Importer.java
index 92f0268..6ded735 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/Importer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/Importer.java
@@ -1181,6 +1181,7 @@
// Ensure that there are minimum two threads available for parallel
// processing of smaller indexes.
dbThreads = Math.max(2, dbThreads);
+ dbThreads = 1; // FIXME JNR
logger.info(NOTE_JEB_IMPORT_LDIF_PHASE_TWO_MEM_REPORT, availableMemory, readAheadSize, buffers);
@@ -1765,7 +1766,6 @@
private final IndexManager indexMgr;
private final int cacheSize;
private final Map<Integer, DNState> dnStateMap = new HashMap<Integer, DNState>();
- private final Map<Integer, Index> indexMap = new HashMap<Integer, Index>();
private final Semaphore permits;
private final int maxPermits;
private final AtomicLong bytesRead = new AtomicLong();
@@ -1993,9 +1993,9 @@
{
final Index index = idContainerMap.get(indexID);
int limit = index.getIndexEntryLimit();
- boolean doCount = index.getMaintainCount();
- insertIDSet = new ImportIDSet(1, limit, doCount);
- deleteIDSet = new ImportIDSet(1, limit, doCount);
+ boolean maintainCount = index.getMaintainCount();
+ insertIDSet = new ImportIDSet(1, limit, maintainCount);
+ deleteIDSet = new ImportIDSet(1, limit, maintainCount);
}
key = ByteBuffer.allocate(b.getKeyLen());
@@ -2009,7 +2009,7 @@
}
else if (b.compare(key, indexID) != 0)
{
- addToDB(insertIDSet, deleteIDSet, indexID);
+ addToDB(indexID, insertIDSet, deleteIDSet);
keyCount.incrementAndGet();
indexID = b.getIndexID();
@@ -2023,9 +2023,9 @@
{
final Index index = idContainerMap.get(indexID);
int limit = index.getIndexEntryLimit();
- boolean doCount = index.getMaintainCount();
- insertIDSet = new ImportIDSet(1, limit, doCount);
- deleteIDSet = new ImportIDSet(1, limit, doCount);
+ boolean maintainCount = index.getMaintainCount();
+ insertIDSet = new ImportIDSet(1, limit, maintainCount);
+ deleteIDSet = new ImportIDSet(1, limit, maintainCount);
}
key.clear();
@@ -2056,7 +2056,7 @@
if (key != null)
{
- addToDB(insertIDSet, deleteIDSet, indexID);
+ addToDB(indexID, insertIDSet, deleteIDSet);
}
}
return null;
@@ -2072,11 +2072,11 @@
}
}
- private void addToDB(ImportIDSet insertSet, ImportIDSet deleteSet, int indexID) throws DirectoryException
+ private void addToDB(int indexID, ImportIDSet insertSet, ImportIDSet deleteSet) throws DirectoryException
{
if (indexMgr.isDN2ID())
{
- addDN2ID(insertSet, indexID);
+ addDN2ID(indexID, insertSet);
}
else
{
@@ -2085,25 +2085,17 @@
ByteString key = deleteSet.keyToByteString();
final Index index = idContainerMap.get(indexID);
index.delete(txn, key, deleteSet);
- if (!indexMap.containsKey(indexID))
- {
- indexMap.put(indexID, index);
- }
}
if (insertSet.size() > 0 || !insertSet.isDefined())
{
ByteString key = insertSet.keyToByteString();
final Index index = idContainerMap.get(indexID);
index.insert(txn, key, insertSet);
- if (!indexMap.containsKey(indexID))
- {
- indexMap.put(indexID, index);
- }
}
}
}
- private void addDN2ID(ImportIDSet record, Integer indexID) throws DirectoryException
+ private void addDN2ID(int indexID, ImportIDSet record) throws DirectoryException
{
DNState dnState;
if (!dnStateMap.containsKey(indexID))
@@ -2402,7 +2394,9 @@
private final int DRAIN_TO = 3;
private final IndexManager indexMgr;
private final BlockingQueue<IndexOutputBuffer> queue;
+ /** Stream where to output insert ImportIDSet data. */
private final ByteArrayOutputStream insertByteStream = new ByteArrayOutputStream(2 * bufferSize);
+ /** Stream where to output delete ImportIDSet data. */
private final ByteArrayOutputStream deleteByteStream = new ByteArrayOutputStream(2 * bufferSize);
private final DataOutputStream bufferStream;
private final DataOutputStream bufferIndexStream;
@@ -2511,7 +2505,7 @@
insertOrDeleteKey(indexBuffer, i);
continue;
}
- if (!indexBuffer.compare(i))
+ if (!indexBuffer.byteArraysEqual(i))
{
bufferLen += writeRecord(indexBuffer);
indexBuffer.setPosition(i);
@@ -2556,7 +2550,7 @@
saveIndexID = b.getIndexID();
insertOrDeleteKey(b, b.getPosition());
}
- else if (!b.compare(saveKey, saveIndexID))
+ else if (!b.recordsEqual(saveKey, saveIndexID))
{
bufferLen += writeRecord(saveKey, saveIndexID);
resetStreams();
@@ -2589,32 +2583,32 @@
deleteKeyCount = 0;
}
- private void insertOrDeleteKey(IndexOutputBuffer indexBuffer, int i)
+ private void insertOrDeleteKey(IndexOutputBuffer indexBuffer, int position)
{
- if (indexBuffer.isInsertRecord(i))
+ if (indexBuffer.isInsertRecord(position))
{
- indexBuffer.writeID(insertByteStream, i);
+ indexBuffer.writeEntryID(insertByteStream, position);
insertKeyCount++;
}
else
{
- indexBuffer.writeID(deleteByteStream, i);
+ indexBuffer.writeEntryID(deleteByteStream, position);
deleteKeyCount++;
}
}
- private void insertOrDeleteKeyCheckEntryLimit(IndexOutputBuffer indexBuffer, int i)
+ private void insertOrDeleteKeyCheckEntryLimit(IndexOutputBuffer indexBuffer, int position)
{
- if (indexBuffer.isInsertRecord(i))
+ if (indexBuffer.isInsertRecord(position))
{
if (insertKeyCount++ <= indexMgr.getLimit())
{
- indexBuffer.writeID(insertByteStream, i);
+ indexBuffer.writeEntryID(insertByteStream, position);
}
}
else
{
- indexBuffer.writeID(deleteByteStream, i);
+ indexBuffer.writeEntryID(deleteByteStream, position);
deleteKeyCount++;
}
}
@@ -2628,15 +2622,15 @@
insertByteStream.write(-1);
}
- int insertSize = 4;
+ int insertSize = INT_SIZE;
bufferStream.writeInt(insertKeyCount);
if (insertByteStream.size() > 0)
{
insertByteStream.writeTo(bufferStream);
}
- int deleteSize = 4;
- bufferStream.write(deleteKeyCount);
+ int deleteSize = INT_SIZE;
+ bufferStream.writeInt(deleteKeyCount);
if (deleteByteStream.size() > 0)
{
deleteByteStream.writeTo(bufferStream);
@@ -2657,7 +2651,7 @@
int headerSize = writeHeader(b.getIndexID(), keySize);
b.writeKey(bufferStream);
int bodySize = writeByteStreams();
- return headerSize + bodySize + keySize;
+ return headerSize + keySize + bodySize;
}
private int writeRecord(byte[] k, int indexID) throws IOException
@@ -2666,7 +2660,7 @@
int headerSize = writeHeader(indexID, keySize);
bufferStream.write(k);
int bodySize = writeByteStreams();
- return headerSize + bodySize + keySize;
+ return headerSize + keySize + bodySize;
}
/** {@inheritDoc} */
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexInputBuffer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexInputBuffer.java
index 1edd331..65033ff 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexInputBuffer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexInputBuffer.java
@@ -27,6 +27,7 @@
package org.opends.server.backends.pluggable;
import static org.opends.messages.JebMessages.*;
+import static org.opends.server.backends.pluggable.IndexOutputBuffer.*;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -35,8 +36,6 @@
import org.forgerock.i18n.slf4j.LocalizedLogger;
import org.opends.server.backends.pluggable.Importer.IndexManager;
-import com.sleepycat.util.PackedInteger;
-
/**
* The buffer class is used to process a buffer from the temporary index files
* during phase 2 processing.
@@ -208,11 +207,7 @@
indexID = getInt();
ensureData(20);
- byte[] ba = cache.array();
- int p = cache.position();
- int len = PackedInteger.getReadIntLength(ba, p);
- int keyLen = PackedInteger.readInt(ba, p);
- cache.position(p + len);
+ int keyLen = getInt();
if (keyLen > keyBuf.capacity())
{
keyBuf = ByteBuffer.allocate(keyLen);
@@ -227,10 +222,16 @@
private int getInt() throws IOException
{
- ensureData(4);
+ ensureData(INT_SIZE);
return cache.getInt();
}
+ private long getLong() throws IOException
+ {
+ ensureData(LONG_SIZE);
+ return cache.getLong();
+ }
+
/**
* Reads the next ID set from the record and merges it with the provided ID
* set.
@@ -248,27 +249,15 @@
}
ensureData(20);
- int p = cache.position();
- byte[] ba = cache.array();
- int len = PackedInteger.getReadIntLength(ba, p);
- int keyCount = PackedInteger.readInt(ba, p);
- p += len;
- cache.position(p);
+ int keyCount = getInt();
for (int k = 0; k < keyCount; k++)
{
- if (ensureData(9))
- {
- p = cache.position();
- }
- len = PackedInteger.getReadLongLength(ba, p);
- long l = PackedInteger.readLong(ba, p);
- p += len;
- cache.position(p);
+ long entryID = getLong();
// idSet will be null if skipping.
if (idSet != null)
{
- idSet.addEntryID(l);
+ idSet.addEntryID(entryID);
}
}
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexOutputBuffer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexOutputBuffer.java
index 6b9a02b..d746127 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexOutputBuffer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexOutputBuffer.java
@@ -33,8 +33,6 @@
import org.forgerock.opendj.ldap.ByteSequence;
-import com.sleepycat.util.PackedInteger;
-
/**
* This class represents a index buffer used to store the keys and entry IDs
* processed from the LDIF file during phase one of an import, or rebuild index
@@ -57,7 +55,9 @@
* The record size is used for fast seeks to quickly "jump" over records.
* </p>
* <p>
- * The records are packed as much as possible, to optimize the buffer space.<br>
+ * The records are packed as much as possible, to optimize the buffer space.
+ * </p>
+ * <p>
* This class is not thread safe.
* </p>
*/
@@ -68,14 +68,16 @@
LT, GT, LE, GE, EQ
}
- /** The size of a Java int. A Java int is 32 bits, i.e. 4 bytes. */
+ /** The number of bytes of a Java int. */
static final int INT_SIZE = 4;
+ /** The number of bytes of a Java long. */
+ static final int LONG_SIZE = 8;
/**
* The record overhead. In addition to entryID, key length and key bytes, the
- * record overhead includes the indexID + INS/DEL bit
+ * record overhead includes the INS/DEL bit + indexID
*/
- private static final int REC_OVERHEAD = INT_SIZE + 1;
+ private static final int REC_OVERHEAD = 1 + INT_SIZE;
/** Buffer records are either insert records or delete records. */
private static final byte DEL = 0, INS = 1;
@@ -198,12 +200,11 @@
}
/**
- * Determines if buffer should be re-cycled by calling {@link #reset()}.
+ * Determines if buffer should be re-cycled by calling {@link IndexOutputBuffer#reset()}.
*
- * @return {@code true} if buffer should be recycled, or {@code false} if it
- * should not.
+ * @return {@code true} if buffer should be recycled, or {@code false} if it should not.
*/
- public boolean isDiscarded()
+ boolean isDiscarded()
{
return discarded;
}
@@ -284,7 +285,7 @@
// before it
recordOffset = addRecord(keyBytes, entryID.longValue(), indexID, insert);
// then write the returned record size
- keyOffset += writeIntBytes(buffer, keyOffset, recordOffset);
+ keyOffset = writeInt(buffer, keyOffset, recordOffset);
bytesLeft = recordOffset - keyOffset;
keys++;
}
@@ -295,17 +296,17 @@
*/
private int addRecord(ByteSequence key, long entryID, int indexID, boolean insert)
{
- int retOffset = recordOffset - getRecordSize(key.length(), entryID);
+ int retOffset = recordOffset - getRecordSize(key.length());
int offSet = retOffset;
// write the INS/DEL bit
buffer[offSet++] = insert ? INS : DEL;
// write the indexID
- offSet += writeIntBytes(buffer, offSet, indexID);
+ offSet = writeInt(buffer, offSet, indexID);
// write the entryID
- offSet = PackedInteger.writeLong(buffer, offSet, entryID);
+ offSet = writeLong(buffer, offSet, entryID);
// write the key length
- offSet = PackedInteger.writeInt(buffer, offSet, key.length());
+ offSet = writeInt(buffer, offSet, key.length());
// write the key bytes
key.copyTo(buffer, offSet);
return retOffset;
@@ -321,31 +322,27 @@
*/
public static int getRequiredSize(int keyLen, long entryID)
{
- // Adds up the key length + key bytes + entryID + indexID + the INS/DEL bit
- // and finally the space needed to store the record size
- return getRecordSize(keyLen, entryID) + INT_SIZE;
+ // also add up the space needed to store the record size
+ return getRecordSize(keyLen) + INT_SIZE;
}
- private static int getRecordSize(int keyLen, long entryID)
+ private static int getRecordSize(int keyLen)
{
- // Adds up the key length + key bytes + ...
- return PackedInteger.getWriteIntLength(keyLen) + keyLen +
- // ... entryID + (indexID + INS/DEL bit).
- PackedInteger.getWriteLongLength(entryID) + REC_OVERHEAD;
+ // Adds up (INS/DEL bit + indexID) + entryID + key length + key bytes
+ return REC_OVERHEAD + LONG_SIZE + INT_SIZE + keyLen;
}
/**
- * Write record at specified position to the specified output stream.
- * Used when when writing the index scratch files.
+ * Write the entryID at the specified position to the specified output stream.
+ * Used when writing the index scratch files.
*
* @param stream The stream to write the record at the index to.
* @param position The position of the record to write.
*/
- public void writeID(ByteArrayOutputStream stream, int position)
+ public void writeEntryID(ByteArrayOutputStream stream, int position)
{
int offSet = getOffset(position);
- int len = PackedInteger.getReadLongLength(buffer, offSet + REC_OVERHEAD);
- stream.write(buffer, offSet + REC_OVERHEAD, len);
+ stream.write(buffer, offSet + REC_OVERHEAD, LONG_SIZE);
}
@@ -370,9 +367,8 @@
*/
public int getKeySize()
{
- int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
- return PackedInteger.readInt(buffer, offSet);
+ int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE;
+ return readInt(buffer, offSet);
}
/**
@@ -390,10 +386,9 @@
private ByteBuffer getKeyBuf(int position)
{
keyBuffer.clear();
- int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
- int keyLen = PackedInteger.readInt(buffer, offSet);
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE;
+ int keyLen = readInt(buffer, offSet);
+ offSet += INT_SIZE;
//Re-allocate if the key is bigger than the capacity.
if(keyLen > keyBuffer.capacity())
{
@@ -413,10 +408,9 @@
*/
private byte[] getKey(int position)
{
- int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
- int keyLen = PackedInteger.readInt(buffer, offSet);
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE;
+ int keyLen = readInt(buffer, offSet);
+ offSet += INT_SIZE;
byte[] key = new byte[keyLen];
System.arraycopy(buffer, offSet, key, 0, keyLen);
return key;
@@ -424,7 +418,7 @@
private int getOffset(int position)
{
- return getIntegerValue(position * INT_SIZE);
+ return readInt(position * INT_SIZE);
}
/**
@@ -444,24 +438,22 @@
private int getIndexIDFromOffset(int offset)
{
- return getIntegerValue(offset + 1);
+ return readInt(offset + 1);
}
private boolean is(CompareOp op, int xPosition, int yPosition)
{
int xoffSet = getOffset(xPosition);
int xIndexID = getIndexIDFromOffset(xoffSet);
- xoffSet += REC_OVERHEAD;
- xoffSet += PackedInteger.getReadIntLength(buffer, xoffSet);
- int xKeyLen = PackedInteger.readInt(buffer, xoffSet);
- int xKey = PackedInteger.getReadIntLength(buffer, xoffSet) + xoffSet;
+ xoffSet += REC_OVERHEAD + LONG_SIZE;
+ int xKeyLen = readInt(buffer, xoffSet);
+ int xKey = INT_SIZE + xoffSet;
int yoffSet = getOffset(yPosition);
int yIndexID = getIndexIDFromOffset(yoffSet);
- yoffSet += REC_OVERHEAD;
- yoffSet += PackedInteger.getReadIntLength(buffer, yoffSet);
- int yKeyLen = PackedInteger.readInt(buffer, yoffSet);
- int yKey = PackedInteger.getReadIntLength(buffer, yoffSet) + yoffSet;
+ yoffSet += REC_OVERHEAD + LONG_SIZE;
+ int yKeyLen = readInt(buffer, yoffSet);
+ int yKey = INT_SIZE + yoffSet;
int cmp = comparator.compare(buffer, xKey, xKeyLen, xIndexID, yKey, yKeyLen, yIndexID);
return evaluateReturnCode(cmp, op);
@@ -471,39 +463,33 @@
{
int xoffSet = getOffset(xPosition);
int xIndexID = getIndexIDFromOffset(xoffSet);
- xoffSet += REC_OVERHEAD;
- xoffSet += PackedInteger.getReadIntLength(buffer, xoffSet);
- int xKeyLen = PackedInteger.readInt(buffer, xoffSet);
- int xKey = PackedInteger.getReadIntLength(buffer, xoffSet) + xoffSet;
+ xoffSet += REC_OVERHEAD + LONG_SIZE;
+ int xKeyLen = readInt(buffer, xoffSet);
+ int xKey = INT_SIZE + xoffSet;
int cmp = comparator.compare(buffer, xKey, xKeyLen, xIndexID, yKey, yKey.length, yIndexID);
return evaluateReturnCode(cmp, op);
}
/**
- * Compare the byte array at the current position with the specified one and
- * using the specified index id. It will return {@code true} if the byte
- * array at the current position is equal to the specified byte array as
- * determined by the comparator and the index ID is is equal. It will
- * return {@code false} otherwise.
+ * Verifies whether the provided byte array and indexID are equal to
+ * the byte array and indexIDs currently pointed to by this index output buffer.
*
* @param b The byte array to compare.
- * @param bIndexID The index key.
+ * @param bIndexID The index key to compare.
* @return <CODE>True</CODE> if the byte arrays are equal.
*/
- public boolean compare(byte[]b, int bIndexID)
+ public boolean recordsEqual(byte[] b, int bIndexID)
{
int offset = getOffset(position);
int indexID = getIndexIDFromOffset(offset);
- offset += REC_OVERHEAD;
- offset += PackedInteger.getReadIntLength(buffer, offset);
- int keyLen = PackedInteger.readInt(buffer, offset);
- int key = PackedInteger.getReadIntLength(buffer, offset) + offset;
+ offset += REC_OVERHEAD + LONG_SIZE;
+ int keyLen = readInt(buffer, offset);
+ int key = INT_SIZE + offset;
return comparator.compare(buffer, key, keyLen, b, b.length) == 0
&& indexID == bIndexID;
}
-
/**
* Compare current IndexBuffer to the specified index buffer using both the
* comparator and index ID of both buffers.
@@ -520,10 +506,9 @@
final ByteBuffer keyBuf = b.getKeyBuf(b.position);
int offset = getOffset(position);
int indexID = getIndexIDFromOffset(offset);
- offset += REC_OVERHEAD;
- offset += PackedInteger.getReadIntLength(buffer, offset);
- int keyLen = PackedInteger.readInt(buffer, offset);
- int key = PackedInteger.getReadIntLength(buffer, offset) + offset;
+ offset += REC_OVERHEAD + LONG_SIZE;
+ int keyLen = readInt(buffer, offset);
+ int key = INT_SIZE + offset;
final int cmp = comparator.compare(buffer, key, keyLen, keyBuf.array(), keyBuf.limit());
if (cmp != 0)
@@ -574,23 +559,22 @@
*/
public void writeKey(DataOutputStream dataStream) throws IOException
{
- int offSet = getOffset(position) + REC_OVERHEAD;
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
- int keyLen = PackedInteger.readInt(buffer, offSet);
- offSet += PackedInteger.getReadIntLength(buffer, offSet);
+ int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE;
+ int keyLen = readInt(buffer, offSet);
+ offSet += INT_SIZE;
dataStream.write(buffer, offSet, keyLen);
}
/**
* Compare the byte array at the current position with the byte array at the
- * specified index.
+ * provided position.
*
- * @param i The index pointing to the byte array to compare.
+ * @param position The index pointing to the byte array to compare.
* @return {@code true} if the byte arrays are equal, or {@code false} otherwise
*/
- public boolean compare(int i)
+ public boolean byteArraysEqual(int position)
{
- return is(CompareOp.EQ, i, position);
+ return is(CompareOp.EQ, position, this.position);
}
/**
@@ -625,16 +609,32 @@
position++;
}
- private int writeIntBytes(byte[] buffer, int offset, int val)
+ private int writeInt(byte[] buffer, int offset, int val)
{
- for (int i = offset + INT_SIZE - 1; i >= offset; i--) {
+ final int endOffset = offset + INT_SIZE;
+ for (int i = endOffset - 1; i >= offset; i--) {
buffer[i] = (byte) (val & 0xff);
val >>>= 8;
}
- return INT_SIZE;
+ return endOffset;
}
- private int getIntegerValue(int index)
+ private int writeLong(byte[] buffer, int offset, long val)
+ {
+ final int endOffset = offset + LONG_SIZE;
+ for (int i = endOffset - 1; i >= offset; i--) {
+ buffer[i] = (byte) (val & 0xff);
+ val >>>= 8;
+ }
+ return endOffset;
+ }
+
+ private int readInt(int index)
+ {
+ return readInt(buffer, index);
+ }
+
+ private int readInt(byte[] buffer, int index)
{
int answer = 0;
for (int i = 0; i < INT_SIZE; i++) {
@@ -729,11 +729,11 @@
private void swap(int a, int b)
{
- int aOffset = a * INT_SIZE;
- int bOffset = b * INT_SIZE;
- int bVal = getIntegerValue(bOffset);
- System.arraycopy(buffer, aOffset, buffer, bOffset, INT_SIZE);
- writeIntBytes(buffer, aOffset, bVal);
+ int aOffset = a * INT_SIZE;
+ int bOffset = b * INT_SIZE;
+ int bVal = readInt(bOffset);
+ System.arraycopy(buffer, aOffset, buffer, bOffset, INT_SIZE);
+ writeInt(buffer, aOffset, bVal);
}
private void vectorSwap(int a, int b, int n)
@@ -854,11 +854,13 @@
{
for(int i = 0; i < length && i < otherLength; i++)
{
- if(b[offset + i] > b[otherOffset + i])
+ byte b1 = b[offset + i];
+ byte b2 = b[otherOffset + i];
+ if (b1 > b2)
{
return 1;
}
- else if (b[offset + i] < b[otherOffset + i])
+ else if (b1 < b2)
{
return -1;
}
--
Gitblit v1.10.0