From 218b2a22d685fef5602021786334a4d78a2460e2 Mon Sep 17 00:00:00 2001
From: ludovicp <ludovicp@localhost>
Date: Fri, 25 Jun 2010 09:40:08 +0000
Subject: [PATCH] Resolves an issue in Import due to entries with multiple RDNs. This patch changes the key format used by the DN2ID database. Rebuilding the index is required.
---
opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java | 209 ++++++++++++++++++++++++++++++++-------------------
1 files changed, 130 insertions(+), 79 deletions(-)
diff --git a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
index e2064cc..4453e79 100644
--- a/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/importLDIF/Importer.java
@@ -95,10 +95,6 @@
//The DN attribute type.
private static AttributeType dnType;
-
- //Comparators for DN and indexes respectively.
- private static final IndexBuffer.DNComparator dnComparator
- = new IndexBuffer.DNComparator();
private static final IndexBuffer.IndexComparator indexComparator =
new IndexBuffer.IndexComparator();
@@ -1133,8 +1129,8 @@
entryContainer.getDN2ID().getComparator();
try {
for(DN excludedDN : suffix.getExcludeBranches()) {
- byte[] bytes =
- StaticUtils.getBytes(excludedDN.toNormalizedString());
+ byte[] bytes = JebFormat.dnToDNKey(
+ excludedDN, suffix.getBaseDN().getNumComponents());
key.setData(bytes);
status = cursor.getSearchKeyRange(key, data, lockMode);
if(status == OperationStatus.SUCCESS &&
@@ -1142,9 +1138,8 @@
// This is the base entry for a branch that was excluded in the
// import so we must migrate all entries in this branch over to
// the new entry container.
- byte[] end = StaticUtils.getBytes("," +
- excludedDN.toNormalizedString());
- end[0] = (byte) (end[0] + 1);
+ byte[] end = Arrays.copyOf(bytes, bytes.length+1);
+ end[end.length-1] = 0x01;
while(status == OperationStatus.SUCCESS &&
comparator.compare(key.getData(), end) < 0 &&
@@ -1190,6 +1185,17 @@
public Void call() throws Exception
{
for(Suffix suffix : dnSuffixMap.values()) {
+ List<byte[]> includeBranches =
+ new ArrayList<byte[]>(suffix.getIncludeBranches().size());
+ for(DN includeBranch : suffix.getIncludeBranches())
+ {
+ if(includeBranch.isDescendantOf(suffix.getBaseDN()))
+ {
+ includeBranches.add(JebFormat.dnToDNKey(
+ includeBranch, suffix.getBaseDN().getNumComponents()));
+ }
+ }
+
EntryContainer entryContainer = suffix.getSrcEntryContainer();
if(entryContainer != null &&
!suffix.getIncludeBranches().isEmpty()) {
@@ -1207,8 +1213,17 @@
status = cursor.getFirst(key, data, lockMode);
while(status == OperationStatus.SUCCESS &&
!importConfiguration.isCancelled() && !isPhaseOneCanceled) {
- DN dn = DN.decode(ByteString.wrap(key.getData()));
- if(!suffix.getIncludeBranches().contains(dn)) {
+
+ boolean found = false;
+ for(byte[] includeBranch : includeBranches)
+ {
+ if(Arrays.equals(includeBranch, key.getData()))
+ {
+ found = true;
+ break;
+ }
+ }
+ if(!found) {
EntryID id = new EntryID(data);
Entry entry =
entryContainer.getID2Entry().get(null,
@@ -1231,9 +1246,8 @@
* (the comma).
* No possibility of overflow here.
*/
- byte[] begin =
- StaticUtils.getBytes("," + dn.toNormalizedString());
- begin[0] = (byte) (begin[0] + 1);
+ byte[] begin = Arrays.copyOf(key.getData(), key.getSize()+1);
+ begin[begin.length-1] = 0x01;
key.setData(begin);
status = cursor.getSearchKeyRange(key, data, lockMode);
}
@@ -1647,14 +1661,7 @@
IndexBuffer indexBuffer = e.getValue();
setIterator.remove();
ImportIndexType indexType = indexKey.getIndexType();
- if(indexType.equals(ImportIndexType.DN))
- {
- indexBuffer.setComparator(dnComparator);
- }
- else
- {
- indexBuffer.setComparator(indexComparator);
- }
+ indexBuffer.setComparator(indexComparator);
indexBuffer.setIndexKey(indexKey);
indexBuffer.setDiscard();
Future<Void> future =
@@ -1717,9 +1724,10 @@
void processDN2ID(Suffix suffix, DN dn, EntryID entryID)
throws ConfigException, InterruptedException
{
- DatabaseContainer dn2id = suffix.getDN2ID();
- byte[] dnBytes = StaticUtils.getBytes(dn.toNormalizedString());
- int id = processKey(dn2id, dnBytes, entryID, dnComparator,
+ DN2ID dn2id = suffix.getDN2ID();
+ byte[] dnBytes =
+ JebFormat.dnToDNKey(dn, suffix.getBaseDN().getNumComponents());
+ int id = processKey(dn2id, dnBytes, entryID, indexComparator,
new IndexKey(dnType, ImportIndexType.DN, 1), true);
idECMap.putIfAbsent(id, suffix.getEntryContainer());
}
@@ -1975,11 +1983,10 @@
{
private final int DN_STATE_CACHE_SIZE = 64 * KB;
- private DN parentDN, lastDN;
+ private ByteBuffer parentDN, lastDN;
private EntryID parentID, lastID, entryID;
private final DatabaseEntry DNKey, DNValue;
- private final TreeMap<DN, EntryID> parentIDMap =
- new TreeMap<DN, EntryID>();
+ private final TreeMap<ByteBuffer, EntryID> parentIDMap;
private final EntryContainer entryContainer;
private final Map<byte[], ImportIDSet> id2childTree;
private final Map<byte[], ImportIDSet> id2subtreeTree;
@@ -1990,6 +1997,7 @@
DNState(EntryContainer entryContainer)
{
this.entryContainer = entryContainer;
+ parentIDMap = new TreeMap<ByteBuffer, EntryID>();
Comparator<byte[]> childComparator =
entryContainer.getID2Children().getComparator();
id2childTree = new TreeMap<byte[], ImportIDSet>(childComparator);
@@ -2002,80 +2010,124 @@
id2subtreeTree = new TreeMap<byte[], ImportIDSet>(subComparator);
DNKey = new DatabaseEntry();
DNValue = new DatabaseEntry();
+ lastDN = ByteBuffer.allocate(BYTE_BUFFER_CAPACITY);
}
+ private ByteBuffer getParent(ByteBuffer buffer)
+ {
+ int parentIndex =
+ JebFormat.findDNKeyParent(buffer.array(), 0, buffer.limit());
+ if(parentIndex < 0)
+ {
+ // This is the root or base DN
+ return null;
+ }
+ ByteBuffer parent = buffer.duplicate();
+ parent.limit(parentIndex);
+ return parent;
+ }
+
+ private ByteBuffer deepCopy(ByteBuffer srcBuffer, ByteBuffer destBuffer)
+ {
+ if(destBuffer == null ||
+ destBuffer.clear().remaining() < srcBuffer.limit())
+ {
+ byte[] bytes = new byte[srcBuffer.limit()];
+ System.arraycopy(srcBuffer.array(), 0, bytes, 0,
+ srcBuffer.limit());
+ return ByteBuffer.wrap(bytes);
+ }
+ else
+ {
+ destBuffer.put(srcBuffer);
+ destBuffer.flip();
+ return destBuffer;
+ }
+ }
+
+ // Why do we still need this if we are checking parents in the first
+ // phase?
private boolean checkParent(ImportIDSet record) throws DirectoryException,
DatabaseException
{
- DN dn = DN.decode(new String(record.getKey().array(), 0 ,
- record.getKey().limit()));
DNKey.setData(record.getKey().array(), 0 , record.getKey().limit());
byte[] v = record.toDatabase();
long v1 = JebFormat.entryIDFromDatabase(v);
DNValue.setData(v);
entryID = new EntryID(v1);
+ parentDN = getParent(record.getKey());
+
//Bypass the cache for append data, lookup the parent in DN2ID and
//return.
if(importConfiguration != null &&
importConfiguration.appendToExistingData())
{
- parentDN = entryContainer.getParentWithinBase(dn);
//If null is returned than this is a suffix DN.
if(parentDN != null)
{
- parentID =
- entryContainer.getDN2ID().get(null, parentDN, LockMode.DEFAULT);
+ DatabaseEntry key = new DatabaseEntry(parentDN.array());
+ DatabaseEntry value = new DatabaseEntry();
+ OperationStatus status;
+ status =
+ entryContainer.getDN2ID().read(null, key, value,
+ LockMode.DEFAULT);
+ if(status == OperationStatus.SUCCESS)
+ {
+ parentID = new EntryID(value);
+ }
+ else
+ {
+ // We have a missing parent. Maybe parent checking was turned off?
+ // Just ignore.
+ parentID = null;
+ return false;
+ }
}
}
else
{
if(parentIDMap.isEmpty())
{
- parentIDMap.put(dn, entryID);
+ parentIDMap.put(deepCopy(record.getKey(), null), entryID);
return true;
}
- else if(lastDN != null && lastDN.isAncestorOf(dn))
+ else if(lastDN != null && lastDN.equals(parentDN))
{
- parentIDMap.put(lastDN, lastID);
- parentDN = lastDN;
+ parentIDMap.put(deepCopy(lastDN, null), lastID);
parentID = lastID;
- lastDN = dn;
+ lastDN = deepCopy(record.getKey(), lastDN);
lastID = entryID;
return true;
}
- else if(parentIDMap.lastKey().isAncestorOf(dn))
+ else if(parentIDMap.lastKey().equals(parentDN))
{
- parentDN = parentIDMap.lastKey();
parentID = parentIDMap.get(parentDN);
- lastDN = dn;
+ lastDN = deepCopy(record.getKey(), lastDN);
lastID = entryID;
return true;
}
else
{
- DN newParentDN = entryContainer.getParentWithinBase(dn);
- if(parentIDMap.containsKey(newParentDN))
+ if(parentIDMap.containsKey(parentDN))
{
- EntryID newParentID = parentIDMap.get(newParentDN);
- DN lastDN = parentIDMap.lastKey();
- while(!newParentDN.equals(lastDN)) {
- parentIDMap.remove(lastDN);
- lastDN = parentIDMap.lastKey();
+ EntryID newParentID = parentIDMap.get(parentDN);
+ ByteBuffer key = parentIDMap.lastKey();
+ while(!parentDN.equals(key)) {
+ parentIDMap.remove(key);
+ key = parentIDMap.lastKey();
}
- parentIDMap.put(dn, entryID);
- parentDN = newParentDN;
+ parentIDMap.put(deepCopy(record.getKey(), null), entryID);
parentID = newParentID;
- lastDN = dn;
+ lastDN = deepCopy(record.getKey(), lastDN);
lastID = entryID;
}
else
{
- Message message =
- NOTE_JEB_IMPORT_LDIF_DN_NO_PARENT.get(dn.toString());
- Entry e = new Entry(dn, null, null, null);
- reader.rejectEntry(e, message);
+ // We have a missing parent. Maybe parent checking was turned off?
+ // Just ignore.
+ parentID = null;
return false;
}
}
@@ -2105,7 +2157,7 @@
}
- private EntryID getParentID(DN dn) throws DatabaseException
+ private EntryID getParentID(ByteBuffer dn) throws DatabaseException
{
EntryID nodeID;
//Bypass the cache for append data, lookup the parent DN in the DN2ID
@@ -2113,7 +2165,20 @@
if (importConfiguration != null &&
importConfiguration.appendToExistingData())
{
- nodeID = entryContainer.getDN2ID().get(null, dn, LockMode.DEFAULT);
+ DatabaseEntry key = new DatabaseEntry(dn.array());
+ DatabaseEntry value = new DatabaseEntry();
+ OperationStatus status;
+ status =
+ entryContainer.getDN2ID().read(null, key, value,
+ LockMode.DEFAULT);
+ if(status == OperationStatus.SUCCESS)
+ {
+ nodeID = new EntryID(value);
+ }
+ else
+ {
+ nodeID = null;
+ }
}
else
{
@@ -2137,8 +2202,10 @@
idSet = id2subtreeTree.get(parentID.getDatabaseEntry().getData());
}
idSet.addEntryID(childID);
- for (DN dn = entryContainer.getParentWithinBase(parentDN); dn != null;
- dn = entryContainer.getParentWithinBase(dn))
+ // TODO:
+ // Instead of doing this, we can just walk to parent cache if available
+ for (ByteBuffer dn = getParent(parentDN); dn != null;
+ dn = getParent(dn))
{
EntryID nodeID = getParentID(dn);
if(nodeID == null)
@@ -2167,7 +2234,7 @@
public void writeToDB() throws DatabaseException, DirectoryException
{
- entryContainer.getDN2ID().putRaw(null, DNKey, DNValue);
+ entryContainer.getDN2ID().put(null, DNKey, DNValue);
indexMgr.addTotDNCount(1);
if(parentDN != null)
{
@@ -2850,16 +2917,8 @@
{
getIndexID();
}
- if(indexMgr.isDN2ID())
- {
- rc = dnComparator.compare(keyBuf.array(), 0, keyBuf.limit(),
- cKey.array(), cKey.limit());
- }
- else
- {
- rc = indexComparator.compare(keyBuf.array(), 0, keyBuf.limit(),
+ rc = indexComparator.compare(keyBuf.array(), 0, keyBuf.limit(),
cKey.array(), cKey.limit());
- }
if(rc != 0) {
returnCode = 1;
}
@@ -2888,16 +2947,8 @@
int returnCode;
byte[] oKey = o.getKeyBuf().array();
int oLen = o.getKeyBuf().limit();
- if(indexMgr.isDN2ID())
- {
- returnCode = dnComparator.compare(keyBuf.array(), 0, keyBuf.limit(),
- oKey, oLen);
- }
- else
- {
- returnCode = indexComparator.compare(keyBuf.array(), 0, keyBuf.limit(),
+ returnCode = indexComparator.compare(keyBuf.array(), 0, keyBuf.limit(),
oKey, oLen);
- }
if(returnCode == 0)
{
if(indexID.intValue() == o.getIndexID().intValue())
@@ -4772,7 +4823,7 @@
{
int pLen = PackedInteger.getReadIntLength(bytes, pos);
len = PackedInteger.readInt(bytes, pos);
- if(dnComparator.compare(bytes, pos + pLen, len, dnBytes,
+ if(indexComparator.compare(bytes, pos + pLen, len, dnBytes,
dnBytes.length) == 0)
{
return true;
--
Gitblit v1.10.0