From 249dd4b1eb993b2e3b9a07203ac50c5384028e76 Mon Sep 17 00:00:00 2001
From: Ludovic Poitou <ludovic.poitou@forgerock.com>
Date: Mon, 16 Apr 2012 10:54:33 +0000
Subject: [PATCH] Fix OPENDJ-469 - LDIFReader has code duplication and suffer from poor performance with highly multi-valued attributes. Refactor to get rid of the readAttribute method that didn't use AttributeBuilders.
---
opends/src/server/org/opends/server/util/LDIFReader.java | 315 +++++++++++-----------------------------------------
1 files changed, 68 insertions(+), 247 deletions(-)
diff --git a/opends/src/server/org/opends/server/util/LDIFReader.java b/opends/src/server/org/opends/server/util/LDIFReader.java
index 64fd6b6..a9d85a6 100644
--- a/opends/src/server/org/opends/server/util/LDIFReader.java
+++ b/opends/src/server/org/opends/server/util/LDIFReader.java
@@ -23,43 +23,36 @@
*
*
* Copyright 2006-2010 Sun Microsystems, Inc.
+ * Portions Copyright 2012 ForgeRock AS
*/
package org.opends.server.util;
-import org.opends.messages.Message;
-import org.opends.messages.MessageBuilder;
-
-
-import static org.opends.server.loggers.debug.DebugLogger.*;
-import org.opends.server.loggers.debug.DebugTracer;
-import static org.opends.server.loggers.ErrorLogger.logError;
-import static org.opends.messages.UtilityMessages.*;
-import static org.opends.server.util.StaticUtils.*;
-import static org.opends.server.util.Validator.*;
-
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
-
+import org.opends.messages.Message;
+import org.opends.messages.MessageBuilder;
+import static org.opends.messages.UtilityMessages.*;
+import org.opends.server.api.plugin.PluginResult;
+import org.opends.server.backends.jeb.EntryID;
+import org.opends.server.backends.jeb.RootContainer;
+import org.opends.server.backends.jeb.importLDIF.Importer;
+import org.opends.server.backends.jeb.importLDIF.Suffix;
import org.opends.server.core.DirectoryServer;
import org.opends.server.core.PluginConfigManager;
+import static org.opends.server.loggers.ErrorLogger.logError;
+import static org.opends.server.loggers.debug.DebugLogger.debugEnabled;
+import static org.opends.server.loggers.debug.DebugLogger.getTracer;
+import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.protocols.ldap.LDAPAttribute;
import org.opends.server.protocols.ldap.LDAPModification;
-
-
import org.opends.server.types.*;
-import org.opends.server.api.plugin.PluginResult;
-import org.opends.server.backends.jeb.RootContainer;
-import org.opends.server.backends.jeb.EntryID;
-import org.opends.server.backends.jeb.importLDIF.Suffix;
-import org.opends.server.backends.jeb.importLDIF.Importer;
+import static org.opends.server.util.StaticUtils.addSuperiorObjectClasses;
+import static org.opends.server.util.StaticUtils.toLowerCase;
+import static org.opends.server.util.Validator.ensureNotNull;
/**
@@ -310,37 +303,70 @@
}
// Read the set of attributes from the entry.
HashMap<ObjectClass,String> objectClasses =
- new HashMap<ObjectClass,String>();
- HashMap<AttributeType,List<Attribute>> userAttributes =
- new HashMap<AttributeType,List<Attribute>>();
- HashMap<AttributeType,List<Attribute>> operationalAttributes =
- new HashMap<AttributeType,List<Attribute>>();
+ new HashMap<ObjectClass,String>();
+ HashMap<AttributeType,List<AttributeBuilder>> userAttrBuilders =
+ new HashMap<AttributeType,List<AttributeBuilder>>();
+ HashMap<AttributeType,List<AttributeBuilder>> operationalAttrBuilders =
+ new HashMap<AttributeType,List<AttributeBuilder>>();
try
{
for (StringBuilder line : lines)
{
- readAttribute(lines, line, entryDN, objectClasses, userAttributes,
- operationalAttributes, checkSchema);
+ readAttribute(lines, line, entryDN, objectClasses, userAttrBuilders,
+ operationalAttrBuilders, checkSchema);
}
}
catch (LDIFException e)
{
- if (debugEnabled())
- {
- TRACER.debugInfo("Skipping entry %s because the it reading" +
- "its attributes failed.", entryDN);
- }
- Message message =
- ERR_LDIF_READ_ATTR_SKIP.get(String.valueOf(entryDN),
+ if (debugEnabled())
+ {
+ TRACER.debugInfo("Skipping entry %s because the it reading" +
+ "its attributes failed.", entryDN);
+ }
+ Message message = ERR_LDIF_READ_ATTR_SKIP.get(String.valueOf(entryDN),
e.getMessage());
- logToSkipWriter(lines, message);
- entriesIgnored.incrementAndGet();
- suffix.removePending(entryDN);
- continue;
+ logToSkipWriter(lines, message);
+ entriesIgnored.incrementAndGet();
+ suffix.removePending(entryDN);
+ continue;
}
// Create the entry and see if it is one that should be included in the
// import.
+ HashMap<AttributeType,List<Attribute>> userAttributes =
+ new HashMap<AttributeType,List<Attribute>>(
+ userAttrBuilders.size());
+ HashMap<AttributeType,List<Attribute>> operationalAttributes =
+ new HashMap<AttributeType,List<Attribute>>(
+ operationalAttrBuilders.size());
+ for (Map.Entry<AttributeType, List<AttributeBuilder>>
+ attrTypeEntry : userAttrBuilders.entrySet())
+ {
+ AttributeType attrType = attrTypeEntry.getKey();
+ List<AttributeBuilder> attrBuilderList = attrTypeEntry.getValue();
+ List<Attribute> attrList =
+ new ArrayList<Attribute>(attrBuilderList.size());
+ for (AttributeBuilder builder : attrBuilderList)
+ {
+ attrList.add(builder.toAttribute());
+ }
+ userAttributes.put(attrType, attrList);
+ }
+ for (Map.Entry<AttributeType, List<AttributeBuilder>>
+ attrTypeEntry : operationalAttrBuilders.entrySet())
+ {
+ AttributeType attrType = attrTypeEntry.getKey();
+ List<AttributeBuilder> attrBuilderList = attrTypeEntry.getValue();
+ List<Attribute> attrList =
+ new ArrayList<Attribute>(attrBuilderList.size());
+ for (AttributeBuilder builder : attrBuilderList)
+ {
+ attrList.add(builder.toAttribute());
+ }
+ operationalAttributes.put(attrType, attrList);
+ }
+ // Create the entry and see if it is one that should be included in the
+ // import.
Entry entry = new Entry(entryDN, objectClasses, userAttributes,
operationalAttributes);
TRACER.debugProtocolElement(DebugLogLevel.VERBOSE, entry.toString());
@@ -2162,211 +2188,6 @@
}
- private void readAttribute(LinkedList<StringBuilder> lines,
- StringBuilder line, DN entryDN, Map<ObjectClass,String> objectClasses,
- Map<AttributeType,List<Attribute>> userAttributes,
- Map<AttributeType,List<Attribute>> operationalAttributes,
- boolean checkSchema) throws LDIFException
- {
- // Parse the attribute type description.
- int colonPos = parseColonPosition(lines, line);
- String attrDescr = line.substring(0, colonPos);
- final Attribute attribute = parseAttrDescription(attrDescr);
- final String attrName = attribute.getName();
- final String lowerName = toLowerCase(attrName);
-
- // Now parse the attribute value.
- ByteString value = parseSingleValue(lines, line, entryDN,
- colonPos, attrName);
-
- // See if this is an objectclass or an attribute. Then get the
- // corresponding definition and add the value to the appropriate hash.
- if (lowerName.equals("objectclass"))
- {
- if (! importConfig.includeObjectClasses())
- {
- if (debugEnabled())
- {
- TRACER.debugVerbose("Skipping objectclass %s for entry %s due to " +
- "the import configuration.", value, entryDN);
- }
- return;
- }
-
- String ocName = value.toString();
- String lowerOCName = toLowerCase(ocName);
-
- ObjectClass objectClass = DirectoryServer.getObjectClass(lowerOCName);
- if (objectClass == null)
- {
- objectClass = DirectoryServer.getDefaultObjectClass(ocName);
- }
-
- if (objectClasses.containsKey(objectClass))
- {
- logError(WARN_LDIF_DUPLICATE_OBJECTCLASS.get(
- String.valueOf(entryDN), lastEntryLineNumber, ocName));
- }
- else
- {
- objectClasses.put(objectClass, ocName);
- }
- }
- else
- {
- AttributeType attrType = DirectoryServer.getAttributeType(lowerName);
- if (attrType == null)
- {
- attrType = DirectoryServer.getDefaultAttributeType(attrName);
- }
-
-
- if (! importConfig.includeAttribute(attrType))
- {
- if (debugEnabled())
- {
- TRACER.debugVerbose("Skipping attribute %s for entry %s due to the " +
- "import configuration.", attrName, entryDN);
- }
- return;
- }
-
- //The attribute is not being ignored so check for binary option.
- if(checkSchema && !attrType.isBinary())
- {
- if(attribute.hasOption("binary"))
- {
- Message message = ERR_LDIF_INVALID_ATTR_OPTION.get(
- String.valueOf(entryDN),lastEntryLineNumber, attrName);
- logToRejectWriter(lines, message);
- throw new LDIFException(message, lastEntryLineNumber,true);
- }
- }
- if (checkSchema &&
- (DirectoryServer.getSyntaxEnforcementPolicy() !=
- AcceptRejectWarn.ACCEPT))
- {
- MessageBuilder invalidReason = new MessageBuilder();
- if (! attrType.getSyntax().valueIsAcceptable(value, invalidReason))
- {
- Message message = WARN_LDIF_VALUE_VIOLATES_SYNTAX.get(
- String.valueOf(entryDN),
- lastEntryLineNumber, value.toString(),
- attrName, invalidReason.toString());
- if (DirectoryServer.getSyntaxEnforcementPolicy() ==
- AcceptRejectWarn.WARN)
- {
- logError(message);
- }
- else
- {
- logToRejectWriter(lines, message);
- throw new LDIFException(message, lastEntryLineNumber,
- true);
- }
- }
- }
-
- AttributeValue attributeValue =
- AttributeValues.create(attrType, value);
- List<Attribute> attrList;
- if (attrType.isOperational())
- {
- attrList = operationalAttributes.get(attrType);
- if (attrList == null)
- {
- AttributeBuilder builder = new AttributeBuilder(attribute, true);
- builder.add(attributeValue);
- attrList = new ArrayList<Attribute>();
- attrList.add(builder.toAttribute());
- operationalAttributes.put(attrType, attrList);
- return;
- }
- }
- else
- {
- attrList = userAttributes.get(attrType);
- if (attrList == null)
- {
- AttributeBuilder builder = new AttributeBuilder(attribute, true);
- builder.add(attributeValue);
- attrList = new ArrayList<Attribute>();
- attrList.add(builder.toAttribute());
- userAttributes.put(attrType, attrList);
- return;
- }
- }
-
-
- // Check to see if any of the attributes in the list have the same set of
- // options. If so, then try to add a value to that attribute.
- for (int i = 0; i < attrList.size(); i++) {
- Attribute a = attrList.get(i);
-
- if (a.optionsEqual(attribute.getOptions()))
- {
- if (a.contains(attributeValue))
- {
- if (! checkSchema)
- {
- // If we're not doing schema checking, then it is possible that
- // the attribute type should use case-sensitive matching and the
- // values differ in capitalization. Only reject the proposed
- // value if we find another value that is exactly the same as the
- // one that was provided.
- for (AttributeValue v : a)
- {
- if (v.getValue().equals(attributeValue.getValue()))
- {
- Message message = WARN_LDIF_DUPLICATE_ATTR.get(
- String.valueOf(entryDN),
- lastEntryLineNumber, attrName,
- value.toString());
- logToRejectWriter(lines, message);
- throw new LDIFException(message, lastEntryLineNumber,
- true);
- }
- }
- }
- else
- {
- Message message = WARN_LDIF_DUPLICATE_ATTR.get(
- String.valueOf(entryDN),
- lastEntryLineNumber, attrName,
- value.toString());
- logToRejectWriter(lines, message);
- throw new LDIFException(message, lastEntryLineNumber,
- true);
- }
- }
-
- if (attrType.isSingleValue() && !a.isEmpty() && checkSchema)
- {
- Message message = ERR_LDIF_MULTIPLE_VALUES_FOR_SINGLE_VALUED_ATTR
- .get(String.valueOf(entryDN),
- lastEntryLineNumber, attrName);
- logToRejectWriter(lines, message);
- throw new LDIFException(message, lastEntryLineNumber, true);
- }
-
- AttributeBuilder builder = new AttributeBuilder(a);
- builder.add(attributeValue);
- attrList.set(i, builder.toAttribute());
- return;
- }
- }
-
-
- // No set of matching options was found, so create a new one and
- // add it to the list.
- AttributeBuilder builder = new AttributeBuilder(attribute, true);
- builder.add(attributeValue);
- attrList.add(builder.toAttribute());
- }
- }
-
-
-
/**
* Adds any missing RDN attributes to the entry that is being imported.
*/
--
Gitblit v1.10.0