Fix relating to issues 2813 and 2578. Make DN string representations more u...

matthew_swift

13.00.2008 4706235303e2afabe1e63a0e2fc44a5b0f785b0f

Fix relating to issues 2813 and 2578. Make DN string representations more user-friendly when they contain non-ascii characters.

This change is a flag day due to the potential for database format incompatibilities introduced by the change in DN normalized form.

Currently the DN and RDN implementations are very conservative regarding the string representation of DNs that they construct. Any non-ascii characters are escaped using back-slashes. For example, the DN:

   uid=Météo.0,ou=People,dc=example,dc=com

Is encoded as:

   uid=M\c3\a9t\c3\a9o.0,ou=People,dc=example,dc=com

Which is not very readable in LDAP client applications. It is also much less space efficient - something we should consider if we wish to have non-western users of OpenDS who will be heavy users of multi-byte UTF8 sequences. For example, a single Chinese character would be encoded in UTF8 as 3 or 4 bytes IIRC which would equate to 9-12 bytes or a 3X increase. This would have implications for database performance (substrings) and space efficiency.

The change is not without its minor problems however:

  1. LDIF cannot contain non-ascii characters so any DNs or attribute
     values must be base-64 encoded in order for the LDIF to be valid.
     This is not very user-friendly, but it's easier for inquiring
     users to decode base 64 than to manually decode UTF8 byte
     sequences. A future change could improve this behavior by making
     our LDIF generation tools (e.g. ldapsearch, ldif-export) output
     comments before each base-64 encoded DN / value containing the DN
     / value in the client's native character set. This is something
     that OpenLDAP clients do and I think it is a nice usability feature

  2. the dn2id index and any DN / RDN syntax attribute indexes will be
     potentially invalid due to the modified DN / RDN normalization
     (hence this change is a flag-day)

  3. DNs returned to LDAPv2 clients will potentially contain non-T.61
     characters (LDAPv3 uses UTF8 and LDAPv2 uses T.61). However, I
     don't think we are bothered by this because we already break
     compatibility for LDAPv2 clients for directory string based
     attribute values which we also return using UTF8.

 opends/src/server/org/opends/server/types/AttributeValue.java

@@ -28,14 +28,13 @@



import org.opends.server.api.EqualityMatchingRule;
import org.opends.server.protocols.asn1.ASN1OctetString;

import static org.opends.server.loggers.debug.DebugLogger.*;
import org.opends.server.loggers.debug.DebugTracer;
import static org.opends.server.util.StaticUtils.*;
import static org.opends.server.util.Validator.*;

import org.opends.server.api.EqualityMatchingRule;
import org.opends.server.loggers.debug.DebugTracer;
import org.opends.server.protocols.asn1.ASN1OctetString;



/**
@@ -241,114 +240,6 @@



  /**
   * Retrieves a string representation of the user-defined form of
   * this attribute value in a form suitable for use in a DN.
   *
   * @return  A string representation of the user-defined form of this
   *          attribute value in a form suitable for use in a DN.
   */
  public String getDNStringValue()
  {
    return getDNValue(getStringValue());
  }



  /**
   * Retrieves a string representation of the normalized form of this
   * attribute value in a form suitable for use in a DN.
   *
   * @return  A string representation of the normalized form of this
   *          attribute value in a form suitable for use in a DN.
   *
   * @throws  DirectoryException  If an error occurs while trying to
   *                              normalize the value (e.g., if it is
   *                              not acceptable for use with the
   *                              associated equality matching rule).
   */
  public String getNormalizedDNStringValue()
         throws DirectoryException
  {
    return getDNValue(getNormalizedStringValue());
  }



  /**
   * Retrieves a version of the provided value in a form that is
   * properly escaped for use in a DN or RDN.
   *
   * @param  value  The value to be represented in a DN-safe form.
   *
   * @return  A version of the provided value in a form that is
   *          properly escaped for use in a DN or RDN.
   */
  private static String getDNValue(String value)
  {
    if ((value == null) || (value.length() == 0))
    {
      return "";
    }

    StringBuilder buffer = new StringBuilder(value);

    int length = buffer.length();
    for (int i=0; i < length; i++)
    {
      char c = buffer.charAt(i);

      if ((c < ' ') || (c > '~'))
      {
        buffer.deleteCharAt(i);
        length -= 1;

        for (byte b : getBytes(String.valueOf(c)))
        {
          buffer.insert(i++, "\\");
          buffer.insert(i++, byteToLowerHex(b));
          i++;

          length += 3;
        }

        i -= 1;
      }
      else
      {
        switch (buffer.charAt(i))
        {
          case ',':
          case '+':
          case '"':
          case '\\':
          case '<':
          case '>':
          case ';':
            buffer.insert(i++, '\\');
            length++;
        }
      }
    }

    char c = buffer.charAt(0);
    if ((c == ' ') || (c == '#'))
    {
      buffer.insert(0, '\\');
      length++;
    }

    if (buffer.charAt(length-1) == ' ')
    {
      buffer.insert(length-1, '\\');
      length++;
    }

    return buffer.toString();
  }




  /**
   * Determines whether this attribute value is equal to the provided

 opends/src/server/org/opends/server/types/RDN.java

@@ -451,14 +451,120 @@


  /**
   * Retrieves a version of the provided value in a form that is
   * properly escaped for use in a DN or RDN.
   *
   * @param  value  The value to be represented in a DN-safe form.
   *
   * @return  A version of the provided value in a form that is
   *          properly escaped for use in a DN or RDN.
   */
  private static String getDNValue(String value) {
    if ((value == null) || (value.length() == 0)) {
      return "";
    }

    // Only copy the string value if required.
    boolean needsEscaping = false;
    int length = value.length();

    needsEscaping: {
      char c = value.charAt(0);
      if ((c == ' ') || (c == '#')) {
        needsEscaping = true;
        break needsEscaping;
      }

      if (value.charAt(length - 1) == ' ') {
        needsEscaping = true;
        break needsEscaping;
      }

      for (int i = 0; i < length; i++) {
        c = value.charAt(i);
        if (c < ' ') {
          needsEscaping = true;
          break needsEscaping;
        } else {
          switch (c) {
          case ',':
          case '+':
          case '"':
          case '\\':
          case '<':
          case '>':
          case ';':
            needsEscaping = true;
            break needsEscaping;
          }
        }
      }
    }

    if (!needsEscaping) {
      return value;
    }

    // We need to copy and escape the string (allow for at least one
    // escaped character).
    StringBuilder buffer = new StringBuilder(length + 3);

    // If the lead character is a space or a # it must be escaped.
    int start = 0;
    char c = value.charAt(0);
    if ((c == ' ') || (c == '#')) {
      buffer.append('\\');
      buffer.append(c);
      start = 1;
    }

    // Escape remaining characters as necessary.
    for (int i = start; i < length; i++) {
      c = value.charAt(i);
      if (c < ' ') {
        for (byte b : getBytes(String.valueOf(c))) {
          buffer.append('\\');
          buffer.append(byteToLowerHex(b));
        }
      } else {
        switch (value.charAt(i)) {
        case ',':
        case '+':
        case '"':
        case '\\':
        case '<':
        case '>':
        case ';':
          buffer.append('\\');
          buffer.append(c);
          break;
        default:
          buffer.append(c);
          break;
        }
      }
    }

    // If the last character is a space it must be escaped.
    if (value.charAt(length - 1) == ' ') {
      length = buffer.length();
      buffer.insert(length - 1, '\\');
    }

    return buffer.toString();
  }



  /**
   * Decodes the provided string as an RDN.
   *
   * @param  rdnString  The string to decode as an RDN.
   *
   * @return  The decoded RDN.
   *
   * @throws  DirectoryException  If a problem occurs while trying to
   *                              decode the provided string as a RDN.
   * @param rdnString
   *          The string to decode as an RDN.
   * @return The decoded RDN.
   * @throws DirectoryException
   *           If a problem occurs while trying to decode the provided
   *           string as a RDN.
   */
  public static RDN decode(String rdnString)
         throws DirectoryException
@@ -899,14 +1005,18 @@

      buffer.append(attributeNames[0]);
      buffer.append("=");
      buffer.append(attributeValues[0].getDNStringValue());

      String s = attributeValues[0].getStringValue();
      buffer.append(getDNValue(s));

      for (int i=1; i < numValues; i++)
      {
        buffer.append("+");
        buffer.append(attributeNames[i]);
        buffer.append("=");
        buffer.append(attributeValues[i].getDNStringValue());

        s = attributeValues[i].getStringValue();
        buffer.append(getDNValue(s));
      }

      rdnString = buffer.toString();
@@ -972,8 +1082,8 @@

      try
      {
        buffer.append(
             attributeValues[0].getNormalizedDNStringValue());
        String s = attributeValues[0].getNormalizedStringValue();
        buffer.append(getDNValue(s));
      }
      catch (Exception e)
      {
@@ -982,7 +1092,8 @@
          TRACER.debugCaught(DebugLogLevel.ERROR, e);
        }

        buffer.append(attributeValues[0].getStringValue());
        String s = attributeValues[0].getStringValue();
        buffer.append(getDNValue(s));
      }
    }
    else
@@ -997,7 +1108,8 @@

        try
        {
          b2.append(attributeValues[i].getNormalizedStringValue());
          String s = attributeValues[i].getNormalizedStringValue();
          b2.append(getDNValue(s));
        }
        catch (Exception e)
        {
@@ -1006,7 +1118,8 @@
            TRACER.debugCaught(DebugLogLevel.ERROR, e);
          }

          b2.append(attributeValues[i].getStringValue());
          String s = attributeValues[i].getStringValue();
          b2.append(getDNValue(s));
        }

        rdnElementStrings.add(b2.toString());
@@ -1048,7 +1161,6 @@
    {
      if (attributeTypes[0].equals(rdn.attributeTypes[0]))
      {
        int valueComparison;
        OrderingMatchingRule omr =
             attributeTypes[0].getOrderingMatchingRule();
        if (omr == null)

 opends/tests/unit-tests-testng/src/server/org/opends/server/types/TestDN.java

@@ -104,11 +104,11 @@
            "1.3.6.1.4.1.1466.0=\\04\\02hi",
            "1.3.6.1.4.1.1466.0=\\04\\02Hi" },
        { "1.1.1=", "1.1.1=", "1.1.1=" },
        { "CN=Lu\\C4\\8Di\\C4\\87", "cn=lu\\c4\\8di\\c4\\87",
            "CN=Lu\\c4\\8di\\c4\\87" },
        { "CN=Lu\\C4\\8Di\\C4\\87", "cn=lu\u010di\u0107",
            "CN=Lu\u010di\u0107" },
        { "ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8,o=Airius",
            "ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8,o=airius",
            "ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8,o=Airius" },
            "ou=\u55b6\u696d\u90e8,o=airius",
            "ou=\u55b6\u696d\u90e8,o=Airius" },
        { "photo=\\ john \\ ,dc=com", "photo=\\ john \\ ,dc=com",
            "photo=\\ john \\ ,dc=com" },
        { "AB-global=", "ab-global=", "AB-global=" },

 opends/tests/unit-tests-testng/src/server/org/opends/server/types/TestRDN.java

@@ -228,11 +228,11 @@
        { "1.3.6.1.4.1.1466.0=#04024869",
            "1.3.6.1.4.1.1466.0=\\04\\02hi",
            "1.3.6.1.4.1.1466.0=\\04\\02Hi" },
        { "CN=Lu\\C4\\8Di\\C4\\87", "cn=lu\\c4\\8di\\c4\\87",
            "CN=Lu\\c4\\8di\\c4\\87" },
        { "CN=Lu\\C4\\8Di\\C4\\87", "cn=lu\u010di\u0107",
            "CN=Lu\u010di\u0107" },
        { "ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8",
            "ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8",
            "ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8" },
            "ou=\u55b6\u696d\u90e8",
            "ou=\u55b6\u696d\u90e8" },
        { "photo=\\ john \\ ", "photo=\\ john \\ ",
            "photo=\\ john \\ " },
        { "AB-global=", "ab-global=", "AB-global=" },

 opends/tests/unit-tests-testng/src/server/org/opends/server/util/TestLDIFWriter.java

@@ -26,7 +26,7 @@
 */
package org.opends.server.util;

import static org.opends.server.util.StaticUtils.toLowerCase;
import static org.opends.server.util.StaticUtils.*;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
@@ -36,10 +36,9 @@
import java.util.LinkedList;
import java.util.List;

import org.opends.server.TestCaseUtils;
import org.opends.messages.Message;
import org.opends.server.TestCaseUtils;
import org.opends.server.core.DirectoryServer;
import org.opends.server.protocols.ldap.LDAPModification;
import org.opends.server.types.Attribute;
import org.opends.server.types.DN;
import org.opends.server.types.Entry;
@@ -167,7 +166,7 @@
        "changetype: modify\n" +
        "delete: description\n" +
        "\n",
        "dn: uid=rogasawara,ou=\\e5\\96\\b6\\e6\\a5\\ad\\e9\\83\\a8,o=Airius\n" +
        "dn:: dWlkPXJvZ2FzYXdhcmEsb3U95Za25qWt6YOoLG89QWlyaXVz\n" +
        "changetype: modify\n" +
        "add: description\n" +
        "description:: dWlkPXJvZ2FzYXdhcmEsb3U95Za25qWt6YOoLG89QWlyaXVz" +
@@ -258,7 +257,7 @@
  }

  /**
   * Test the {@link LDIFWriter#writeComment(String, int)} method.
   * Test the {@link LDIFWriter#writeComment(Message, int)} method.
   *
   * @param comment
   *          The input comment string.