OPENDJ-1308 Migrate schema support

Jean-Noel Rouvignac

08.24.2014 9e6e6a27778d880d913a940738d18324854d1e70

OPENDJ-1308 Migrate schema support

Added unit tests, simplified code and fixed one bug.

AbstractSubstringMatchingRuleImpl.java:
In DefaultSubstringAssertion.matches(), greatly simplified the code.
In rangeMatch(), forced byte comparison to fix a bug where overflowing was not happening due to int comparison being used.
Extracted BACKSLASH constant.

AbstractSubstringMatchingRuleImplTest.java: ADDED

SubstringMatchingRuleTest.java:
Improved the test code.

ApproximateMatchingRuleTest.java:
Fixed a comment.

 opendj-core/src/main/java/org/forgerock/opendj/ldap/schema/AbstractSubstringMatchingRuleImpl.java

@@ -53,6 +53,9 @@
 */
abstract class AbstractSubstringMatchingRuleImpl extends AbstractMatchingRuleImpl {

    /** The backslash character. */
    private static final int BACKSLASH = 0x5C;

    /**
     * Default assertion implementation for substring matching rules.
     * For example, with the assertion value "initial*any1*any2*any3*final",
@@ -97,36 +100,28 @@
                }
            }

            if (normAnys != null && normAnys.length != 0) {
            if (normAnys != null) {
            matchEachSubstring:
                for (final ByteSequence element : normAnys) {
                    final int anyLength = element.length();
                    if (anyLength == 0) {
                        continue;
                    }
                    final int end = valueLength - anyLength;
                    boolean match = false;
                matchCurrentSubstring:
                    for (; pos <= end; pos++) {
                        if (element.byteAt(0) == normalizedAttributeValue.byteAt(pos)) {
                            boolean subMatch = true;
                            for (int i = 1; i < anyLength; i++) {
                                if (element.byteAt(i) != normalizedAttributeValue.byteAt(pos + i)) {
                                    subMatch = false;
                                    break;
                                }
                            }

                            if (subMatch) {
                                match = subMatch;
                                break;
                        // Try to match all characters from the substring
                        for (int i = 0; i < anyLength; i++) {
                            if (element.byteAt(i) != normalizedAttributeValue.byteAt(pos + i)) {
                                // not a match,
                                // try to find a match in the rest of this value
                                continue matchCurrentSubstring;
                            }
                        }
                    }

                    if (match) {
                        // we just matched current substring,
                        // go try to match the next substring
                        pos += anyLength;
                    } else {
                        return ConditionResult.FALSE;
                        continue matchEachSubstring;
                    }
                    // Could not match current substring
                    return ConditionResult.FALSE;
                }
            }

@@ -181,7 +176,7 @@
            final ByteStringBuilder upper = new ByteStringBuilder(lower);

            for (int i = upper.length() - 1; i >= 0; i--) {
                if (upper.byteAt(i) == 0xFF) {
                if (upper.byteAt(i) == (byte) 0xFF) {
                    // We have to carry the overflow to the more significant byte.
                    upper.setByte(i, (byte) 0);
                } else {
@@ -394,7 +389,7 @@
            b = (byte) 0xF0;
            break;
        default:
            if (c1 == 0x5C) {
            if (c1 == BACKSLASH) {
                return c1;
            }
            if (escapeChars != null) {
@@ -483,6 +478,7 @@

    private ByteString evaluateEscapes(final SubstringReader reader, final char[] escapeChars,
            final boolean trim) throws DecodeException {
        // FIXME JNR I believe the trim parameter is dead code
        return evaluateEscapes(reader, escapeChars, escapeChars, trim);
    }

@@ -500,7 +496,7 @@
        reader.mark();
        while (reader.remaining() > 0) {
            c = reader.read();
            if (c == 0x5C /* The backslash character */) {
            if (c == BACKSLASH) {
                if (valueBuffer == null) {
                    valueBuffer = new ByteStringBuilder();
                }

 opendj-core/src/test/java/org/forgerock/opendj/ldap/schema/AbstractSubstringMatchingRuleImplTest.java

New file
@@ -0,0 +1,262 @@
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt
 * or http://forgerock.org/license/CDDLv1.0.html.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at legal-notices/CDDLv1_0.txt.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information:
 *      Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 *      Copyright 2014 ForgeRock AS
 */
package org.forgerock.opendj.ldap.schema;

import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.TreeSet;

import org.fest.assertions.Assertions;
import org.forgerock.opendj.ldap.*;
import org.forgerock.opendj.ldap.spi.IndexQueryFactory;
import org.forgerock.opendj.ldap.spi.Indexer;
import org.forgerock.opendj.ldap.spi.IndexingOptions;
import org.forgerock.util.Utils;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

import static org.forgerock.opendj.ldap.ByteString.*;
import static org.mockito.Mockito.*;
import static org.testng.Assert.*;

/**
 * Tests all generic code of AbstractSubstringMatchingRuleImpl.
 */
@SuppressWarnings("javadoc")
public class AbstractSubstringMatchingRuleImplTest extends SchemaTestCase {

    private static class FakeSubstringMatchingRuleImpl extends AbstractSubstringMatchingRuleImpl {

        /** {@inheritDoc} */
        @Override
        public ByteString normalizeAttributeValue(Schema schema, ByteSequence value) throws DecodeException {
            return value.toByteString();
        }

    }

    private static class FakeIndexQueryFactory implements IndexQueryFactory<String> {

        private final IndexingOptions options;

        public FakeIndexQueryFactory(IndexingOptions options) {
            this.options = options;
        }

        @Override
        public String createExactMatchQuery(String indexID, ByteSequence key) {
            return "exactMatch(" + indexID + ", value=='" + key + "')";
        }

        @Override
        public String createMatchAllQuery() {
            return "matchAll()";
        }

        @Override
        public String createRangeMatchQuery(String indexID, ByteSequence lower,
                ByteSequence upper, boolean lowerIncluded, boolean upperIncluded) {
            final StringBuilder sb = new StringBuilder("rangeMatch");
            sb.append("(");
            sb.append(indexID);
            sb.append(", '");
            sb.append(lower);
            sb.append("' <");
            if (lowerIncluded) {
                sb.append("=");
            }
            sb.append(" value <");
            if (upperIncluded) {
                sb.append("=");
            }
            sb.append(" '");
            sb.append(upper);
            sb.append("')");
            return sb.toString();
        }

        @Override
        public String createIntersectionQuery(Collection<String> subqueries) {
            return "intersect[" + Utils.joinAsString(", ", subqueries) + "]";
        }

        @Override
        public String createUnionQuery(Collection<String> subqueries) {
            return "union[" + Utils.joinAsString(", ", subqueries) + "]";
        }

        @Override
        public IndexingOptions getIndexingOptions() {
            return options;
        }

    }

    private MatchingRuleImpl getRule() {
        return new FakeSubstringMatchingRuleImpl();
    }

    private IndexingOptions newIndexingOptions() {
        final IndexingOptions options = mock(IndexingOptions.class);
        when(options.substringKeySize()).thenReturn(3);
        return options;
    }

    @DataProvider
    public Object[][] invalidAssertions() {
        return new Object[][] {
            { "" },
            { "abc" },
            { "**" },
            { "\\g" },
            { "\\0" },
            { "\\0g" },
        };
    }

    @Test(dataProvider = "invalidAssertions", expectedExceptions = { DecodeException.class })
    public void testInvalidAssertion(String assertionValue) throws Exception {
        getRule().getAssertion(null, valueOf(assertionValue));
    }

    @DataProvider
    public Object[][] validAssertions() {
        return new Object[][] {
            { "this is a string", "*", ConditionResult.TRUE },
            { "this is a string", "that*", ConditionResult.FALSE },
            { "this is a string", "*that", ConditionResult.FALSE },
            { "this is a string", "this*is*a*string", ConditionResult.TRUE },
            { "this is a string", "this*my*string", ConditionResult.FALSE },
            { "this is a string", "string*a*is*this", ConditionResult.FALSE },
            // FIXME next line is not working (StringIndexOutOfBoundsException), is it incorrect?
            // { "this is a string", "\\00", ConditionResult.FALSE },
            // FIXME next line is not working (DecodeException), is it incorrect?
            // { "this is a string", gen(), ConditionResult.FALSE },
            // initial longer than value
            { "tt", "this*", ConditionResult.FALSE },
            { "tt", "*this", ConditionResult.FALSE },
        };
    }

    private String gen() {
        final char[] array = new char[] {
            '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
        final StringBuilder sb = new StringBuilder();
        for (char c : array) {
            sb.append("\\").append(c).append(c);
        }
        return sb.toString();
    }

    @Test(dataProvider = "validAssertions")
    public void testValidAssertions(String attrValue, String assertionValue, ConditionResult expected)
            throws Exception {
        final MatchingRuleImpl rule = getRule();
        final ByteString normValue = rule.normalizeAttributeValue(null, valueOf(attrValue));
        Assertion assertion = rule.getAssertion(null, valueOf(assertionValue));
        assertEquals(assertion.matches(normValue), expected);
    }

    @Test
    @SuppressWarnings("unchecked")
    public void testSubstringCreateIndexQueryForFinalWithMultipleSubqueries() throws Exception {
        Assertion assertion = getRule().getSubstringAssertion(
            null, null, Collections.EMPTY_LIST, valueOf("this"));

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            "intersect["
                    + "exactMatch(substring, value=='his'), "
                    + "exactMatch(substring, value=='thi')"
                    + "]");
    }

    @Test
    public void testSubstringCreateIndexQueryForAllNoSubqueries() throws Exception {
        Assertion assertion = getRule().getSubstringAssertion(
            null, valueOf("abc"), Arrays.asList(toByteStrings("def", "ghi")), valueOf("jkl"));

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            "intersect["
                    + "rangeMatch(equality, 'abc' <= value < 'abd'), "
                    + "exactMatch(substring, value=='def'), "
                    + "exactMatch(substring, value=='ghi'), "
                    + "exactMatch(substring, value=='jkl'), "
                    + "exactMatch(substring, value=='abc')"
                    + "]");
    }

    @Test
    @SuppressWarnings("unchecked")
    public void testSubstringCreateIndexQueryWithInitial() throws Exception {
        Assertion assertion = getRule().getSubstringAssertion(
            null, valueOf("aa"), Collections.EMPTY_LIST, null);

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            "intersect["
                    + "rangeMatch(equality, 'aa' <= value < 'ab'), "
                    + "rangeMatch(substring, 'aa' <= value < 'ab')"
                    + "]");
    }

    @Test
    @SuppressWarnings("unchecked")
    public void testSubstringCreateIndexQueryWithInitialOverflowsInRange() throws Exception {
        ByteString lower = wrap(new byte[] { 'a', (byte) 0XFF });
        Assertion assertion = getRule().getSubstringAssertion(
            null, lower, Collections.EMPTY_LIST, null);

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            // 0x00 is the nul byte, a.k.a. string terminator
            // so everything after it is not part of the string
            "intersect["
                    + "rangeMatch(equality, '" + lower + "' <= value < 'b\u0000'), "
                    + "rangeMatch(substring, '" + lower + "' <= value < 'b\u0000')"
                    + "]");
    }

    @Test
    public void testIndexer() throws Exception {
        final Indexer indexer = getRule().getIndexers().iterator().next();
        Assertions.assertThat(indexer.getIndexID()).isEqualTo("substring");

        final IndexingOptions options = newIndexingOptions();
        final TreeSet<ByteString> keys = new TreeSet<ByteString>();
        indexer.createKeys(Schema.getCoreSchema(), valueOf("ABCDE"), options, keys);
        Assertions.assertThat(keys).containsOnly((Object[]) toByteStrings("ABC", "BCD", "CDE", "DE", "E"));
    }

    private ByteString[] toByteStrings(String... strings) {
        final ByteString[] results = new ByteString[strings.length];
        for (int i = 0; i < strings.length; i++) {
            results[i] = valueOf(strings[i]);
        }
        return results;
    }
}

 opendj-core/src/test/java/org/forgerock/opendj/ldap/schema/ApproximateMatchingRuleTest.java

@@ -62,7 +62,7 @@
    @DataProvider(name = "approximatematchingrules")
    public Object[][] createapproximateMatchingRuleTest() {
        // fill this table with tables containing :
        // - the name of the approxiamtematchingrule to test
        // - the name of the approximate matching rule to test
        // - 2 values that must be tested for matching
        // - a boolean indicating if the values match or not
        return new Object[][] { { metaphone, "celebre", "selebre", ConditionResult.TRUE },

 opendj-core/src/test/java/org/forgerock/opendj/ldap/schema/SubstringMatchingRuleTest.java

@@ -25,7 +25,7 @@
 */
package org.forgerock.opendj.ldap.schema;

import static org.testng.Assert.fail;
import static org.testng.Assert.*;

import java.util.ArrayList;
import java.util.List;
@@ -94,11 +94,13 @@
        // normalize the 2 provided values and check that they are equals
        final ByteString normalizedValue = rule.normalizeAttributeValue(ByteString.valueOf(value));

        if (rule.getSubstringAssertion(null, null, ByteString.valueOf(finalValue)).matches(normalizedValue) != result
                || rule.getAssertion(ByteString.valueOf("*" + finalValue)).matches(normalizedValue) != result) {
            fail("final substring matching rule " + rule + " does not give expected result ("
                    + result + ") for values : " + value + " and " + finalValue);
        }
        final ConditionResult substringAssertionMatches =
            rule.getSubstringAssertion(null, null, ByteString.valueOf(finalValue)).matches(normalizedValue);
        final ConditionResult assertionMatches =
            rule.getAssertion(ByteString.valueOf("*" + finalValue)).matches(normalizedValue);
        final String message = getMessage("final", rule, value, finalValue);
        assertEquals(substringAssertionMatches, result, message);
        assertEquals(assertionMatches, result, message);
    }

    /**
@@ -112,11 +114,19 @@
        // normalize the 2 provided values and check that they are equals
        final ByteString normalizedValue = rule.normalizeAttributeValue(ByteString.valueOf(value));

        if (rule.getSubstringAssertion(ByteString.valueOf(initial), null, null).matches(normalizedValue) != result
                || rule.getAssertion(ByteString.valueOf(initial + "*")).matches(normalizedValue) != result) {
            fail("initial substring matching rule " + rule + " does not give expected result ("
                    + result + ") for values : " + value + " and " + initial);
        }
        final ConditionResult substringAssertionMatches =
            rule.getSubstringAssertion(ByteString.valueOf(initial), null, null).matches(normalizedValue);
        final ConditionResult assertionMatches =
            rule.getAssertion(ByteString.valueOf(initial + "*")).matches(normalizedValue);
        final String message = getMessage("initial", rule, value, initial);
        assertEquals(substringAssertionMatches, result, message);
        assertEquals(assertionMatches, result, message);
    }

    private String getMessage(final String prefix, final MatchingRule rule,
            final String value, final String assertionValue) {
        return prefix + " substring matching rule " + rule
                + " failed for values : \"" + value + "\" and \"" + assertionValue + "\".";
    }

    /**
@@ -182,12 +192,13 @@
            middleList.add(ByteString.valueOf(middleSub));
        }

        if (rule.getSubstringAssertion(null, middleList, null).matches(normalizedValue) != result
                || rule.getAssertion(ByteString.valueOf(printableMiddleSubs)).matches(
                        normalizedValue) != result) {
            fail("middle substring matching rule " + rule + " does not give expected result ("
                    + result + ") for values : " + value + " and " + printableMiddleSubs);
        }
        final ConditionResult substringAssertionMatches =
            rule.getSubstringAssertion(null, middleList, null).matches(normalizedValue);
        final ConditionResult assertionMatches =
            rule.getAssertion(ByteString.valueOf(printableMiddleSubs)).matches(normalizedValue);
        final String message = getMessage("middle", rule, value, printableMiddleSubs.toString());
        assertEquals(substringAssertionMatches, result, message);
        assertEquals(assertionMatches, result, message);
    }

    /**