OPENDJ-2731 Middle and final substring indexes fail to return resulting in ...

Fabio Pistolesi

02.13.2016 1a7e6221734e874231667f3a8a3896c2be06b4f3

OPENDJ-2731 Middle and final substring indexes fail to return resulting in an unindexed search.

Aligning Indexer and SubString Assertion with regards to generated name for
substring indexes, who must contain the substring length configured for
the index, since it is used for the name of the tree in the Storage and
for the index itself.

 opendj-sdk/opendj-core/src/main/java/org/forgerock/opendj/ldap/schema/AbstractSubstringMatchingRuleImpl.java

@@ -12,7 +12,7 @@
 * information: "Portions Copyright [year] [name of copyright owner]".
 *
 * Copyright 2009 Sun Microsystems, Inc.
 * Portions copyright 2014-2015 ForgeRock AS.
 * Portions copyright 2014-2016 ForgeRock AS.
 */
package org.forgerock.opendj.ldap.schema;

@@ -185,12 +185,13 @@

        private <T> void substringMatch(final IndexQueryFactory<T> factory, final ByteString normSubstring,
                final Collection<T> subqueries) {
            int substrLength = factory.getIndexingOptions().substringKeySize();
            final int substrLength = factory.getIndexingOptions().substringKeySize();
            final String indexId = substringIndexId + ":" + substrLength;

            // There are two cases, depending on whether the user-provided
            // substring is smaller than the configured index substring length or not.
            if (normSubstring.length() < substrLength) {
                subqueries.add(rangeMatch(factory, substringIndexId, normSubstring));
                subqueries.add(rangeMatch(factory, indexId, normSubstring));
            } else {
                // Break the value up into fragments of length equal to the
                // index substring length, and read those keys.
@@ -206,7 +207,7 @@
                }

                for (ByteSequence key : substringKeys) {
                    subqueries.add(factory.createExactMatchQuery(substringIndexId, key));
                    subqueries.add(factory.createExactMatchQuery(indexId, key));
                }
            }
        }

 opendj-sdk/opendj-core/src/test/java/org/forgerock/opendj/ldap/schema/AbstractSubstringMatchingRuleImplTest.java

@@ -11,7 +11,7 @@
 * Header, with the fields enclosed by brackets [] replaced by your own identifying
 * information: "Portions Copyright [year] [name of copyright owner]".
 *
 * Copyright 2014-2015 ForgeRock AS.
 * Copyright 2014-2016 ForgeRock AS.
 */
package org.forgerock.opendj.ldap.schema;

@@ -44,6 +44,8 @@
@SuppressWarnings("javadoc")
public class AbstractSubstringMatchingRuleImplTest extends AbstractSchemaTestCase {

    private int subStringLength = 3;

    private static class FakeSubstringMatchingRuleImpl extends AbstractSubstringMatchingRuleImpl {

        FakeSubstringMatchingRuleImpl() {
@@ -135,8 +137,12 @@
    }

    static IndexingOptions newIndexingOptions() {
        return newIndexingOptions(3);
    }

    static IndexingOptions newIndexingOptions(int subStringLength) {
        final IndexingOptions options = mock(IndexingOptions.class);
        when(options.substringKeySize()).thenReturn(3);
        when(options.substringKeySize()).thenReturn(subStringLength);
        return options;
    }

@@ -188,6 +194,10 @@
        return sb.toString();
    }

    private String subStringIndexID(String matchingRule) {
        return matchingRule + ":" + subStringLength;
    }

    @Test(dataProvider = "validAssertions")
    public void testValidAssertions(String attrValue, String assertionValue, ConditionResult expected)
            throws Exception {
@@ -204,10 +214,10 @@
            null, null, Collections.EMPTY_LIST, valueOfUtf8("this"));

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions(subStringLength))),
            "intersect["
                    + "exactMatch(" + SMR_CASE_EXACT_OID + ", value=='his'), "
                    + "exactMatch(" + SMR_CASE_EXACT_OID + ", value=='thi')"
                    + "exactMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", value=='his'), "
                    + "exactMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", value=='thi')"
                    + "]");
    }

@@ -217,13 +227,13 @@
            null, valueOfUtf8("abc"), Arrays.asList(toByteStrings("def", "ghi")), valueOfUtf8("jkl"));

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions(subStringLength))),
            "intersect["
                    + "rangeMatch(" + EMR_CASE_EXACT_OID + ", 'abc' <= value < 'abd'), "
                    + "exactMatch(" + SMR_CASE_EXACT_OID + ", value=='def'), "
                    + "exactMatch(" + SMR_CASE_EXACT_OID + ", value=='ghi'), "
                    + "exactMatch(" + SMR_CASE_EXACT_OID + ", value=='jkl'), "
                    + "exactMatch(" + SMR_CASE_EXACT_OID + ", value=='abc')"
                    + "exactMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", value=='def'), "
                    + "exactMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", value=='ghi'), "
                    + "exactMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", value=='jkl'), "
                    + "exactMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", value=='abc')"
                    + "]");
    }

@@ -234,10 +244,10 @@
            null, valueOfUtf8("aa"), Collections.EMPTY_LIST, null);

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions(subStringLength))),
            "intersect["
                    + "rangeMatch(" + EMR_CASE_EXACT_OID + ", 'aa' <= value < 'ab'), "
                    + "rangeMatch(" + SMR_CASE_EXACT_OID + ", 'aa' <= value < 'ab')"
                    + "rangeMatch(" + EMR_CASE_EXACT_OID +  ", 'aa' <= value < 'ab'), "
                    + "rangeMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", 'aa' <= value < 'ab')"
                    + "]");
    }

@@ -249,12 +259,12 @@
            null, lower, Collections.EMPTY_LIST, null);

        assertEquals(
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions())),
            assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions(subStringLength))),
            // 0x00 is the nul byte, a.k.a. string terminator
            // so everything after it is not part of the string
            "intersect["
                    + "rangeMatch(" + EMR_CASE_EXACT_OID + ", '" + lower + "' <= value < 'b\u0000'), "
                    + "rangeMatch(" + SMR_CASE_EXACT_OID + ", '" + lower + "' <= value < 'b\u0000')"
                    + "rangeMatch(" + subStringIndexID(SMR_CASE_EXACT_OID) + ", '" + lower + "' <= value < 'b\u0000')"
                    + "]");
    }


 opendj-sdk/opendj-core/src/test/java/org/forgerock/opendj/ldap/schema/CollationSubstringMatchingRuleTest.java

@@ -145,15 +145,17 @@
        MatchingRule matchingRule = getRule();
        Assertion assertion = matchingRule.getAssertion(value);

        String indexQuery = assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions(), false));
        int subStringLength = 3;
        String indexQuery = assertion.createIndexQuery(new FakeIndexQueryFactory(newIndexingOptions(subStringLength),
            false));

        ByteString binit = matchingRule.normalizeAttributeValue(ByteString.valueOfUtf8("a"));
        ByteString bfinal = matchingRule.normalizeAttributeValue(ByteString.valueOfUtf8("c"));
        assertEquals(indexQuery,
            "intersect["
            + "rangeMatch(fr.shared, '" + binit.toHexString() + "' <= value < '0054'), "
            + "rangeMatch(fr.substring, '" + bfinal.toHexString() + "' <= value < '0056'), "
            + "rangeMatch(fr.substring, '" + binit.toHexString() + "' <= value < '0054')]"
            + "rangeMatch(fr.substring:" + subStringLength + ", '" + bfinal.toHexString() + "' <= value < '0056'), "
            + "rangeMatch(fr.substring:" + subStringLength + ", '" + binit.toHexString() + "' <= value < '0054')]"
        );
    }
}