mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
opendj-sdk/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/ImportRecord.java
New file @@ -0,0 +1,161 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at legal-notices/CDDLv1_0.txt * or http://forgerock.org/license/CDDLv1.0.html. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at legal-notices/CDDLv1_0.txt. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: * Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END * * Copyright 2015 ForgeRock AS */ package org.opends.server.backends.pluggable; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteString; /** * Record for import composed of a byte sequence key and an indexID. */ final class ImportRecord implements Comparable<ImportRecord> { /** The number of bytes of a Java int. */ static final int INT_SIZE = 4; /** The number of bytes of a Java long. */ static final int LONG_SIZE = 8; /** * The record overhead. In addition to entryID, key length and key bytes, the record overhead * includes the INS/DEL bit + indexID */ private static final int REC_OVERHEAD = 1 + INT_SIZE; static ImportRecord fromBufferAndPosition(byte[] buffer, int position) { int offSet = readOffset(buffer, position); int indexID = readIndexIDFromOffset(buffer, offSet); offSet += REC_OVERHEAD + LONG_SIZE; int keyLength = readInt(buffer, offSet); ByteString key = ByteString.wrap(buffer, INT_SIZE + offSet, keyLength); return new ImportRecord(key, indexID); } static ImportRecord fromBufferAndIndexID(byte[] buffer, int indexID) { int offSet = readOffset(buffer, 0); offSet += REC_OVERHEAD + LONG_SIZE; int keyLength = readInt(buffer, offSet); ByteString key = ByteString.wrap(buffer, INT_SIZE + offSet, keyLength); return new ImportRecord(key, indexID); } static ImportRecord from(ByteSequence key, int indexID) { return new ImportRecord(key, indexID); } private static int readOffset(byte[] buffer, int position) { return readInt(buffer, position * INT_SIZE); } private static int readIndexIDFromOffset(byte[] buffer, int offset) { return readInt(buffer, offset + 1); } private static int readInt(byte[] buffer, int index) { int answer = 0; for (int i = 0; i < INT_SIZE; i++) { byte b = buffer[index + i]; answer <<= 8; answer |= (b & 0xff); } return answer; } private final ByteSequence key; /** * The indexID, computed by calling {@link System#identityHashCode(Object)} * on the in-memory {@link Index} object. */ private final int indexID; public ImportRecord(ByteSequence key, int indexID) { this.key = key; this.indexID = indexID; } public int getIndexID() { return indexID; } public ByteSequence getKey() { return key; } @Override public int compareTo(ImportRecord o) { if (o == null) { return -1; } int cmp = key.compareTo(o.getKey()); if (cmp == 0) { return indexID - o.getIndexID(); } return cmp; } @Override public boolean equals(Object o) { if (this == o) { return true; } if (o instanceof ImportRecord) { return this.compareTo((ImportRecord) o) == 0; } return false; } @Override public int hashCode() { final int prime = 31; int result = 1; result = prime * result + indexID; result = prime * result + ((key == null) ? 0 : key.hashCode()); return result; } /** {@inheritDoc} */ @Override public String toString() { return "ImportRecord(key=" + key + ", indexID=" + indexID + ")"; } } opendj-sdk/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/Importer.java
@@ -162,8 +162,6 @@ /** The DN attribute type. */ private static final AttributeType dnType; static final IndexOutputBuffer.IndexComparator indexComparator = new IndexOutputBuffer.IndexComparator(); /** Phase one buffer count. */ private final AtomicInteger bufferCount = new AtomicInteger(0); @@ -1991,7 +1989,7 @@ final ByteStringBuilder key = new ByteStringBuilder(BYTE_BUFFER_CAPACITY); ImportIDSet insertIDSet = null; ImportIDSet deleteIDSet = null; Integer indexID = null; ImportRecord previousRecord = null; try { beginWriteTask(); @@ -2007,17 +2005,17 @@ while (!bufferSet.isEmpty()) { IndexInputBuffer b = bufferSet.pollFirst(); if (!b.sameKeyAndIndexID(key, indexID)) if (!b.currentRecord().equals(previousRecord)) { if (indexID != null) if (previousRecord != null) { // save the previous record addToDB(indexID, insertIDSet, deleteIDSet); addToDB(previousRecord.getIndexID(), insertIDSet, deleteIDSet); } // this is a new record, reinitialize all indexID = b.getIndexID(); int indexID = b.getIndexID(); b.fetchKey(key); previousRecord = ImportRecord.from(key, indexID); insertIDSet = newImportIDSet(key, indexID); deleteIDSet = newImportIDSet(key, indexID); @@ -2034,9 +2032,9 @@ } } if (indexID != null) if (previousRecord != null) { addToDB(indexID, insertIDSet, deleteIDSet); addToDB(previousRecord.getIndexID(), insertIDSet, deleteIDSet); } } return null; @@ -2444,7 +2442,7 @@ else if (!indexBuffer.sameKeyAndIndexID(i)) { // this is a new record, save previous record ... bufferLen += writeRecord(indexBuffer); bufferLen += writeRecord(indexBuffer.currentRecord()); // ... and reinitialize all indexBuffer.setPosition(i); resetStreams(); @@ -2454,7 +2452,7 @@ if (numberKeys > 0) { // save the last record bufferLen += writeRecord(indexBuffer); bufferLen += writeRecord(indexBuffer.currentRecord()); } return bufferLen; } @@ -2478,22 +2476,20 @@ indexSortedSet.add(b); } } byte[] saveKey = null; int saveIndexID = 0; ImportRecord previousRecord = null; while (!indexSortedSet.isEmpty()) { final IndexOutputBuffer b = indexSortedSet.pollFirst(); if (!b.sameKeyAndIndexID(saveKey, saveIndexID)) if (!b.currentRecord().equals(previousRecord)) { if (saveKey != null) if (previousRecord != null) { // save the previous record bufferLen += writeRecord(saveKey, saveIndexID); bufferLen += writeRecord(previousRecord); resetStreams(); } // this is a new record, reinitialize all saveKey = b.getKey(); saveIndexID = b.getIndexID(); previousRecord = b.currentRecord(); } appendNextEntryIDToStream(b, b.getPosition()); @@ -2504,9 +2500,9 @@ indexSortedSet.add(b); } } if (saveKey != null) if (previousRecord != null) { bufferLen += writeRecord(saveKey, saveIndexID); bufferLen += writeRecord(previousRecord); } return bufferLen; } @@ -2570,20 +2566,12 @@ return 2 * INT_SIZE; } private int writeRecord(IndexOutputBuffer b) throws IOException private int writeRecord(ImportRecord record) throws IOException { int keySize = b.getKeySize(); int headerSize = writeHeader(b.getIndexID(), keySize); b.writeKey(bufferStream); int bodySize = writeByteStreams(); return headerSize + keySize + bodySize; } private int writeRecord(byte[] k, int indexID) throws IOException { int keySize = k.length; int headerSize = writeHeader(indexID, keySize); bufferStream.write(k); final ByteSequence key = record.getKey(); int keySize = key.length(); int headerSize = writeHeader(record.getIndexID(), keySize); key.copyTo(bufferStream); int bodySize = writeByteStreams(); return headerSize + keySize + bodySize; } @@ -4039,7 +4027,9 @@ { int pLen = INT_SIZE; int len = reader.getInt(); if (indexComparator.compare(existingDnsBytes, previousPos+pLen, len, dn.getBackingArray(), dn.length()) == 0) ImportRecord r1 = ImportRecord.from(ByteString.wrap(existingDnsBytes, previousPos + pLen, len), 0); ImportRecord r2 = ImportRecord.from(dn, 0); if (r1.equals(r2)) { return true; } opendj-sdk/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexInputBuffer.java
@@ -34,6 +34,7 @@ import java.nio.channels.FileChannel; import org.forgerock.i18n.slf4j.LocalizedLogger; import org.forgerock.opendj.ldap.ByteString; import org.forgerock.opendj.ldap.ByteStringBuilder; import org.opends.server.backends.pluggable.Importer.IndexManager; @@ -62,7 +63,7 @@ private final ByteBuffer cache; /** Next fields are the fetched record data. */ private Integer indexID; private ImportRecord record; private final ByteStringBuilder keyBuffer = new ByteStringBuilder(128); private RecordState recordState = RecordState.START; @@ -163,7 +164,7 @@ */ public Integer getIndexID() { if (indexID == null) if (record == null) { try { @@ -175,7 +176,7 @@ throw new RuntimeException(ex); } } return indexID; return record != null ? record.getIndexID() : null; } /** @@ -203,14 +204,20 @@ break; } indexID = getInt(); int indexID = getInt(); ByteString key = toKey(); record = ImportRecord.from(key, indexID); recordState = RecordState.NEED_INSERT_ID_SET; } private ByteString toKey() throws IOException { ensureData(20); int keyLen = getInt(); ensureData(keyLen); keyBuffer.clear().append(cache, keyLen); recordState = RecordState.NEED_INSERT_ID_SET; return keyBuffer.toByteString(); } private int getInt() throws IOException @@ -286,22 +293,6 @@ return false; } /** * Compares this buffer with the provided key and index ID. * * @param key * The key. * @param indexID * The index ID. * @return true if this buffer represent the same key and indexID, false otherwise. */ boolean sameKeyAndIndexID(final ByteStringBuilder key, Integer indexID) { ensureRecordFetched(); return Importer.indexComparator.compare(keyBuffer, key) == 0 && this.indexID.equals(indexID); } /** {@inheritDoc} */ @Override public int compareTo(IndexInputBuffer o) @@ -312,21 +303,20 @@ return 0; } ensureRecordFetched(); o.ensureRecordFetched(); int cmp = Importer.indexComparator.compare(keyBuffer, o.keyBuffer); int cmp = currentRecord().compareTo(o.currentRecord()); if (cmp == 0) { cmp = indexID.intValue() - o.getIndexID().intValue(); if (cmp == 0) { return bufferID - o.bufferID; } return bufferID - o.bufferID; } return cmp; } ImportRecord currentRecord() { ensureRecordFetched(); return record; } private void ensureRecordFetched() { if (keyBuffer.length() == 0) opendj-sdk/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/IndexOutputBuffer.java
@@ -26,14 +26,9 @@ */ package org.opends.server.backends.pluggable; import static org.opends.server.backends.pluggable.Importer.indexComparator; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; import org.forgerock.opendj.ldap.ByteSequence; import org.forgerock.opendj.ldap.ByteStringBuilder; /** * This class represents a index buffer used to store the keys and entry IDs @@ -107,21 +102,13 @@ */ private Importer.IndexKey indexKey; /** Initial capacity of re-usable buffer used in key compares. */ private static final int CAP = 32; /** * This buffer is reused during key compares. It's main purpose is to keep * memory footprint as small as possible. */ private ByteStringBuilder keyBuffer = new ByteStringBuilder(CAP); /** * Set to {@code true} if the buffer should not be recycled. Used when the * importer/rebuild index process is doing phase one cleanup and flushing * buffers not completed. */ private boolean discarded; private ImportRecord currentRecord; /** @@ -146,6 +133,7 @@ recordOffset = size - 1; keys = 0; position = 0; currentRecord = null; indexKey = null; } @@ -241,6 +229,7 @@ public void setPosition(int position) { this.position = position; this.currentRecord = toRecord(position); } /** @@ -338,132 +327,24 @@ return buffer[recOffset] == INS; } /** * Return the size of the key part of the record. * * @return The size of the key part of the record. */ public int getKeySize() { int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE; return readInt(buffer, offSet); } /** * Return the key value part of a record indicated by the current buffer * position. * * @return byte array containing the key value. */ public byte[] getKey() { return getKey(position); } /** Used to minimized memory usage when comparing keys. */ private ByteStringBuilder getKeyBuf(int position) { keyBuffer.clear(); int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE; int keyLen = readInt(buffer, offSet); offSet += INT_SIZE; keyBuffer.append(buffer, offSet, keyLen); return keyBuffer; } /** * Return the key value part of a record specified by the index. * * @param position position to return. * @return byte array containing the key value. */ private byte[] getKey(int position) { int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE; int keyLen = readInt(buffer, offSet); offSet += INT_SIZE; byte[] key = new byte[keyLen]; System.arraycopy(buffer, offSet, key, 0, keyLen); return key; } private int getOffset(int position) { return readInt(position * INT_SIZE); } /** * Return index id associated with the current position's record. * * @return The index id. */ public int getIndexID() { return getIndexID(position); } private int getIndexID(int position) { return getIndexIDFromOffset(getOffset(position)); } private int getIndexIDFromOffset(int offset) { return readInt(offset + 1); } private int compare(int xPosition, int yPosition) { int xoffSet = getOffset(xPosition); int xIndexID = getIndexIDFromOffset(xoffSet); xoffSet += REC_OVERHEAD + LONG_SIZE; int xKeyLen = readInt(buffer, xoffSet); int xKey = INT_SIZE + xoffSet; int yoffSet = getOffset(yPosition); int yIndexID = getIndexIDFromOffset(yoffSet); yoffSet += REC_OVERHEAD + LONG_SIZE; int yKeyLen = readInt(buffer, yoffSet); int yKey = INT_SIZE + yoffSet; return indexComparator.compare(buffer, xKey, xKeyLen, xIndexID, buffer, yKey, yKeyLen, yIndexID); return toRecord(xPosition).compareTo(toRecord(yPosition)); } private int compare(int xPosition, byte[] yKey, int yIndexID) private ImportRecord toRecord(int position) { int xoffSet = getOffset(xPosition); int xIndexID = getIndexIDFromOffset(xoffSet); xoffSet += REC_OVERHEAD + LONG_SIZE; int xKeyLen = readInt(buffer, xoffSet); int xKey = INT_SIZE + xoffSet; return indexComparator.compare(buffer, xKey, xKeyLen, xIndexID, yKey, 0, yKey.length, yIndexID); return ImportRecord.fromBufferAndPosition(buffer, position); } /** * Verifies whether the provided byte array and indexID are equal to * the byte array and indexIDs currently pointed to by this index output buffer. * * @param b The byte array to compare. * @param bIndexID The index key to compare. * @return <CODE>True</CODE> if the byte arrays are equal. */ public boolean sameKeyAndIndexID(byte[] b, int bIndexID) ImportRecord currentRecord() { if (b == null) { return false; } int offset = getOffset(position); int indexID = getIndexIDFromOffset(offset); offset += REC_OVERHEAD + LONG_SIZE; int keyLen = readInt(buffer, offset); int key = INT_SIZE + offset; return indexComparator.compare(buffer, key, keyLen, b, b.length) == 0 && indexID == bIndexID; return currentRecord; } /** @@ -479,22 +360,11 @@ @Override public int compareTo(IndexOutputBuffer b) { final ByteStringBuilder keyBuf = b.getKeyBuf(b.position); int offset = getOffset(position); int indexID = getIndexIDFromOffset(offset); offset += REC_OVERHEAD + LONG_SIZE; int keyLen = readInt(buffer, offset); int key = INT_SIZE + offset; int cmp = indexComparator.compare(buffer, key, keyLen, keyBuf.getBackingArray(), keyBuf.length()); int cmp = currentRecord().compareTo(b.currentRecord()); if (cmp == 0) { cmp = compareInts(indexID, b.getIndexID()); if (cmp == 0) { // This is tested in a tree set remove when a buffer is removed from the tree set. return compareLongs(bufferID, b.getBufferID()); } // This is tested in a tree set remove when a buffer is removed from the tree set. return compareLongs(bufferID, b.getBufferID()); } return cmp; } @@ -515,23 +385,6 @@ } } /** * Write a record to specified output stream using the record pointed to by * the current position and the specified byte stream of ids. * * @param dataStream The data output stream to write to. * * @throws IOException If an I/O error occurs writing the record. */ public void writeKey(DataOutputStream dataStream) throws IOException { int offSet = getOffset(position) + REC_OVERHEAD + LONG_SIZE; int keyLen = readInt(buffer, offSet); offSet += INT_SIZE; dataStream.write(buffer, offSet, keyLen); } /** * Compare the byte array at the current position with the byte array at the * provided position. @@ -541,7 +394,7 @@ */ public boolean sameKeyAndIndexID(int position) { return compare(position, this.position) == 0; return currentRecord().equals(toRecord(position)); } /** @@ -573,7 +426,7 @@ */ public void nextRecord() { position++; setPosition(position + 1); } private int writeInt(byte[] buffer, int offset, int val) @@ -598,11 +451,6 @@ private int readInt(int index) { return readInt(buffer, index); } private int readInt(byte[] buffer, int index) { int answer = 0; for (int i = 0; i < INT_SIZE; i++) { byte b = buffer[index + i]; @@ -614,9 +462,12 @@ private int med3(int a, int b, int c) { return compare(a,b) < 0 ? (compare(b,c) < 0 ? b : compare(a,c) < 0 ? c : a) : (compare(b,c) > 0 ? b : compare(a,c) > 0 ? c : a); ImportRecord pa = toRecord(a); ImportRecord pb = toRecord(b); ImportRecord pc = toRecord(c); return pa.compareTo(pb) < 0 ? (pb.compareTo(pc) < 0 ? b : pa.compareTo(pc) < 0 ? c : a) : (pb.compareTo(pc) > 0 ? b : pa.compareTo(pc) > 0 ? c : a); } private void sort(int off, int len) @@ -645,23 +496,20 @@ m = med3(l, m, n); } byte[] mKey = getKey(m); int mIndexID = getIndexID(m); int a = off, b = a, c = off + len - 1, d = c; while(true) { while (b <= c && compare(b, mKey, mIndexID) <= 0) while (b <= c && toRecord(b).compareTo(toRecord(m)) <= 0) { if (compare(b, mKey, mIndexID) == 0) if (toRecord(b).equals(toRecord(m))) { swap(a++, b); } b++; } while (c >= b && compare(c, mKey, mIndexID) >= 0) while (c >= b && toRecord(c).compareTo(toRecord(m)) >= 0) { if (compare(c, mKey, mIndexID) == 0) if (toRecord(c).equals(toRecord(m))) { swap(c, d--); } @@ -698,9 +546,9 @@ { int aOffset = a * INT_SIZE; int bOffset = b * INT_SIZE; int bVal = readInt(bOffset); int tmp = readInt(bOffset); System.arraycopy(buffer, aOffset, buffer, bOffset, INT_SIZE); writeInt(buffer, aOffset, bVal); writeInt(buffer, aOffset, tmp); } private void vectorSwap(int a, int b, int n) @@ -712,109 +560,6 @@ } /** * Used to compare keys when they are non-DN indexes. * <p> * The Comparator interface cannot be used in this class, so this * special one is used that knows about the special properties of this class. */ public static class IndexComparator { /** * Compare an offset in a byte array and indexID with the specified offset in the other byte array * and other indexID, using the DN compare algorithm. * * @param array1 The first byte array. * @param offset1 The first byte array's offset. * @param length1 The first byte array's length. * @param indexID1 The first index id. * @param array2 The second byte array to compare to. * @param offset1 The second byte array's offset. * @param length2 The second byte array's length. * @param indexID2 The second index id. * @return a negative integer, zero, or a positive integer as the first * offset value is less than, equal to, or greater than the second * byte array. */ private int compare(byte[] array1, int offset1, int length1, int indexID1, byte[] array2, int offset2, int length2, int indexID2) { int cmp = compareArrays(array1, offset1, length1, array2, offset2, length2); if (cmp == 0) { cmp = compareInts(length1, length2); if (cmp == 0) { return compareInts(indexID1, indexID2); } } return cmp; } int compare(ByteStringBuilder key1, ByteStringBuilder key2) { return compare(key1.getBackingArray(), 0, key1.length(), key2.getBackingArray(), key2.length()); } /** * Compare an offset in an byte array with the specified byte array, * using the DN compare algorithm. * * @param b The byte array. * @param offset The first offset. * @param length The first length. * @param other The second byte array to compare to. * @param otherLength The second byte array's length. * @return a negative integer, zero, or a positive integer as the first * offset value is less than, equal to, or greater than the second * byte array. */ int compare(byte[] b, int offset, int length, byte[] other, int otherLength) { int cmp = compareArrays(b, offset, length, other, 0, otherLength); if (cmp == 0) { return compareInts(length, otherLength); } return cmp; } private int compareArrays(byte[] array1, int offset1, int length1, byte[] array2, int offset2, int length2) { for (int i = 0; i < length1 && i < length2; i++) { byte b1 = array1[offset1 + i]; byte b2 = array2[offset2 + i]; if (b1 > b2) { return 1; } else if (b1 < b2) { return -1; } } return 0; } } private static int compareInts(int i1, int i2) { if (i1 == i2) { return 0; } else if (i1 > i2) { return 1; } else { return -1; } } /** * Set the index key associated with an index buffer. * * @param indexKey The index key.
