mirror of https://github.com/OpenIdentityPlatform/OpenDJ.git
opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java
@@ -56,6 +56,7 @@ import org.forgerock.opendj.server.config.server.JEBackendCfg; import org.opends.server.api.Backupable; import org.opends.server.api.DiskSpaceMonitorHandler; import org.opends.server.backends.pluggable.spi.EmptyCursor; import org.opends.server.backends.pluggable.spi.AccessMode; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.Importer; @@ -352,7 +353,7 @@ * <ol> * <li>Opening the EntryContainer calls {@link #openTree(TreeName, boolean)} for each index</li> * <li>Then the underlying storage is closed</li> * <li>Then {@link Importer#startImport()} is called</li> * <li>Then {@link #startImport()} is called</li> * <li>Then ID2Entry#put() is called</li> * <li>Which in turn calls ID2Entry#encodeEntry()</li> * <li>Which in turn finally calls PersistentCompressedSchema#store()</li> @@ -520,7 +521,7 @@ } } /** JE read-only implementation of {@link StorageImpl} interface. */ /** JE read-only implementation of {@link WriteableTransaction} interface. */ private final class ReadOnlyTransactionImpl implements WriteableTransaction { private final WriteableTransactionImpl delegate; @@ -583,8 +584,69 @@ } } /** No operation storage transaction faking database files are present and empty. */ private final class ReadOnlyEmptyTransactionImpl implements WriteableTransaction { @Override public void openTree(TreeName name, boolean createOnDemand) { if (createOnDemand) { throw new ReadOnlyStorageException(); } } @Override public void deleteTree(TreeName name) { throw new ReadOnlyStorageException(); } @Override public void put(TreeName treeName, ByteSequence key, ByteSequence value) { throw new ReadOnlyStorageException(); } @Override public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f) { throw new ReadOnlyStorageException(); } @Override public boolean delete(TreeName treeName, ByteSequence key) { throw new ReadOnlyStorageException(); } @Override public ByteString read(TreeName treeName, ByteSequence key) { return null; } @Override public Cursor<ByteString, ByteString> openCursor(TreeName treeName) { return new EmptyCursor<>(); } @Override public long getRecordCount(TreeName treeName) { return 0; } } private WriteableTransaction newWriteableTransaction(Transaction txn) { // If no database files have been created yet and we're opening READ-ONLY // there is no db to use, since open was not called. Fake it. if (env == null) { return new ReadOnlyEmptyTransactionImpl(); } final WriteableTransactionImpl writeableStorage = new WriteableTransactionImpl(txn); return accessMode.isWriteable() ? writeableStorage : new ReadOnlyTransactionImpl(writeableStorage); } @@ -601,6 +663,7 @@ private JEBackendCfg config; private AccessMode accessMode; /** It is NULL when opening the storage READ-ONLY and no files have been created yet. */ private Environment env; private EnvironmentConfig envConfig; private MemoryQuota memQuota; @@ -713,26 +776,58 @@ } } if (config.getDBCacheSize() > 0) if (memQuota != null) { memQuota.releaseMemory(config.getDBCacheSize()); } else { memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); if (config.getDBCacheSize() > 0) { memQuota.releaseMemory(config.getDBCacheSize()); } else { memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); } } config.removeJEChangeListener(this); diskMonitor.deregisterMonitoredDirectory(getDirectory(), this); envConfig = null; if (diskMonitor != null) { diskMonitor.deregisterMonitoredDirectory(getDirectory(), this); } } @Override public void open(AccessMode accessMode) throws ConfigException, StorageRuntimeException { Reject.ifNull(accessMode, "accessMode must not be null"); if (isBackendIncomplete(accessMode)) { envConfig = new EnvironmentConfig(); envConfig.setAllowCreate(false).setTransactional(false); // Do not open files on disk return; } buildConfiguration(accessMode, false); open0(); } private boolean isBackendIncomplete(AccessMode accessMode) { return !accessMode.isWriteable() && (!backendDirectory.exists() || backendDirectoryIncomplete()); } // TODO: it belongs to disk-based Storage Interface. private boolean backendDirectoryIncomplete() { try { return !getFilesToBackup().hasNext(); } catch (DirectoryException ignored) { return true; } } private void open0() throws ConfigException { setupStorageFiles(backendDirectory, config.getDBDirectoryPermissions(), config.dn()); @@ -1073,6 +1168,10 @@ @Override public Set<TreeName> listTrees() { if (env == null) { return Collections.<TreeName>emptySet(); } try { List<String> treeNames = env.getDatabaseNames(); opendj-server-legacy/src/main/java/org/opends/server/backends/pdb/PDBStorage.java
@@ -32,6 +32,7 @@ import java.nio.file.Paths; import java.rmi.RemoteException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -52,6 +53,7 @@ import org.forgerock.util.Reject; import org.opends.server.api.Backupable; import org.opends.server.api.DiskSpaceMonitorHandler; import org.opends.server.backends.pluggable.spi.EmptyCursor; import org.opends.server.backends.pluggable.spi.AccessMode; import org.opends.server.backends.pluggable.spi.Cursor; import org.opends.server.backends.pluggable.spi.Importer; @@ -388,6 +390,8 @@ /** Common interface for internal WriteableTransaction implementations. */ private interface StorageImpl extends WriteableTransaction, Closeable { <T>T read(ReadOperation<T> operation) throws Exception; void write(WriteOperation operation) throws Exception; } /** PersistIt implementation of the {@link WriteableTransaction} interface. */ @@ -589,6 +593,65 @@ } exchanges.clear(); } @Override public <T> T read(ReadOperation<T> operation) throws Exception { final Transaction txn = db.getTransaction(); for (;;) { txn.begin(); try { final T result = operation.run(this); txn.commit(commitPolicy); return result; } catch (final RollbackException e) { // retry } catch (final Exception e) { txn.rollback(); throw e; } finally { txn.end(); } } } @Override public void write(WriteOperation operation) throws Exception { final Transaction txn = db.getTransaction(); for (;;) { txn.begin(); try { operation.run(this); txn.commit(commitPolicy); return; } catch (final RollbackException e) { // retry after random sleep (reduces transactions collision. Drawback: increased latency) Thread.sleep((long) (Math.random() * MAX_SLEEP_ON_RETRY_MS)); } catch (final Exception e) { txn.rollback(); throw e; } finally { txn.end(); } } } } /** PersistIt read-only implementation of {@link StorageImpl} interface. */ @@ -673,6 +736,91 @@ { throw new ReadOnlyStorageException(); } @Override public <T> T read(ReadOperation<T> operation) throws Exception { return delegate.read(operation); } @Override public void write(WriteOperation operation) throws Exception { operation.run(this); } } /** No operation storage faking database files are present and empty. */ private final class ReadOnlyEmptyStorageImpl implements StorageImpl { @Override public void close() throws IOException { // Nothing to do } @Override public void openTree(TreeName name, boolean createOnDemand) { if (createOnDemand) { throw new ReadOnlyStorageException(); } } @Override public void deleteTree(TreeName name) { throw new ReadOnlyStorageException(); } @Override public void put(TreeName treeName, ByteSequence key, ByteSequence value) { throw new ReadOnlyStorageException(); } @Override public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f) { throw new ReadOnlyStorageException(); } @Override public boolean delete(TreeName treeName, ByteSequence key) { throw new ReadOnlyStorageException(); } @Override public ByteString read(TreeName treeName, ByteSequence key) { return null; } @Override public Cursor<ByteString, ByteString> openCursor(TreeName treeName) { return new EmptyCursor<>(); } @Override public long getRecordCount(TreeName treeName) { return 0; } @Override public <T> T read(ReadOperation<T> operation) throws Exception { return operation.run(this); } @Override public void write(WriteOperation operation) throws Exception { operation.run(this); } } Exchange getNewExchange(final TreeName treeName, final boolean create) throws PersistitException @@ -693,6 +841,12 @@ } private StorageImpl newStorageImpl() { // If no persistent files have been created yet and we're opening READ-ONLY // there is no volume and no db to use, since open was not called. Fake it. if (db == null) { return new ReadOnlyEmptyStorageImpl(); } final WriteableStorageImpl writeableStorage = new WriteableStorageImpl(); return accessMode.isWriteable() ? writeableStorage : new ReadOnlyStorageImpl(writeableStorage); } @@ -703,7 +857,9 @@ private final File backendDirectory; private CommitPolicy commitPolicy; private AccessMode accessMode; /** It is NULL when opening the storage READ-ONLY and no files have been created yet. */ private Persistit db; /** It is NULL when opening the storage READ-ONLY and no files have been created yet, same as {@link #db}. */ private Volume volume; private PDBBackendCfg config; private DiskSpaceMonitor diskMonitor; @@ -785,16 +941,22 @@ throw new IllegalStateException(e); } } if (config.getDBCacheSize() > 0) if (memQuota != null) { memQuota.releaseMemory(config.getDBCacheSize()); } else { memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); if (config.getDBCacheSize() > 0) { memQuota.releaseMemory(config.getDBCacheSize()); } else { memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); } } config.removePDBChangeListener(this); diskMonitor.deregisterMonitoredDirectory(getDirectory(), this); if (diskMonitor != null) { diskMonitor.deregisterMonitoredDirectory(getDirectory(), this); } } private static BufferPoolConfiguration getBufferPoolCfg(Configuration dbCfg) @@ -806,9 +968,32 @@ public void open(AccessMode accessMode) throws ConfigException, StorageRuntimeException { Reject.ifNull(accessMode, "accessMode must not be null"); if (isBackendIncomplete(accessMode)) { // Do not open volume on disk return; } open0(buildConfiguration(accessMode)); } private boolean isBackendIncomplete(AccessMode accessMode) { return !accessMode.isWriteable() && (!backendDirectory.exists() || backendDirectoryIncomplete()); } // TODO: it belongs to disk-based Storage Interface. private boolean backendDirectoryIncomplete() { try { return !getFilesToBackup().hasNext(); } catch (DirectoryException ignored) { return true; } } private void open0(final Configuration dbCfg) throws ConfigException { setupStorageFiles(backendDirectory, config.getDBDirectoryPermissions(), config.dn()); @@ -843,42 +1028,18 @@ @Override public <T> T read(final ReadOperation<T> operation) throws Exception { // This check may be unnecessary for PDB, but it will help us detect bad business logic // in the pluggable backend that would cause problems for JE. final Transaction txn = db.getTransaction(); for (;;) try (final StorageImpl storageImpl = newStorageImpl()) { txn.begin(); try final T result = storageImpl.read(operation); return result; } catch (final StorageRuntimeException e) { if (e.getCause() != null) { try (final StorageImpl storageImpl = newStorageImpl()) { final T result = operation.run(storageImpl); txn.commit(commitPolicy); return result; } catch (final StorageRuntimeException e) { if (e.getCause() != null) { throw (Exception) e.getCause(); } throw e; } throw (Exception) e.getCause(); } catch (final RollbackException e) { // retry } catch (final Exception e) { txn.rollback(); throw e; } finally { txn.end(); } throw e; } } @@ -892,41 +1053,17 @@ @Override public void write(final WriteOperation operation) throws Exception { final Transaction txn = db.getTransaction(); for (;;) try (final StorageImpl storageImpl = newStorageImpl()) { txn.begin(); try storageImpl.write(operation); } catch (final StorageRuntimeException e) { if (e.getCause() != null) { try (final StorageImpl storageImpl = newStorageImpl()) { operation.run(storageImpl); txn.commit(commitPolicy); return; } catch (final StorageRuntimeException e) { if (e.getCause() != null) { throw (Exception) e.getCause(); } throw e; } throw (Exception) e.getCause(); } catch (final RollbackException e) { // retry after random sleep (reduces transactions collision. Drawback: increased latency) Thread.sleep((long) (Math.random() * MAX_SLEEP_ON_RETRY_MS)); } catch (final Exception e) { txn.rollback(); throw e; } finally { txn.end(); } throw e; } } @@ -1101,6 +1238,10 @@ @Override public Set<TreeName> listTrees() { if (volume == null) { return Collections.<TreeName>emptySet(); } try { String[] treeNames = volume.getTreeNames(); opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/DefaultIndex.java
@@ -102,7 +102,7 @@ codec = new EntryIDSet.EntryIDSetCodecV3(codec, cryptoSuite); } trusted = flags.contains(TRUSTED); if (!trusted && entryContainer.getHighestEntryID(txn).longValue() == 0) if (createOnDemand && !trusted && entryContainer.isEmpty(txn)) { // If there are no entries in the entry container then there // is no reason why this index can't be upgraded to trusted. opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
@@ -421,6 +421,7 @@ state.open(txn, shouldCreate); dn2uri.open(txn, shouldCreate); final boolean isNotEmpty = !isEmpty(txn); for (String idx : config.listBackendIndexes()) { BackendIndexCfg indexCfg = config.getBackendIndex(idx); @@ -428,7 +429,7 @@ CryptoSuite cryptoSuite = newCryptoSuite(indexCfg.isConfidentialityEnabled()); final AttributeIndex index = newAttributeIndex(indexCfg, cryptoSuite); index.open(txn, shouldCreate); if(!index.isTrusted()) if(!index.isTrusted() && isNotEmpty) { logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, index.getName()); } @@ -442,7 +443,7 @@ VLVIndex vlvIndex = new VLVIndex(vlvIndexCfg, state, storage, this, txn); vlvIndex.open(txn, shouldCreate); if(!vlvIndex.isTrusted()) if(!vlvIndex.isTrusted() && isNotEmpty) { logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, vlvIndex.getName()); } @@ -458,6 +459,11 @@ } } boolean isEmpty(ReadableTransaction txn) { return getHighestEntryID(txn).longValue() == 0; } /** * Closes the entry container. * opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/spi/EmptyCursor.java
New file @@ -0,0 +1,98 @@ /* * The contents of this file are subject to the terms of the Common Development and * Distribution License (the License). You may not use this file except in compliance with the * License. * * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the * specific language governing permission and limitations under the License. * * When distributing Covered Software, include this CDDL Header Notice in each file and include * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL * Header, with the fields enclosed by brackets [] replaced by your own identifying * information: "Portions Copyright [year] [name of copyright owner]". * * Copyright 2016 ForgeRock AS. */ package org.opends.server.backends.pluggable.spi; import org.forgerock.opendj.ldap.ByteSequence; import java.util.NoSuchElementException; /** * Implementation of an empty {@link Cursor}, for simulating no records to cursor on. * <p> * Cursor behaves as follows: * <ul> * <li>Positioning to a key or index will fail, returning {@code false}.</li> * <li>Reading the key or value will return {@code null}.</li> * <li>Deleting the current element is not supported, {@link UnsupportedOperationException} will be thrown.</li> * </ul> * </p> * * @param <K> Type of the simulated record's key * @param <V> Type of the simulated record's value */ public final class EmptyCursor<K, V> implements Cursor<K, V> { @Override public boolean positionToKey(ByteSequence key) { return false; } @Override public boolean positionToKeyOrNext(ByteSequence key) { return false; } @Override public boolean positionToLastKey() { return false; } @Override public boolean positionToIndex(int index) { return false; } @Override public boolean next() { return false; } @Override public boolean isDefined() { return false; } @Override public K getKey() throws NoSuchElementException { return null; } @Override public V getValue() throws NoSuchElementException { return null; } @Override public void delete() throws NoSuchElementException, UnsupportedOperationException { throw new UnsupportedOperationException(); } @Override public void close() { // Nothing to do } }
