From 74fea9c73aa679eebe68f78d34ae80fa0f263c24 Mon Sep 17 00:00:00 2001
From: Fabio Pistolesi <fabio.pistolesi@forgerock.com>
Date: Fri, 21 Oct 2016 09:41:24 +0000
Subject: [PATCH] OPENDJ-3400 Fake a no-operation storage for offline commands when a suffix has no files
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/DefaultIndex.java | 2
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java | 10
opendj-server-legacy/src/main/java/org/opends/server/backends/pdb/PDBStorage.java | 285 +++++++++++++++++++++++--------
opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java | 117 ++++++++++++-
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/spi/EmptyCursor.java | 98 ++++++++++
5 files changed, 428 insertions(+), 84 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java
index e6115f3..210a267 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/jeb/JEStorage.java
@@ -56,6 +56,7 @@
import org.forgerock.opendj.server.config.server.JEBackendCfg;
import org.opends.server.api.Backupable;
import org.opends.server.api.DiskSpaceMonitorHandler;
+import org.opends.server.backends.pluggable.spi.EmptyCursor;
import org.opends.server.backends.pluggable.spi.AccessMode;
import org.opends.server.backends.pluggable.spi.Cursor;
import org.opends.server.backends.pluggable.spi.Importer;
@@ -352,7 +353,7 @@
* <ol>
* <li>Opening the EntryContainer calls {@link #openTree(TreeName, boolean)} for each index</li>
* <li>Then the underlying storage is closed</li>
- * <li>Then {@link Importer#startImport()} is called</li>
+ * <li>Then {@link #startImport()} is called</li>
* <li>Then ID2Entry#put() is called</li>
* <li>Which in turn calls ID2Entry#encodeEntry()</li>
* <li>Which in turn finally calls PersistentCompressedSchema#store()</li>
@@ -520,7 +521,7 @@
}
}
- /** JE read-only implementation of {@link StorageImpl} interface. */
+ /** JE read-only implementation of {@link WriteableTransaction} interface. */
private final class ReadOnlyTransactionImpl implements WriteableTransaction
{
private final WriteableTransactionImpl delegate;
@@ -583,8 +584,69 @@
}
}
+ /** No operation storage transaction faking database files are present and empty. */
+ private final class ReadOnlyEmptyTransactionImpl implements WriteableTransaction
+ {
+ @Override
+ public void openTree(TreeName name, boolean createOnDemand)
+ {
+ if (createOnDemand)
+ {
+ throw new ReadOnlyStorageException();
+ }
+ }
+
+ @Override
+ public void deleteTree(TreeName name)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public void put(TreeName treeName, ByteSequence key, ByteSequence value)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public boolean delete(TreeName treeName, ByteSequence key)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public ByteString read(TreeName treeName, ByteSequence key)
+ {
+ return null;
+ }
+
+ @Override
+ public Cursor<ByteString, ByteString> openCursor(TreeName treeName)
+ {
+ return new EmptyCursor<>();
+ }
+
+ @Override
+ public long getRecordCount(TreeName treeName)
+ {
+ return 0;
+ }
+ }
+
private WriteableTransaction newWriteableTransaction(Transaction txn)
{
+ // If no database files have been created yet and we're opening READ-ONLY
+ // there is no db to use, since open was not called. Fake it.
+ if (env == null)
+ {
+ return new ReadOnlyEmptyTransactionImpl();
+ }
final WriteableTransactionImpl writeableStorage = new WriteableTransactionImpl(txn);
return accessMode.isWriteable() ? writeableStorage : new ReadOnlyTransactionImpl(writeableStorage);
}
@@ -601,6 +663,7 @@
private JEBackendCfg config;
private AccessMode accessMode;
+ /** It is NULL when opening the storage READ-ONLY and no files have been created yet. */
private Environment env;
private EnvironmentConfig envConfig;
private MemoryQuota memQuota;
@@ -713,26 +776,58 @@
}
}
- if (config.getDBCacheSize() > 0)
+ if (memQuota != null)
{
- memQuota.releaseMemory(config.getDBCacheSize());
- }
- else
- {
- memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent()));
+ if (config.getDBCacheSize() > 0)
+ {
+ memQuota.releaseMemory(config.getDBCacheSize());
+ }
+ else
+ {
+ memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent()));
+ }
}
config.removeJEChangeListener(this);
- diskMonitor.deregisterMonitoredDirectory(getDirectory(), this);
+ envConfig = null;
+ if (diskMonitor != null)
+ {
+ diskMonitor.deregisterMonitoredDirectory(getDirectory(), this);
+ }
}
@Override
public void open(AccessMode accessMode) throws ConfigException, StorageRuntimeException
{
Reject.ifNull(accessMode, "accessMode must not be null");
+ if (isBackendIncomplete(accessMode))
+ {
+ envConfig = new EnvironmentConfig();
+ envConfig.setAllowCreate(false).setTransactional(false);
+ // Do not open files on disk
+ return;
+ }
buildConfiguration(accessMode, false);
open0();
}
+ private boolean isBackendIncomplete(AccessMode accessMode)
+ {
+ return !accessMode.isWriteable() && (!backendDirectory.exists() || backendDirectoryIncomplete());
+ }
+
+ // TODO: it belongs to disk-based Storage Interface.
+ private boolean backendDirectoryIncomplete()
+ {
+ try
+ {
+ return !getFilesToBackup().hasNext();
+ }
+ catch (DirectoryException ignored)
+ {
+ return true;
+ }
+ }
+
private void open0() throws ConfigException
{
setupStorageFiles(backendDirectory, config.getDBDirectoryPermissions(), config.dn());
@@ -1073,6 +1168,10 @@
@Override
public Set<TreeName> listTrees()
{
+ if (env == null)
+ {
+ return Collections.<TreeName>emptySet();
+ }
try
{
List<String> treeNames = env.getDatabaseNames();
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pdb/PDBStorage.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pdb/PDBStorage.java
index f51a980..eaffd4d 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pdb/PDBStorage.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pdb/PDBStorage.java
@@ -32,6 +32,7 @@
import java.nio.file.Paths;
import java.rmi.RemoteException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -52,6 +53,7 @@
import org.forgerock.util.Reject;
import org.opends.server.api.Backupable;
import org.opends.server.api.DiskSpaceMonitorHandler;
+import org.opends.server.backends.pluggable.spi.EmptyCursor;
import org.opends.server.backends.pluggable.spi.AccessMode;
import org.opends.server.backends.pluggable.spi.Cursor;
import org.opends.server.backends.pluggable.spi.Importer;
@@ -388,6 +390,8 @@
/** Common interface for internal WriteableTransaction implementations. */
private interface StorageImpl extends WriteableTransaction, Closeable {
+ <T>T read(ReadOperation<T> operation) throws Exception;
+ void write(WriteOperation operation) throws Exception;
}
/** PersistIt implementation of the {@link WriteableTransaction} interface. */
@@ -589,6 +593,65 @@
}
exchanges.clear();
}
+
+ @Override
+ public <T> T read(ReadOperation<T> operation) throws Exception
+ {
+ final Transaction txn = db.getTransaction();
+ for (;;)
+ {
+ txn.begin();
+ try
+ {
+ final T result = operation.run(this);
+ txn.commit(commitPolicy);
+ return result;
+ }
+ catch (final RollbackException e)
+ {
+ // retry
+ }
+ catch (final Exception e)
+ {
+ txn.rollback();
+ throw e;
+ }
+ finally
+ {
+ txn.end();
+ }
+ }
+ }
+
+ @Override
+ public void write(WriteOperation operation) throws Exception
+ {
+ final Transaction txn = db.getTransaction();
+ for (;;)
+ {
+ txn.begin();
+ try
+ {
+ operation.run(this);
+ txn.commit(commitPolicy);
+ return;
+ }
+ catch (final RollbackException e)
+ {
+ // retry after random sleep (reduces transactions collision. Drawback: increased latency)
+ Thread.sleep((long) (Math.random() * MAX_SLEEP_ON_RETRY_MS));
+ }
+ catch (final Exception e)
+ {
+ txn.rollback();
+ throw e;
+ }
+ finally
+ {
+ txn.end();
+ }
+ }
+ }
}
/** PersistIt read-only implementation of {@link StorageImpl} interface. */
@@ -673,6 +736,91 @@
{
throw new ReadOnlyStorageException();
}
+
+ @Override
+ public <T> T read(ReadOperation<T> operation) throws Exception
+ {
+ return delegate.read(operation);
+ }
+
+ @Override
+ public void write(WriteOperation operation) throws Exception
+ {
+ operation.run(this);
+ }
+ }
+
+ /** No operation storage faking database files are present and empty. */
+ private final class ReadOnlyEmptyStorageImpl implements StorageImpl
+ {
+ @Override
+ public void close() throws IOException
+ {
+ // Nothing to do
+ }
+
+ @Override
+ public void openTree(TreeName name, boolean createOnDemand)
+ {
+ if (createOnDemand)
+ {
+ throw new ReadOnlyStorageException();
+ }
+ }
+
+ @Override
+ public void deleteTree(TreeName name)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public void put(TreeName treeName, ByteSequence key, ByteSequence value)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public boolean delete(TreeName treeName, ByteSequence key)
+ {
+ throw new ReadOnlyStorageException();
+ }
+
+ @Override
+ public ByteString read(TreeName treeName, ByteSequence key)
+ {
+ return null;
+ }
+
+ @Override
+ public Cursor<ByteString, ByteString> openCursor(TreeName treeName)
+ {
+ return new EmptyCursor<>();
+ }
+
+ @Override
+ public long getRecordCount(TreeName treeName)
+ {
+ return 0;
+ }
+
+ @Override
+ public <T> T read(ReadOperation<T> operation) throws Exception
+ {
+ return operation.run(this);
+ }
+
+ @Override
+ public void write(WriteOperation operation) throws Exception
+ {
+ operation.run(this);
+ }
}
Exchange getNewExchange(final TreeName treeName, final boolean create) throws PersistitException
@@ -693,6 +841,12 @@
}
private StorageImpl newStorageImpl() {
+ // If no persistent files have been created yet and we're opening READ-ONLY
+ // there is no volume and no db to use, since open was not called. Fake it.
+ if (db == null)
+ {
+ return new ReadOnlyEmptyStorageImpl();
+ }
final WriteableStorageImpl writeableStorage = new WriteableStorageImpl();
return accessMode.isWriteable() ? writeableStorage : new ReadOnlyStorageImpl(writeableStorage);
}
@@ -703,7 +857,9 @@
private final File backendDirectory;
private CommitPolicy commitPolicy;
private AccessMode accessMode;
+ /** It is NULL when opening the storage READ-ONLY and no files have been created yet. */
private Persistit db;
+ /** It is NULL when opening the storage READ-ONLY and no files have been created yet, same as {@link #db}. */
private Volume volume;
private PDBBackendCfg config;
private DiskSpaceMonitor diskMonitor;
@@ -785,16 +941,22 @@
throw new IllegalStateException(e);
}
}
- if (config.getDBCacheSize() > 0)
+ if (memQuota != null)
{
- memQuota.releaseMemory(config.getDBCacheSize());
- }
- else
- {
- memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent()));
+ if (config.getDBCacheSize() > 0)
+ {
+ memQuota.releaseMemory(config.getDBCacheSize());
+ }
+ else
+ {
+ memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent()));
+ }
}
config.removePDBChangeListener(this);
- diskMonitor.deregisterMonitoredDirectory(getDirectory(), this);
+ if (diskMonitor != null)
+ {
+ diskMonitor.deregisterMonitoredDirectory(getDirectory(), this);
+ }
}
private static BufferPoolConfiguration getBufferPoolCfg(Configuration dbCfg)
@@ -806,9 +968,32 @@
public void open(AccessMode accessMode) throws ConfigException, StorageRuntimeException
{
Reject.ifNull(accessMode, "accessMode must not be null");
+ if (isBackendIncomplete(accessMode))
+ {
+ // Do not open volume on disk
+ return;
+ }
open0(buildConfiguration(accessMode));
}
+ private boolean isBackendIncomplete(AccessMode accessMode)
+ {
+ return !accessMode.isWriteable() && (!backendDirectory.exists() || backendDirectoryIncomplete());
+ }
+
+ // TODO: it belongs to disk-based Storage Interface.
+ private boolean backendDirectoryIncomplete()
+ {
+ try
+ {
+ return !getFilesToBackup().hasNext();
+ }
+ catch (DirectoryException ignored)
+ {
+ return true;
+ }
+ }
+
private void open0(final Configuration dbCfg) throws ConfigException
{
setupStorageFiles(backendDirectory, config.getDBDirectoryPermissions(), config.dn());
@@ -843,42 +1028,18 @@
@Override
public <T> T read(final ReadOperation<T> operation) throws Exception
{
- // This check may be unnecessary for PDB, but it will help us detect bad business logic
- // in the pluggable backend that would cause problems for JE.
- final Transaction txn = db.getTransaction();
- for (;;)
+ try (final StorageImpl storageImpl = newStorageImpl())
{
- txn.begin();
- try
+ final T result = storageImpl.read(operation);
+ return result;
+ }
+ catch (final StorageRuntimeException e)
+ {
+ if (e.getCause() != null)
{
- try (final StorageImpl storageImpl = newStorageImpl())
- {
- final T result = operation.run(storageImpl);
- txn.commit(commitPolicy);
- return result;
- }
- catch (final StorageRuntimeException e)
- {
- if (e.getCause() != null)
- {
- throw (Exception) e.getCause();
- }
- throw e;
- }
+ throw (Exception) e.getCause();
}
- catch (final RollbackException e)
- {
- // retry
- }
- catch (final Exception e)
- {
- txn.rollback();
- throw e;
- }
- finally
- {
- txn.end();
- }
+ throw e;
}
}
@@ -892,41 +1053,17 @@
@Override
public void write(final WriteOperation operation) throws Exception
{
- final Transaction txn = db.getTransaction();
- for (;;)
+ try (final StorageImpl storageImpl = newStorageImpl())
{
- txn.begin();
- try
+ storageImpl.write(operation);
+ }
+ catch (final StorageRuntimeException e)
+ {
+ if (e.getCause() != null)
{
- try (final StorageImpl storageImpl = newStorageImpl())
- {
- operation.run(storageImpl);
- txn.commit(commitPolicy);
- return;
- }
- catch (final StorageRuntimeException e)
- {
- if (e.getCause() != null)
- {
- throw (Exception) e.getCause();
- }
- throw e;
- }
+ throw (Exception) e.getCause();
}
- catch (final RollbackException e)
- {
- // retry after random sleep (reduces transactions collision. Drawback: increased latency)
- Thread.sleep((long) (Math.random() * MAX_SLEEP_ON_RETRY_MS));
- }
- catch (final Exception e)
- {
- txn.rollback();
- throw e;
- }
- finally
- {
- txn.end();
- }
+ throw e;
}
}
@@ -1101,6 +1238,10 @@
@Override
public Set<TreeName> listTrees()
{
+ if (volume == null)
+ {
+ return Collections.<TreeName>emptySet();
+ }
try
{
String[] treeNames = volume.getTreeNames();
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/DefaultIndex.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/DefaultIndex.java
index bd6e01c..99faae0 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/DefaultIndex.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/DefaultIndex.java
@@ -102,7 +102,7 @@
codec = new EntryIDSet.EntryIDSetCodecV3(codec, cryptoSuite);
}
trusted = flags.contains(TRUSTED);
- if (!trusted && entryContainer.getHighestEntryID(txn).longValue() == 0)
+ if (createOnDemand && !trusted && entryContainer.isEmpty(txn))
{
// If there are no entries in the entry container then there
// is no reason why this index can't be upgraded to trusted.
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
index 82f78e8..9c10a92 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/EntryContainer.java
@@ -421,6 +421,7 @@
state.open(txn, shouldCreate);
dn2uri.open(txn, shouldCreate);
+ final boolean isNotEmpty = !isEmpty(txn);
for (String idx : config.listBackendIndexes())
{
BackendIndexCfg indexCfg = config.getBackendIndex(idx);
@@ -428,7 +429,7 @@
CryptoSuite cryptoSuite = newCryptoSuite(indexCfg.isConfidentialityEnabled());
final AttributeIndex index = newAttributeIndex(indexCfg, cryptoSuite);
index.open(txn, shouldCreate);
- if(!index.isTrusted())
+ if(!index.isTrusted() && isNotEmpty)
{
logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, index.getName());
}
@@ -442,7 +443,7 @@
VLVIndex vlvIndex = new VLVIndex(vlvIndexCfg, state, storage, this, txn);
vlvIndex.open(txn, shouldCreate);
- if(!vlvIndex.isTrusted())
+ if(!vlvIndex.isTrusted() && isNotEmpty)
{
logger.info(NOTE_INDEX_ADD_REQUIRES_REBUILD, vlvIndex.getName());
}
@@ -458,6 +459,11 @@
}
}
+ boolean isEmpty(ReadableTransaction txn)
+ {
+ return getHighestEntryID(txn).longValue() == 0;
+ }
+
/**
* Closes the entry container.
*
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/spi/EmptyCursor.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/spi/EmptyCursor.java
new file mode 100644
index 0000000..d22f3ff
--- /dev/null
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/spi/EmptyCursor.java
@@ -0,0 +1,98 @@
+/*
+ * The contents of this file are subject to the terms of the Common Development and
+ * Distribution License (the License). You may not use this file except in compliance with the
+ * License.
+ *
+ * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
+ * specific language governing permission and limitations under the License.
+ *
+ * When distributing Covered Software, include this CDDL Header Notice in each file and include
+ * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
+ * Header, with the fields enclosed by brackets [] replaced by your own identifying
+ * information: "Portions Copyright [year] [name of copyright owner]".
+ *
+ * Copyright 2016 ForgeRock AS.
+ */
+
+package org.opends.server.backends.pluggable.spi;
+
+import org.forgerock.opendj.ldap.ByteSequence;
+
+import java.util.NoSuchElementException;
+
+/**
+ * Implementation of an empty {@link Cursor}, for simulating no records to cursor on.
+ * <p>
+ * Cursor behaves as follows:
+ * <ul>
+ * <li>Positioning to a key or index will fail, returning {@code false}.</li>
+ * <li>Reading the key or value will return {@code null}.</li>
+ * <li>Deleting the current element is not supported, {@link UnsupportedOperationException} will be thrown.</li>
+ * </ul>
+ * </p>
+ *
+ * @param <K> Type of the simulated record's key
+ * @param <V> Type of the simulated record's value
+ */
+public final class EmptyCursor<K, V> implements Cursor<K, V>
+{
+ @Override
+ public boolean positionToKey(ByteSequence key)
+ {
+ return false;
+ }
+
+ @Override
+ public boolean positionToKeyOrNext(ByteSequence key)
+ {
+ return false;
+ }
+
+ @Override
+ public boolean positionToLastKey()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean positionToIndex(int index)
+ {
+ return false;
+ }
+
+ @Override
+ public boolean next()
+ {
+ return false;
+ }
+
+ @Override
+ public boolean isDefined()
+ {
+ return false;
+ }
+
+ @Override
+ public K getKey() throws NoSuchElementException
+ {
+ return null;
+ }
+
+ @Override
+ public V getValue() throws NoSuchElementException
+ {
+ return null;
+ }
+
+ @Override
+ public void delete() throws NoSuchElementException, UnsupportedOperationException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close()
+ {
+ // Nothing to do
+ }
+}
--
Gitblit v1.10.0