From 7eda83737e5c2a09bef758ac2bcd3b7ea8b32ce3 Mon Sep 17 00:00:00 2001
From: boli <boli@localhost>
Date: Wed, 20 Jun 2007 18:27:41 +0000
Subject: [PATCH] This refactoring includes the following changes to the JE backend: - Extracted common interface DatabaseContainer from DN2ID, ID2Entry, etc... classes. - Moved database read and write methods from EntryContainer to DatabaseContainer. - Added index configuration to the XML based admin framework. - Removed redundant configuration objects (Config, IndexConfig). - Added exclusive/shared lock to EntryContainer. All access to an EntryContainer must acquire a lock before using the internal DatabaseContainers or making configuration changes. - Added the ability to add/remove/modify indexes with the backend online. Server will issue rebuild required warning when adding new indexes or sub-indexes (equality, substring, presence...). - Added the ability to change the index entry limit for both the backend and each index with the backend online. Server will issue rebuild required warning if the previous limit has been exceeded. - Added the ability to change entry compression and index substring length setting while the backend is online. - Added a persistent state database to each EntryContainer to persist backend configuration between server restarts. Server will issue rebuild required warning if a new index is added when the backend is offline. - Added a trusted flag to indexes so that non existent keys will not be interpreted as an empty entry ID set when an index is untrusted. An index is untrusted when it is added to an non-empty EntryContainer or an inconsistency is detected. Server will issue warning on startup to rebuild the index. - Fixed a issue where the LDIF import process stops responding if the temporary import dir is full or unwritable.
---
opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestImportJob.java | 298 +++++++++++++++++++++++++++++++---------------------------
1 files changed, 159 insertions(+), 139 deletions(-)
diff --git a/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestImportJob.java b/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestImportJob.java
index 390899c..cda9633 100644
--- a/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestImportJob.java
+++ b/opends/tests/unit-tests-testng/src/server/org/opends/server/backends/jeb/TestImportJob.java
@@ -27,6 +27,7 @@
package org.opends.server.backends.jeb;
import org.opends.server.TestCaseUtils;
+import org.opends.server.tasks.TaskUtils;
import org.opends.server.admin.std.server.JEBackendCfg;
import org.opends.server.admin.std.meta.JEBackendCfgDefn;
import org.opends.server.admin.server.AdminTestCaseUtils;
@@ -51,33 +52,31 @@
public class TestImportJob extends JebTestCase
{
+ private String beID="importRoot";
private File tempDir;
private String homeDirName;
- private String importDirName;
-
- private LDIFImportConfig ldifImportConfig;
- private Config backendConfig;
private DN[] baseDNs;
+ private BackendImpl be;
private String errorCount="verify-error-count";
- private String top = "dn: dc=example,dc=com\n" +
+ private String top = "dn: dc=importtest,dc=com\n" +
"objectclass: top\n" +
"objectclass: domain\n" +
"dc: example\n" +
"\n" +
- "dn: ou=People,dc=example,dc=com\n" +
+ "dn: ou=People,dc=importtest,dc=com\n" +
"objectclass: top\n" +
"objectclass: organizationalUnit\n" +
"ou: People\n" +
"\n" +
- "dn: dc=example1,dc=com\n" +
+ "dn: dc=importtest1,dc=com\n" +
"objectclass: top\n" +
"objectclass: domain\n" +
"dc: example1\n";
private String entries1 =
- "dn: uid=user.0,ou=People,dc=example,dc=com\n" +
+ "dn: uid=user.0,ou=People,dc=importtest,dc=com\n" +
"objectClass: top\n" +
"objectClass: person\n" +
"objectClass: organizationalPerson\n" +
@@ -101,7 +100,7 @@
"postalAddress: Aaccf Amar$99262 Eleventh Street$Salem, NM 36530\n" +
"description: This is the description for Aaccf Amar.\n" +
"\n" +
- "dn: uid=user.539,ou=People,dc=example,dc=com\n" +
+ "dn: uid=user.539,ou=People,dc=importtest,dc=com\n" +
"objectClass: top\n" +
"objectClass: person\n" +
"objectClass: organizationalPerson\n" +
@@ -125,7 +124,7 @@
"postalAddress: Ardyth Bainton$81170 Taylor Street$Syracuse, WV 93507\n" +
"description: This is the description for Ardyth Bainton.\n" +
"\n" +
- "dn: uid=user.446,dc=example1,dc=com\n" +
+ "dn: uid=user.446,dc=importtest1,dc=com\n" +
"objectClass: top\n" +
"objectClass: person\n" +
"objectClass: organizationalPerson\n" +
@@ -149,7 +148,7 @@
"postalAddress: Annalee Avard$46168 Mill Street$Charleston, CO 60948\n" +
"description: This is the description for Annalee Avard.\n" +
"\n" +
- "dn: uid=user.362,dc=example1,dc=com\n" +
+ "dn: uid=user.362,dc=importtest1,dc=com\n" +
"objectClass: top\n" +
"objectClass: person\n" +
"objectClass: organizationalPerson\n" +
@@ -174,7 +173,7 @@
"description: This is the description for Andaree Asawa.\n";
private String replacement1 =
- "dn: uid=user.446,dc=example1,dc=com\n" +
+ "dn: uid=user.446,dc=importtest1,dc=com\n" +
"objectClass: top\n" +
"objectClass: person\n" +
"objectClass: organizationalPerson\n" +
@@ -208,7 +207,6 @@
tempDir = TestCaseUtils.createTemporaryDirectory("jebimporttest");
homeDirName = tempDir.getAbsolutePath();
- importDirName = homeDirName + File.separator + "importTmp";
EnvManager.createHomeDir(homeDirName);
@@ -233,56 +231,13 @@
writer.close();
ldifFile.close();
- Entry configEntry = TestCaseUtils.makeEntry(
- "dn: ds-cfg-backend-id=userRoot,cn=Backends,cn=config",
- "objectClass: top",
- "objectClass: ds-cfg-backend",
- "objectClass: ds-cfg-je-backend",
- "ds-cfg-backend-base-dn: dc=example,dc=com",
- "ds-cfg-backend-base-dn: dc=example1,dc=com",
- "ds-cfg-backend-writability-mode: enabled",
- "ds-cfg-backend-enabled: true",
- "ds-cfg-backend-class: org.opends.server.backends.jeb.BackendImpl",
- "ds-cfg-backend-id: userRoot",
- "ds-cfg-backend-directory:: " + Base64.encode(homeDirName.getBytes()),
- "ds-cfg-backend-import-temp-directory:: " + Base64.encode(importDirName.getBytes()));
-
- ConfigEntry backendConfigEntry = new ConfigEntry(configEntry, null);
- JEBackendCfg cfg = AdminTestCaseUtils.getConfiguration(
- JEBackendCfgDefn.getInstance(), configEntry);
-
- Entry indexEntry = TestCaseUtils.makeEntry(
- "dn: cn=Index,ds-cfg-backend-id=userRoot,cn=Backends,cn=config\n" +
- "objectClass: top\n" +
- "objectClass: ds-cfg-branch\n" +
- "cn: Index");
-
- ConfigEntry indexConfigEntry = new ConfigEntry(indexEntry, backendConfigEntry);
-
- Entry cnIndexEntry = TestCaseUtils.makeEntry(
- "dn: ds-cfg-index-attribute=cn,cn=Index,ds-cfg-backend-id=userRoot,cn=Backends,cn=config\n" +
- "objectClass: top\n" +
- "objectClass: ds-cfg-je-index\n" +
- "ds-cfg-index-attribute: cn\n" +
- "ds-cfg-index-type: presence\n" +
- "ds-cfg-index-type: equality\n" +
- "ds-cfg-index-type: substring\n" +
- "ds-cfg-index-type: ordering");
-
- ConfigEntry cnIndexConfigEntry = new ConfigEntry(cnIndexEntry, indexConfigEntry);
-
- indexConfigEntry.addChild(cnIndexConfigEntry);
- backendConfigEntry.addChild(indexConfigEntry);
-
baseDNs = new DN[]
{
- DN.decode("dc=example,dc=com"),
- DN.decode("dc=example1,dc=com")
+ DN.decode("dc=importtest,dc=com"),
+ DN.decode("dc=importtest1,dc=com")
};
- backendConfig = new Config();
- backendConfig.initializeConfig(cfg);
}
@AfterClass
@@ -294,6 +249,7 @@
@Test
public void testImportAll() throws Exception
{
+ TestCaseUtils.clearJEBackend(false, beID, null);
ArrayList<String> fileList = new ArrayList<String>();
fileList.add(homeDirName + File.separator + "top.ldif");
fileList.add(homeDirName + File.separator + "entries1.ldif");
@@ -305,55 +261,68 @@
importConfig.setValidateSchema(true);
importConfig.writeRejectedEntries(rejectedEntries);
- ImportJob importJob = new ImportJob(null, backendConfig, importConfig);
- importJob.importLDIF();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ TaskUtils.disableBackend(beID);
+ try
+ {
+ be.importLDIF(importConfig);
+ }
+ finally
+ {
+ TaskUtils.enableBackend(beID);
+ }
- RootContainer rootContainer = new RootContainer(backendConfig, null);
- rootContainer.open();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ RootContainer rootContainer = be.getRootContainer();
EntryContainer entryContainer;
assertTrue(rejectedEntries.size() <= 0);
for(DN baseDN : baseDNs)
{
- entryContainer = rootContainer.openEntryContainer(baseDN);
-
-
- assertNotNull(entryContainer);
-
- if(baseDN.toString().equals("dc=example,dc=com"))
+ entryContainer = rootContainer.getEntryContainer(baseDN);
+ entryContainer.sharedLock.lock();
+ try
{
- assertEquals(entryContainer.getEntryCount(), 4);
- assertTrue(entryContainer.entryExists(baseDN));
- assertTrue(entryContainer.entryExists(DN.decode("ou=People,dc=example,dc=com")));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.0,ou=People,dc=example,dc=com")));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.539,ou=People,dc=example,dc=com")));
+ assertNotNull(entryContainer);
- VerifyConfig verifyConfig = new VerifyConfig();
- verifyConfig.setBaseDN(baseDN);
+ if(baseDN.toString().equals("dc=importtest,dc=com"))
+ {
+ assertEquals(entryContainer.getEntryCount(), 4);
+ assertTrue(entryContainer.entryExists(baseDN));
+ assertTrue(entryContainer.entryExists(DN.decode("ou=People,dc=importtest,dc=com")));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.0,ou=People,dc=importtest,dc=com")));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.539,ou=People,dc=importtest,dc=com")));
- VerifyJob verifyJob = new VerifyJob(backendConfig, verifyConfig);
- Entry statEntry=bldStatEntry("");
- verifyJob.verifyBackend(rootContainer, statEntry);
- assertEquals(getStatEntryCount(statEntry, errorCount), 0);
+ VerifyConfig verifyConfig = new VerifyConfig();
+ verifyConfig.setBaseDN(baseDN);
+
+ Entry statEntry=bldStatEntry("");
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ be.verifyBackend(verifyConfig, statEntry);
+ assertEquals(getStatEntryCount(statEntry, errorCount), 0);
+ }
+ else if(baseDN.toString().equals("dc=importtest1,dc=com"))
+ {
+ assertEquals(entryContainer.getEntryCount(), 3);
+ assertTrue(entryContainer.entryExists(baseDN));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.446,dc=importtest1,dc=com")));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.362,dc=importtest1,dc=com")));
+
+ VerifyConfig verifyConfig = new VerifyConfig();
+ verifyConfig.setBaseDN(baseDN);
+
+ Entry statEntry=bldStatEntry("");
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ be.verifyBackend(verifyConfig, statEntry);
+ assertEquals(getStatEntryCount(statEntry, errorCount), 0);
+ }
}
- else if(baseDN.toString().equals("dc=example1,dc=com"))
+ finally
{
- assertEquals(entryContainer.getEntryCount(), 3);
- assertTrue(entryContainer.entryExists(baseDN));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.446,dc=example1,dc=com")));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.362,dc=example1,dc=com")));
+ entryContainer.sharedLock.unlock();
- VerifyConfig verifyConfig = new VerifyConfig();
- verifyConfig.setBaseDN(baseDN);
-
- VerifyJob verifyJob = new VerifyJob(backendConfig, verifyConfig);
- Entry statEntry=bldStatEntry("");
- verifyJob.verifyBackend(rootContainer, statEntry);
- assertEquals(getStatEntryCount(statEntry, errorCount), 0);
}
}
-
- rootContainer.close();
}
@Test(dependsOnMethods = "testImportAll")
@@ -366,33 +335,47 @@
importConfig.setValidateSchema(true);
importConfig.writeRejectedEntries(rejectedEntries);
- ImportJob importJob = new ImportJob(null, backendConfig, importConfig);
- importJob.importLDIF();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ TaskUtils.disableBackend(beID);
+ try
+ {
+ be.importLDIF(importConfig);
+ }
+ finally
+ {
+ TaskUtils.enableBackend(beID);
+ }
- RootContainer rootContainer = new RootContainer(backendConfig, null);
- rootContainer.open();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ RootContainer rootContainer = be.getRootContainer();
EntryContainer entryContainer;
- entryContainer = rootContainer.openEntryContainer(DN.decode("dc=example1,dc=com"));
+ entryContainer = rootContainer.getEntryContainer(DN.decode("dc=importtest1,dc=com"));
assertNotNull(entryContainer);
- assertTrue(rejectedEntries.size() <= 0);
- Entry entry = entryContainer.getEntry(DN.decode("uid=user.446,dc=example1,dc=com"));
- assertNotNull(entry);
+ entryContainer.sharedLock.lock();
+ try
+ {
+ assertTrue(rejectedEntries.size() <= 0);
+ Entry entry = entryContainer.getEntry(DN.decode("uid=user.446,dc=importtest1,dc=com"));
+ assertNotNull(entry);
- AttributeType attribute = entry.getAttribute("cn").get(0).getAttributeType();
+ AttributeType attribute = entry.getAttribute("cn").get(0).getAttributeType();
- assertTrue(entry.hasValue(attribute, null, new AttributeValue(attribute,"Annalee Bogard")));
+ assertTrue(entry.hasValue(attribute, null, new AttributeValue(attribute,"Annalee Bogard")));
- VerifyConfig verifyConfig = new VerifyConfig();
- verifyConfig.setBaseDN(DN.decode("dc=example1,dc=com"));
+ VerifyConfig verifyConfig = new VerifyConfig();
+ verifyConfig.setBaseDN(DN.decode("dc=importtest1,dc=com"));
- VerifyJob verifyJob = new VerifyJob(backendConfig, verifyConfig);
- Entry statEntry=bldStatEntry("");
- verifyJob.verifyBackend(rootContainer, statEntry);
- assertEquals(getStatEntryCount(statEntry, errorCount), 0);
-
- rootContainer.close();
+ Entry statEntry=bldStatEntry("");
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ be.verifyBackend(verifyConfig, statEntry);
+ assertEquals(getStatEntryCount(statEntry, errorCount), 0);
+ }
+ finally
+ {
+ entryContainer.sharedLock.unlock();
+ }
}
@Test(dependsOnMethods = "testImportReplaceExisting")
@@ -405,10 +388,17 @@
importConfig.setValidateSchema(true);
importConfig.writeRejectedEntries(rejectedEntries);
- ImportJob importJob = new ImportJob(null, backendConfig, importConfig);
- importJob.importLDIF();
-
- assertTrue(rejectedEntries.toString().contains("uid=user.446,dc=example1,dc=com"));
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ TaskUtils.disableBackend(beID);
+ try
+ {
+ be.importLDIF(importConfig);
+ }
+ finally
+ {
+ TaskUtils.enableBackend(beID);
+ }
+ assertTrue(rejectedEntries.toString().contains("uid=user.446,dc=importtest1,dc=com"));
}
@Test(dependsOnMethods = "testImportReplaceExisting")
@@ -419,44 +409,66 @@
importConfig.setReplaceExistingEntries(false);
importConfig.setValidateSchema(true);
- ImportJob importJob = new ImportJob(null, backendConfig, importConfig);
- importJob.importLDIF();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ TaskUtils.disableBackend(beID);
+ try
+ {
+ be.importLDIF(importConfig);
+ }
+ finally
+ {
+ TaskUtils.enableBackend(beID);
+ }
importConfig = new LDIFImportConfig(homeDirName + File.separator + "entries1.ldif");
importConfig.setAppendToExistingData(true);
importConfig.setReplaceExistingEntries(false);
importConfig.setValidateSchema(true);
- importJob = new ImportJob(null, backendConfig, importConfig);
- importJob.importLDIF();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ TaskUtils.disableBackend(beID);
+ try
+ {
+ be.importLDIF(importConfig);
+ }
+ finally
+ {
+ TaskUtils.enableBackend(beID);
+ }
- RootContainer rootContainer = new RootContainer(backendConfig, null);
- rootContainer.open();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ RootContainer rootContainer = be.getRootContainer();
EntryContainer entryContainer;
for(DN baseDN : baseDNs)
{
- entryContainer = rootContainer.openEntryContainer(baseDN);
+ entryContainer = rootContainer.getEntryContainer(baseDN);
assertNotNull(entryContainer);
-
- if(baseDN.toString().equals("dc=example,dc=com"))
+ entryContainer.sharedLock.lock();
+ try
{
- assertEquals(entryContainer.getEntryCount(), 4);
- assertTrue(entryContainer.entryExists(baseDN));
- assertTrue(entryContainer.entryExists(DN.decode("ou=People,dc=example,dc=com")));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.0,ou=People,dc=example,dc=com")));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.539,ou=People,dc=example,dc=com")));
+ if(baseDN.toString().equals("dc=importtest,dc=com"))
+ {
+ assertEquals(entryContainer.getEntryCount(), 4);
+ assertTrue(entryContainer.entryExists(baseDN));
+ assertTrue(entryContainer.entryExists(DN.decode("ou=People,dc=importtest,dc=com")));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.0,ou=People,dc=importtest,dc=com")));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.539,ou=People,dc=importtest,dc=com")));
+ }
+ else if(baseDN.toString().equals("dc=importtest1,dc=com"))
+ {
+ assertEquals(entryContainer.getEntryCount(), 3);
+ assertTrue(entryContainer.entryExists(baseDN));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.446,dc=importtest1,dc=com")));
+ assertTrue(entryContainer.entryExists(DN.decode("uid=user.362,dc=importtest1,dc=com")));
+ }
}
- else if(baseDN.toString().equals("dc=example1,dc=com"))
+ finally
{
- assertEquals(entryContainer.getEntryCount(), 3);
- assertTrue(entryContainer.entryExists(baseDN));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.446,dc=example1,dc=com")));
- assertTrue(entryContainer.entryExists(DN.decode("uid=user.362,dc=example1,dc=com")));
+ entryContainer.sharedLock.unlock();
+ TaskUtils.enableBackend(beID);
}
}
-
- rootContainer.close();
}
@Test(dependsOnMethods = "testImportAll")
@@ -469,10 +481,18 @@
importConfig.setValidateSchema(true);
importConfig.writeRejectedEntries(rejectedEntries);
- ImportJob importJob = new ImportJob(null, backendConfig, importConfig);
- importJob.importLDIF();
+ be=(BackendImpl) DirectoryServer.getBackend(beID);
+ TaskUtils.disableBackend(beID);
+ try
+ {
+ be.importLDIF(importConfig);
+ }
+ finally
+ {
+ TaskUtils.enableBackend(beID);
+ }
- assertTrue(rejectedEntries.toString().contains("uid=user.446,dc=example1,dc=com"));
+ assertTrue(rejectedEntries.toString().contains("uid=user.446,dc=importtest1,dc=com"));
}
/**
--
Gitblit v1.10.0