From e4076b2991c9604907f6a2f5ba2d526d0072adf6 Mon Sep 17 00:00:00 2001
From: ludovicp <ludovicp@localhost>
Date: Fri, 30 Jul 2010 12:19:32 +0000
Subject: [PATCH] import-ldif: Change second phase import strategy in order to better handle large LDIF files with low memory. In first phase write buffer positions to index files instead of storing in memory and suffering OOME due to O(N) memory growth. In second phase, read buffer positions from index files and fall-back to batch import of indexes when the number of buffers for an index would cause OOME if they were all opened at once. Also, improve second phase progress statistics to report batch count, kb remaining/rate, and fix several race conditions in the statistics.
---
opends/src/server/org/opends/server/backends/jeb/importLDIF/IndexInputBuffer.java | 35 +++++++++++++++--------------------
1 files changed, 15 insertions(+), 20 deletions(-)
diff --git a/opends/src/server/org/opends/server/backends/jeb/importLDIF/IndexInputBuffer.java b/opends/src/server/org/opends/server/backends/jeb/importLDIF/IndexInputBuffer.java
index a209975..7323151 100644
--- a/opends/src/server/org/opends/server/backends/jeb/importLDIF/IndexInputBuffer.java
+++ b/opends/src/server/org/opends/server/backends/jeb/importLDIF/IndexInputBuffer.java
@@ -50,12 +50,13 @@
public final class IndexInputBuffer implements Comparable<IndexInputBuffer>
{
private final IndexManager indexMgr;
+ private final FileChannel channel;
private final long begin;
private final long end;
private final int id;
private long offset;
- private ByteBuffer cache;
+ private final ByteBuffer cache;
private Integer indexID = null;
private ByteBuffer keyBuf = ByteBuffer.allocate(128);
@@ -80,35 +81,30 @@
*
* @param indexMgr
* The index manager.
+ * @param channel
+ * The index file channel.
* @param begin
* The position of the start of the buffer in the scratch file.
* @param end
* The position of the end of the buffer in the scratch file.
* @param id
* The index ID.
+ * @param cacheSize
+ * The cache size.
+ * @throws IOException
+ * If an IO error occurred when priming the cache.
*/
- public IndexInputBuffer(IndexManager indexMgr, long begin, long end, int id)
+ public IndexInputBuffer(IndexManager indexMgr, FileChannel channel,
+ long begin, long end, int id, int cacheSize) throws IOException
{
this.indexMgr = indexMgr;
+ this.channel = channel;
this.begin = begin;
this.end = end;
this.offset = 0;
this.id = id;
- }
+ this.cache = ByteBuffer.allocate(Math.max(cacheSize - 384, 256));
-
-
- /**
- * Initializes this index input buffer using the provided cache size.
- *
- * @param cacheSize
- * The cache size.
- * @throws IOException
- * If an IO error occurred.
- */
- void initializeCache(long cacheSize) throws IOException
- {
- cache = ByteBuffer.allocate((int) Math.max(cacheSize - 256, 256));
loadCache();
cache.flip();
keyBuf.flip();
@@ -118,8 +114,7 @@
private void loadCache() throws IOException
{
- FileChannel fileChannel = indexMgr.getChannel();
- fileChannel.position(begin + offset);
+ channel.position(begin + offset);
long leftToRead = end - (begin + offset);
long bytesToRead;
if (leftToRead < cache.remaining())
@@ -134,7 +129,7 @@
int bytesRead = 0;
while (bytesRead < bytesToRead)
{
- bytesRead += fileChannel.read(cache);
+ bytesRead += channel.read(cache);
}
offset += bytesRead;
indexMgr.addBytesRead(bytesRead);
@@ -199,7 +194,7 @@
catch (IOException ex)
{
Message message = ERR_JEB_IMPORT_BUFFER_IO_ERROR.get(indexMgr
- .getFileName());
+ .getBufferFileName());
logError(message);
ex.printStackTrace();
System.exit(1);
--
Gitblit v1.10.0