From b326ec684b78ad80beb7600da95c5bf29e58900b Mon Sep 17 00:00:00 2001
From: Yannick Lecaillez <ylecaillez@forgerock.com>
Date: Wed, 27 Jan 2016 10:51:13 +0000
Subject: [PATCH] OPENDJ-2631: OOME error while importing 100M entries.
---
opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java | 40 +++++++++++++++++++++++++++-------------
1 files changed, 27 insertions(+), 13 deletions(-)
diff --git a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
index fdca497..1325758 100644
--- a/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
+++ b/opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java
@@ -1825,20 +1825,14 @@
*/
mmapBuffer.force();
mmapBuffer = null;
- try
- {
- return new FileRegionChunkCursor(channel.map(MapMode.READ_ONLY, startOffset, size));
- }
- catch (IOException e)
- {
- throw new StorageRuntimeException(e);
- }
+ return new FileRegionChunkCursor(startOffset, size);
}
/** Cursor through the specific memory-mapped file's region. */
private final class FileRegionChunkCursor implements MeteredCursor<ByteString, ByteString>
{
- private final ByteBuffer region;
+ private final long regionOffset;
+ private final long regionSize;
private final InputStream asInputStream = new InputStream()
{
@Override
@@ -1847,16 +1841,19 @@
return region.get() & 0xFF;
}
};
+ private ByteBuffer region;
private ByteString key, value;
- FileRegionChunkCursor(MappedByteBuffer data)
+ FileRegionChunkCursor(long regionOffset, long regionSize)
{
- this.region = data;
+ this.regionOffset = regionOffset;
+ this.regionSize = regionSize;
}
@Override
public boolean next()
{
+ ensureRegionIsMemoryMapped();
if (!region.hasRemaining())
{
key = value = null;
@@ -1885,6 +1882,22 @@
return true;
}
+ private void ensureRegionIsMemoryMapped()
+ {
+ if (region == null)
+ {
+ // Because mmap() regions are a counted and limited resources, we create them lazily.
+ try
+ {
+ region = channel.map(MapMode.READ_ONLY, regionOffset, regionSize);
+ }
+ catch (IOException e)
+ {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
@Override
public boolean isDefined()
{
@@ -1915,6 +1928,7 @@
public void close()
{
key = value = null;
+ region = null;
}
@Override
@@ -1926,13 +1940,13 @@
@Override
public long getNbBytesRead()
{
- return region.position();
+ return region == null ? 0 : region.position();
}
@Override
public long getNbBytesTotal()
{
- return region.limit();
+ return regionSize;
}
}
}
--
Gitblit v1.10.0