OPENDJ-2631: OOME error while importing 100M entries.

Yannick Lecaillez

25.20.2016 b326ec684b78ad80beb7600da95c5bf29e58900b

OPENDJ-2631: OOME error while importing 100M entries.

Because mmap() regions are counted and limited resources, we have to use
it sparingly. Rather than pre-opening all the existing regions, this fix
creates the mmap() region only when it is actually needed by the cursor
(on the first next() call) and "release" it when the cursor is close().

 opendj-server-legacy/src/main/java/org/opends/server/backends/pluggable/OnDiskMergeImporter.java

@@ -1825,20 +1825,14 @@
         */
        mmapBuffer.force();
        mmapBuffer = null;
        try
        {
          return new FileRegionChunkCursor(channel.map(MapMode.READ_ONLY, startOffset, size));
        }
        catch (IOException e)
        {
          throw new StorageRuntimeException(e);
        }
        return new FileRegionChunkCursor(startOffset, size);
      }

      /** Cursor through the specific memory-mapped file's region. */
      private final class FileRegionChunkCursor implements MeteredCursor<ByteString, ByteString>
      {
        private final ByteBuffer region;
        private final long regionOffset;
        private final long regionSize;
        private final InputStream asInputStream = new InputStream()
        {
          @Override
@@ -1847,16 +1841,19 @@
            return region.get() & 0xFF;
          }
        };
        private ByteBuffer region;
        private ByteString key, value;

        FileRegionChunkCursor(MappedByteBuffer data)
        FileRegionChunkCursor(long regionOffset, long regionSize)
        {
          this.region = data;
          this.regionOffset = regionOffset;
          this.regionSize = regionSize;
        }

        @Override
        public boolean next()
        {
          ensureRegionIsMemoryMapped();
          if (!region.hasRemaining())
          {
            key = value = null;
@@ -1885,6 +1882,22 @@
          return true;
        }

        private void ensureRegionIsMemoryMapped()
        {
          if (region == null)
          {
            // Because mmap() regions are a counted and limited resources, we create them lazily.
            try
            {
              region = channel.map(MapMode.READ_ONLY, regionOffset, regionSize);
            }
            catch (IOException e)
            {
              throw new IllegalStateException(e);
            }
          }
        }

        @Override
        public boolean isDefined()
        {
@@ -1915,6 +1928,7 @@
        public void close()
        {
          key = value = null;
          region = null;
        }

        @Override
@@ -1926,13 +1940,13 @@
        @Override
        public long getNbBytesRead()
        {
          return region.position();
          return region == null ? 0 : region.position();
        }

        @Override
        public long getNbBytesTotal()
        {
          return region.limit();
          return regionSize;
        }
      }
    }