Skip to content

Commit

Permalink
Cleanup code and move IOContext mapping to Posix, as the constants we…
Browse files Browse the repository at this point in the history
… use are Poxix only. On Windows we can have a totally different way to madvise the kernel
  • Loading branch information
uschindler committed Mar 22, 2024
1 parent 242e5e9 commit a729f02
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 48 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.OptionalInt;
import org.apache.lucene.store.IOContext.Context;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.Unwrappable;

Expand Down Expand Up @@ -83,14 +81,6 @@ private final MemorySegment[] map(
throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription);
}

final OptionalInt advice;
if (chunkSizePower < 21) {
// if chunk size is too small (2 MiB), disable madvise support (incorrect alignment):
advice = OptionalInt.empty();
} else {
advice = mapContextToMadvise(context);
}

final long chunkSize = 1L << chunkSizePower;

// we always allocate one more segments, the last one may be a 0 byte one
Expand All @@ -108,29 +98,16 @@ private final MemorySegment[] map(
} catch (IOException ioe) {
throw convertMapFailedIOException(ioe, resourceDescription, segSize);
}
// if preload apply it without madvise.
// if chunk size is too small (2 MiB), disable madvise support (incorrect alignment)
if (preload) {
segment.load();
} else if (segSize > 0L && advice.isPresent()) { // not when preloading!
nativeAccess.madvise(segment, advice.getAsInt());
} else if (chunkSizePower >= 21) {
nativeAccess.madvise(segment, context);
}
segments[segNr] = segment;
startOffset += segSize;
}
return segments;
}

private OptionalInt mapContextToMadvise(IOContext context) {
// Merging always wins and implies sequential access, because kernel is advised to free pages
// after use:
if (context.context == Context.MERGE) {
return OptionalInt.of(NativeAccess.POSIX_MADV_SEQUENTIAL);
}
if (context.randomAccess) {
return OptionalInt.of(NativeAccess.POSIX_MADV_RANDOM);
}
if (context.readOnce) {
return OptionalInt.of(NativeAccess.POSIX_MADV_SEQUENTIAL);
}
return OptionalInt.empty();
}
}
19 changes: 1 addition & 18 deletions lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,25 +26,8 @@
abstract class NativeAccess {
private static final Logger LOG = Logger.getLogger(NativeAccess.class.getName());

// these constants were extracted from glibc and macos header files - luckily they are the same:

/** No further special treatment. */
public static final int POSIX_MADV_NORMAL = 0;

/** Expect random page references. */
public static final int POSIX_MADV_RANDOM = 1;

/** Expect sequential page references. */
public static final int POSIX_MADV_SEQUENTIAL = 2;

/** Will need these pages. */
public static final int POSIX_MADV_WILLNEED = 3;

/** Don't need these pages. */
public static final int POSIX_MADV_DONTNEED = 4;

/** Invoke the {@code madvise} call for the given {@link MemorySegment}. */
public abstract void madvise(MemorySegment segment, int advice) throws IOException;
public abstract void madvise(MemorySegment segment, IOContext context) throws IOException;

/**
* Return the NativeAccess instance for this platform. At moment we only support Linux and MacOS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,5 @@
final class NoopNativeAccess extends NativeAccess {

@Override
public void madvise(MemorySegment segment, int advice) {}
public void madvise(MemorySegment segment, IOContext context) {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,30 @@
import java.lang.invoke.MethodHandle;
import java.util.Locale;
import java.util.logging.Logger;
import org.apache.lucene.store.IOContext.Context;

@SuppressWarnings("preview")
final class PosixNativeAccess extends NativeAccess {

private static final Logger LOG = Logger.getLogger(PosixNativeAccess.class.getName());

// these constants were extracted from glibc and macos header files - luckily they are the same:

/** No further special treatment. */
public static final int POSIX_MADV_NORMAL = 0;

/** Expect random page references. */
public static final int POSIX_MADV_RANDOM = 1;

/** Expect sequential page references. */
public static final int POSIX_MADV_SEQUENTIAL = 2;

/** Will need these pages. */
public static final int POSIX_MADV_WILLNEED = 3;

/** Don't need these pages. */
public static final int POSIX_MADV_DONTNEED = 4;

private final MethodHandle mh$posix_madvise;

public PosixNativeAccess() {
Expand Down Expand Up @@ -62,10 +80,17 @@ private static MethodHandle findFunction(
}

@Override
public void madvise(MemorySegment segment, int advice) throws IOException {
public void madvise(MemorySegment segment, IOContext context) throws IOException {
if (segment.byteSize() == 0L) {
return; // empty segments should be excluded, because they may have no address at all
}
final Integer advice = mapIOContext(context);
if (advice == null) {
return; // do nothing
}
final int ret;
try {
ret = (int) mh$posix_madvise.invokeExact(segment, segment.byteSize(), advice);
ret = (int) mh$posix_madvise.invokeExact(segment, segment.byteSize(), advice.intValue());
} catch (Throwable th) {
throw new AssertionError(th);
}
Expand All @@ -79,4 +104,19 @@ public void madvise(MemorySegment segment, int advice) throws IOException {
ret));
}
}

private Integer mapIOContext(IOContext ctx) {
// Merging always wins and implies sequential access, because kernel is advised to free pages
// after use:
if (ctx.context == Context.MERGE) {
return POSIX_MADV_SEQUENTIAL;
}
if (ctx.randomAccess) {
return POSIX_MADV_RANDOM;
}
if (ctx.readOnce) {
return POSIX_MADV_SEQUENTIAL;
}
return null;
}
}

0 comments on commit a729f02

Please sign in to comment.