Skip to content

Minimal changes required to implement Foreign Function and Memory API #258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stroomdev66 merged 1 commit into from
Oct 30, 2025
Merged

Minimal changes required to implement Foreign Function and Memory API #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 6 additions & 10 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@
<description>Low latency Java API for the ultra-fast, embedded Symas Lightning Database (LMDB)</description>
<properties>
<maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ss'Z'</maven.build.timestamp.format>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.enforcer.java>1.8</maven.enforcer.java>
<maven.compiler.source>1.25</maven.compiler.source>
<maven.compiler.target>1.25</maven.compiler.target>
<maven.enforcer.java>1.25</maven.enforcer.java>
<maven.enforcer.mvn>3.5.4</maven.enforcer.mvn>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<trimStackTrace>false</trimStackTrace>
Expand All @@ -39,11 +39,6 @@
<artifactId>jnr-constants</artifactId>
<version>0.10.4</version>
</dependency>
<dependency>
<groupId>com.github.jnr</groupId>
<artifactId>jnr-ffi</artifactId>
<version>2.2.17</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down Expand Up @@ -89,8 +84,8 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>4.11.0</version>
<artifactId>mockito-core</artifactId>
<version>5.20.0</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down Expand Up @@ -218,6 +213,7 @@
</manifest>
<manifestEntries>
<Implementation-Build>${buildNumber}</Implementation-Build>
<Enable-Native-Access>true</Enable-Native-Access>
</manifestEntries>
</archive>
</configuration>
Expand Down
18 changes: 8 additions & 10 deletions src/main/java/org/lmdbjava/BufferProxy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,16 @@
*/
package org.lmdbjava;

import org.lmdbjava.Lmdb.MDB_val;

import static java.lang.Long.BYTES;
import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY;
import static org.lmdbjava.DbiFlags.MDB_UNSIGNEDKEY;
import static org.lmdbjava.MaskedFlag.isSet;
import static org.lmdbjava.MaskedFlag.mask;

import java.lang.foreign.Arena;
import java.util.Comparator;
import jnr.ffi.Pointer;

/**
* The strategy for mapping memory address to a given buffer type.
Expand Down Expand Up @@ -107,37 +109,33 @@ protected Comparator<T> getComparator(DbiFlags... flags) {
*
* @param buffer the buffer to write to <code>MDB_val</code>
* @param ptr the pointer to the <code>MDB_val</code>
* @param ptrAddr the address of the <code>MDB_val</code> pointer
*/
protected abstract void in(T buffer, Pointer ptr, long ptrAddr);
protected abstract void in(T buffer, MDB_val ptr);

/**
* Called when the <code>MDB_val</code> should be set to reflect the passed buffer.
*
* @param buffer the buffer to write to <code>MDB_val</code>
* @param size the buffer size to write to <code>MDB_val</code>
* @param ptr the pointer to the <code>MDB_val</code>
* @param ptrAddr the address of the <code>MDB_val</code> pointer
*/
protected abstract void in(T buffer, int size, Pointer ptr, long ptrAddr);
protected abstract void in(T buffer, int size, MDB_val ptr);

/**
* Called when the <code>MDB_val</code> may have changed and the passed buffer should be modified
* to reflect the new <code>MDB_val</code>.
*
* @param buffer the buffer to write to <code>MDB_val</code>
* @param ptr the pointer to the <code>MDB_val</code>
* @param ptrAddr the address of the <code>MDB_val</code> pointer
* @return the buffer for <code>MDB_val</code>
*/
protected abstract T out(T buffer, Pointer ptr, long ptrAddr);
protected abstract T out(MDB_val ptr);

/**
* Create a new {@link KeyVal} to hold pointers for this buffer proxy.
*
* @return a non-null key value holder
*/
final KeyVal<T> keyVal() {
return new KeyVal<>(this);
final KeyVal<T> keyVal(final Arena arena) {
return new KeyVal<>(arena,this);
}
}
43 changes: 21 additions & 22 deletions src/main/java/org/lmdbjava/ByteArrayProxy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,17 @@
*/
package org.lmdbjava;

import static java.lang.Math.min;
import static java.util.Objects.requireNonNull;
import static org.lmdbjava.Library.RUNTIME;
import org.lmdbjava.Lmdb.MDB_val;

import java.lang.foreign.Arena;
import java.lang.foreign.MemorySegment;
import java.lang.foreign.ValueLayout;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Comparator;
import jnr.ffi.Pointer;
import jnr.ffi.provider.MemoryManager;

import static java.lang.Math.min;
import static java.util.Objects.requireNonNull;

/**
* Byte array proxy.
Expand All @@ -31,15 +34,14 @@
*/
public final class ByteArrayProxy extends BufferProxy<byte[]> {

/** The byte array proxy. Guaranteed to never be null. */
public static final BufferProxy<byte[]> PROXY_BA = new ByteArrayProxy();

private static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager();
private final Arena arena;

private static final Comparator<byte[]> signedComparator = ByteArrayProxy::compareArraysSigned;
private static final Comparator<byte[]> unsignedComparator = ByteArrayProxy::compareArrays;

private ByteArrayProxy() {}
public ByteArrayProxy(final Arena arena) {
this.arena = arena;
}

/**
* Lexicographically compare two byte arrays.
Expand Down Expand Up @@ -114,25 +116,22 @@ protected Comparator<byte[]> getUnsignedComparator() {
}

@Override
protected void in(final byte[] buffer, final Pointer ptr, final long ptrAddr) {
final Pointer pointer = MEM_MGR.allocateDirect(buffer.length);
pointer.put(0, buffer, 0, buffer.length);
ptr.putLong(STRUCT_FIELD_OFFSET_SIZE, buffer.length);
ptr.putAddress(STRUCT_FIELD_OFFSET_DATA, pointer.address());
protected void in(final byte[] buffer, final MDB_val ptr) {
final MemorySegment segment = arena.allocateFrom(ValueLayout.JAVA_BYTE, buffer);
ptr.mvSize(buffer.length);
ptr.mvData(segment);
}

@Override
protected void in(final byte[] buffer, final int size, final Pointer ptr, final long ptrAddr) {
protected void in(final byte[] buffer, final int size, final MDB_val ptr) {
// cannot reserve for byte arrays
}

@Override
protected byte[] out(final byte[] buffer, final Pointer ptr, final long ptrAddr) {
final long addr = ptr.getAddress(STRUCT_FIELD_OFFSET_DATA);
final int size = (int) ptr.getLong(STRUCT_FIELD_OFFSET_SIZE);
final Pointer pointer = MEM_MGR.newPointer(addr, size);
final byte[] bytes = new byte[size];
pointer.get(0, bytes, 0, size);
protected byte[] out(final MDB_val ptr) {
final ByteBuffer byteBuffer = ptr.mvData().reinterpret(ptr.mvSize()).asByteBuffer();
final byte[] bytes = new byte[byteBuffer.remaining()];
byteBuffer.get(0, bytes, 0, byteBuffer.remaining());
return bytes;
}
}
86 changes: 24 additions & 62 deletions src/main/java/org/lmdbjava/ByteBufProxy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,22 @@
*/
package org.lmdbjava;

import static io.netty.buffer.PooledByteBufAllocator.DEFAULT;
import static java.lang.Class.forName;
import static java.util.Objects.requireNonNull;
import static org.lmdbjava.UnsafeAccess.UNSAFE;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.PooledByteBufAllocator;
import java.lang.reflect.Field;
import io.netty.buffer.Unpooled;
import org.lmdbjava.Lmdb.MDB_val;

import java.lang.foreign.MemorySegment;
import java.nio.ByteBuffer;
import java.util.Comparator;
import jnr.ffi.Pointer;

import static io.netty.buffer.PooledByteBufAllocator.DEFAULT;
import static java.util.Objects.requireNonNull;

/**
* A buffer proxy backed by Netty's {@link ByteBuf}.
*
* <p>This class requires {@link UnsafeAccess} and netty-buffer must be in the classpath.
* <p>This class requires netty-buffer in the classpath.
*/
public final class ByteBufProxy extends BufferProxy<ByteBuf> {

Expand All @@ -41,18 +42,14 @@ public final class ByteBufProxy extends BufferProxy<ByteBuf> {
public static final BufferProxy<ByteBuf> PROXY_NETTY = new ByteBufProxy();

private static final int BUFFER_RETRIES = 10;
private static final String FIELD_NAME_ADDRESS = "memoryAddress";
private static final String FIELD_NAME_LENGTH = "length";
private static final String NAME = "io.netty.buffer.PooledUnsafeDirectByteBuf";
private static final Comparator<ByteBuf> comparator =
(o1, o2) -> {
requireNonNull(o1);
requireNonNull(o2);
(o1, o2) -> {
requireNonNull(o1);
requireNonNull(o2);

return o1.compareTo(o2);
};
private final long lengthOffset;
private final long addressOffset;
return o1.compareTo(o2);
};

private final PooledByteBufAllocator nettyAllocator;

Expand All @@ -68,36 +65,6 @@ private ByteBufProxy() {
public ByteBufProxy(final PooledByteBufAllocator allocator) {
super();
this.nettyAllocator = allocator;

try {
final ByteBuf initBuf = this.allocate();
initBuf.release();
final Field address = findField(NAME, FIELD_NAME_ADDRESS);
final Field length = findField(NAME, FIELD_NAME_LENGTH);
addressOffset = UNSAFE.objectFieldOffset(address);
lengthOffset = UNSAFE.objectFieldOffset(length);
} catch (final SecurityException e) {
throw new LmdbException("Field access error", e);
}
}

static Field findField(final String c, final String name) {
Class<?> clazz;
try {
clazz = forName(c);
} catch (final ClassNotFoundException e) {
throw new LmdbException(c + " class unavailable", e);
}
do {
try {
final Field field = clazz.getDeclaredField(name);
field.setAccessible(true);
return field;
} catch (final NoSuchFieldException e) {
clazz = clazz.getSuperclass();
}
} while (clazz != null);
throw new LmdbException(name + " not found");
}

@Override
Expand Down Expand Up @@ -136,26 +103,21 @@ protected byte[] getBytes(final ByteBuf buffer) {
}

@Override
protected void in(final ByteBuf buffer, final Pointer ptr, final long ptrAddr) {
UNSAFE.putLong(ptrAddr + STRUCT_FIELD_OFFSET_SIZE, buffer.writerIndex() - buffer.readerIndex());
UNSAFE.putLong(
ptrAddr + STRUCT_FIELD_OFFSET_DATA, buffer.memoryAddress() + buffer.readerIndex());
protected void in(final ByteBuf buffer, final MDB_val ptr) {
ptr.mvSize(buffer.writerIndex() - buffer.readerIndex());
ptr.mvData(MemorySegment.ofBuffer(buffer.nioBuffer()));
}

@Override
protected void in(final ByteBuf buffer, final int size, final Pointer ptr, final long ptrAddr) {
UNSAFE.putLong(ptrAddr + STRUCT_FIELD_OFFSET_SIZE, size);
UNSAFE.putLong(
ptrAddr + STRUCT_FIELD_OFFSET_DATA, buffer.memoryAddress() + buffer.readerIndex());
protected void in(final ByteBuf buffer, final int size, final MDB_val ptr) {
ptr.mvSize(size);
ptr.mvData(MemorySegment.ofBuffer(buffer.nioBuffer()));
}

@Override
protected ByteBuf out(final ByteBuf buffer, final Pointer ptr, final long ptrAddr) {
final long addr = UNSAFE.getLong(ptrAddr + STRUCT_FIELD_OFFSET_DATA);
final long size = UNSAFE.getLong(ptrAddr + STRUCT_FIELD_OFFSET_SIZE);
UNSAFE.putLong(buffer, addressOffset, addr);
UNSAFE.putInt(buffer, lengthOffset, (int) size);
buffer.clear().writerIndex((int) size);
return buffer;
protected ByteBuf out(final MDB_val ptr) {
final long size = ptr.mvSize();
final ByteBuffer byteBuffer = ptr.mvData().reinterpret(size).asByteBuffer();
return Unpooled.wrappedBuffer(byteBuffer);
}
}
Loading