Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for memory-mapped files #604

Merged
merged 18 commits into from
May 16, 2024
Merged

Add support for memory-mapped files #604

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
230e8c8
Change io_cb read API to allow passing back an existing pointer
JanSellner May 2, 2024
1c04f36
New IO mode using memory mapped files (no Windows support yet)
JanSellner May 2, 2024
782b990
Add Windows support for memory-mapped files
JanSellner May 3, 2024
56461f3
fix warnings
JanSellner May 3, 2024
de42f28
Ensure borders of memory-mapped file are respected
JanSellner May 3, 2024
c0f4336
Use mapping size independent of file size and start with large enough…
JanSellner May 3, 2024
d7278bc
Improve extended mapping size
JanSellner May 4, 2024
63c9389
New default initialization, added test for c mode
JanSellner May 4, 2024
221af5f
Use different define to detect Windows systems
JanSellner May 4, 2024
7280ea6
Use different write file mapping mode on Windows which does not crash
JanSellner May 8, 2024
32eed17
Introduce position argument to read/write functions instead of seekin…
JanSellner May 8, 2024
5e1a292
Resolve review comments
JanSellner May 8, 2024
0c8787d
Remove seek and tell callbacks, introduce size instead
JanSellner May 13, 2024
694f0a1
Merge commit 'd2167d34120f617f64f48090f2495fc6e9ae8cf5' into mmap
JanSellner May 14, 2024
115a697
add function to retrieve default struct
JanSellner May 14, 2024
5d7e4e1
Add helper script for aggregated benchmarks across multiple runs
JanSellner May 14, 2024
cceb06d
Expand documentation and add a check that only one file is opened
JanSellner May 14, 2024
40e101b
Use msync only in the end
JanSellner May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
389 changes: 387 additions & 2 deletions blosc/blosc2-stdio.c
View file
Open in desktop

Large diffs are not rendered by default.

24 changes: 23 additions & 1 deletion blosc/blosc2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1638,7 +1638,7 @@ static int blosc_d(
io_cb->seek(fp, frame->file_offset + chunk_offset + src_offset, SEEK_SET);
}
// We can make use of tmp3 because it will be used after src is not needed anymore
int64_t rbytes = io_cb->read(tmp3, 1, block_csize, fp);
int64_t rbytes = io_cb->read((void**)&tmp3, 1, block_csize, fp);
io_cb->close(fp);
if ((int32_t)rbytes != block_csize) {
BLOSC_TRACE_ERROR("Cannot read the (lazy) block out of the fileframe.");
Expand Down Expand Up @@ -3789,20 +3789,35 @@ void blosc_set_schunk(blosc2_schunk* schunk) {

blosc2_io *blosc2_io_global = NULL;
blosc2_io_cb BLOSC2_IO_CB_DEFAULTS;
blosc2_io_cb BLOSC2_IO_CB_MMAP;

void blosc2_init(void) {
/* Return if Blosc is already initialized */
if (g_initlib) return;

BLOSC2_IO_CB_DEFAULTS.id = BLOSC2_IO_FILESYSTEM;
BLOSC2_IO_CB_DEFAULTS.name = "filesystem";
BLOSC2_IO_CB_DEFAULTS.is_allocation_necessary = true;
BLOSC2_IO_CB_DEFAULTS.open = (blosc2_open_cb) blosc2_stdio_open;
BLOSC2_IO_CB_DEFAULTS.close = (blosc2_close_cb) blosc2_stdio_close;
BLOSC2_IO_CB_DEFAULTS.tell = (blosc2_tell_cb) blosc2_stdio_tell;
BLOSC2_IO_CB_DEFAULTS.seek = (blosc2_seek_cb) blosc2_stdio_seek;
BLOSC2_IO_CB_DEFAULTS.write = (blosc2_write_cb) blosc2_stdio_write;
BLOSC2_IO_CB_DEFAULTS.read = (blosc2_read_cb) blosc2_stdio_read;
BLOSC2_IO_CB_DEFAULTS.truncate = (blosc2_truncate_cb) blosc2_stdio_truncate;
BLOSC2_IO_CB_DEFAULTS.free = NULL;

BLOSC2_IO_CB_MMAP.id = BLOSC2_IO_FILESYSTEM_MMAP;
BLOSC2_IO_CB_MMAP.name = "filesystem_mmap";
BLOSC2_IO_CB_MMAP.is_allocation_necessary = false;
BLOSC2_IO_CB_MMAP.open = (blosc2_open_cb) blosc2_stdio_mmap_open;
BLOSC2_IO_CB_MMAP.close = (blosc2_close_cb) blosc2_stdio_mmap_close;
BLOSC2_IO_CB_MMAP.read = (blosc2_read_cb) blosc2_stdio_mmap_read;
BLOSC2_IO_CB_MMAP.tell = (blosc2_tell_cb) blosc2_stdio_mmap_tell;
BLOSC2_IO_CB_MMAP.seek = (blosc2_seek_cb) blosc2_stdio_mmap_seek;
BLOSC2_IO_CB_MMAP.write = (blosc2_write_cb) blosc2_stdio_mmap_write;
BLOSC2_IO_CB_MMAP.truncate = (blosc2_truncate_cb) blosc2_stdio_mmap_truncate;
BLOSC2_IO_CB_MMAP.free = (blosc2_free_cb) blosc2_stdio_mmap_free;

g_ncodecs = 0;
g_nfilters = 0;
Expand Down Expand Up @@ -4644,6 +4659,13 @@ blosc2_io_cb *blosc2_get_io_cb(uint8_t id) {
}
return blosc2_get_io_cb(id);
}
else if (id == BLOSC2_IO_FILESYSTEM_MMAP) {
if (_blosc2_register_io_cb(&BLOSC2_IO_CB_MMAP) < 0) {
BLOSC_TRACE_ERROR("Error registering the mmap IO API");
return NULL;
}
return blosc2_get_io_cb(id);
}
return NULL;
}

Expand Down
Loading