Skip to content

Commit

Permalink
Merge pull request #207 from nyurik/alloc-util
Browse files Browse the repository at this point in the history
Simplify mem cell alloc usage
  • Loading branch information
danielrh authored May 19, 2024
2 parents 859f722 + 6b01faf commit 1730d38
Show file tree
Hide file tree
Showing 20 changed files with 260 additions and 413 deletions.
2 changes: 1 addition & 1 deletion src/bin/validate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ pub fn compress_validate<InputType: Read, OutputType: Write>(
num_threads: usize,
) -> Result<(), io::Error> {
let mut m8 = HeapAllocator::default();
let buffer = <HeapAllocator as Allocator<u8>>::alloc_cell(&mut m8, buffer_size);
let buffer = m8.alloc_cell(buffer_size);
// FIXME: could reuse the dictionary to seed the compressor, but that violates the abstraction right now
// also dictionaries are not very popular since they are mostly an internal concept, given their deprecation in
// the standard brotli spec
Expand Down
18 changes: 4 additions & 14 deletions src/enc/backward_references/benchmark.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,11 @@ fn make_generic_hasher() -> AdvHasher<H5Sub, StandardAlloc> {
let block_size = 1u64 << params_hasher.block_bits;
let bucket_size = 1u64 << params_hasher.bucket_bits;
let mut alloc = StandardAlloc::default();
let buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
let num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);

AdvHasher::<H5Sub, StandardAlloc> {
buckets: buckets,
buckets: alloc.alloc_cell((bucket_size * block_size) as usize),
h9_opts: H9Opts::new(&params_hasher),
num: num,
num: alloc.alloc_cell(bucket_size as usize),
GetHasherCommon: Struct1 {
params: params_hasher,
is_prepared_: 1,
Expand All @@ -65,16 +60,11 @@ fn make_specialized_hasher() -> AdvHasher<HQ7Sub, StandardAlloc> {
let block_size = 1u64 << params_hasher.block_bits;
let bucket_size = 1u64 << params_hasher.bucket_bits;
let mut alloc = StandardAlloc::default();
let buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
let num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);

AdvHasher::<HQ7Sub, StandardAlloc> {
buckets: buckets,
buckets: alloc.alloc_cell((bucket_size * block_size) as usize),
h9_opts: H9Opts::new(&params_hasher),
num: num,
num: alloc.alloc_cell(bucket_size as usize),
GetHasherCommon: Struct1 {
params: params_hasher,
is_prepared_: 1,
Expand Down
3 changes: 2 additions & 1 deletion src/enc/backward_references/hash_to_binary_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use super::{
kHashMul32, AnyHasher, BrotliEncoderParams, CloneWithAlloc, H9Opts, HasherSearchResult,
HowPrepared, Struct1,
};
use crate::enc::combined_alloc::allocate;
use crate::enc::static_dict::{
BrotliDictionary, FindMatchLengthWithLimit, BROTLI_UNALIGNED_LOAD32,
};
Expand Down Expand Up @@ -206,7 +207,7 @@ where
common: self.common.clone(),
buckets_: Buckets::new_uninit(m),
invalid_pos_: self.invalid_pos_,
forest: <Alloc as Allocator<u32>>::alloc_cell(m, self.forest.len()),
forest: allocate::<u32, _>(m, self.forest.len()),
_params: core::marker::PhantomData::<Params>,
};
ret.buckets_
Expand Down
54 changes: 16 additions & 38 deletions src/enc/backward_references/hq.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use super::{
kDistanceCacheIndex, kDistanceCacheOffset, kHashMul32, kInvalidMatch, AnyHasher,
BrotliEncoderParams,
};
use crate::enc::combined_alloc::{alloc_if, alloc_or_default};
use crate::enc::command::{
combine_length_codes, BrotliDistanceParams, Command, GetCopyLengthCode, GetInsertLengthCode,
PrefixEncodeCopyDistance,
Expand Down Expand Up @@ -209,16 +210,14 @@ impl<AllocF: Allocator<floatX>> ZopfliCostModel<AllocF> {
num_bytes_: num_bytes,
cost_cmd_: [0.0; 704],
min_cost_cmd_: 0.0,
literal_costs_: if num_bytes.wrapping_add(2) > 0usize {
m.alloc_cell(num_bytes.wrapping_add(2))
} else {
AllocF::AllocatedMemory::default()
},
cost_dist_: if dist.alphabet_size > 0u32 {
m.alloc_cell(num_bytes.wrapping_add(dist.alphabet_size as usize))
} else {
AllocF::AllocatedMemory::default()
},
// FIXME: makes little sense to test if N+2 > 0 -- always true unless wrapping. Perhaps use allocate() instead?
literal_costs_: alloc_or_default::<floatX, _>(m, num_bytes + 2),
// FIXME: possible bug because allocation size is different from the condition
cost_dist_: alloc_if::<floatX, _>(
dist.alphabet_size > 0,
m,
num_bytes + dist.alphabet_size as usize,
),
distance_histogram_size: min(dist.alphabet_size, 544),
}
}
Expand Down Expand Up @@ -988,12 +987,8 @@ pub fn BrotliCreateZopfliBackwardReferences<
Buckets: PartialEq<Buckets>,
{
let max_backward_limit: usize = (1usize << params.lgwin).wrapping_sub(16);
let mut nodes: <Alloc as Allocator<ZopfliNode>>::AllocatedMemory;
nodes = if num_bytes.wrapping_add(1) > 0usize {
<Alloc as Allocator<ZopfliNode>>::alloc_cell(alloc, num_bytes.wrapping_add(1))
} else {
<Alloc as Allocator<ZopfliNode>>::AllocatedMemory::default()
};
// FIXME: makes little sense to test if N+1 > 0 -- always true unless wrapping. Perhaps use allocate() instead?
let mut nodes = alloc_or_default::<ZopfliNode, _>(alloc, num_bytes + 1);
if !(0i32 == 0) {
return;
}
Expand Down Expand Up @@ -1244,11 +1239,7 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
Buckets: PartialEq<Buckets>,
{
let max_backward_limit: usize = (1usize << params.lgwin).wrapping_sub(16);
let mut num_matches: <Alloc as Allocator<u32>>::AllocatedMemory = if num_bytes > 0usize {
<Alloc as Allocator<u32>>::alloc_cell(alloc, num_bytes)
} else {
<Alloc as Allocator<u32>>::AllocatedMemory::default()
};
let mut num_matches = alloc_or_default::<u32, _>(alloc, num_bytes);
let mut matches_size: usize = (4usize).wrapping_mul(num_bytes);
let store_end: usize = if num_bytes >= STORE_LOOKAHEAD_H_10 {
position
Expand All @@ -1264,12 +1255,7 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
let mut orig_dist_cache = [0i32; 4];

let mut model: ZopfliCostModel<Alloc>;
let mut nodes: <Alloc as Allocator<ZopfliNode>>::AllocatedMemory;
let mut matches: <Alloc as Allocator<u64>>::AllocatedMemory = if matches_size > 0usize {
<Alloc as Allocator<u64>>::alloc_cell(alloc, matches_size)
} else {
<Alloc as Allocator<u64>>::AllocatedMemory::default()
};
let mut matches = alloc_or_default::<u64, _>(alloc, matches_size);
let gap: usize = 0usize;
let shadow_matches: usize = 0usize;
i = 0usize;
Expand All @@ -1287,15 +1273,10 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
} else {
matches_size
};
let mut new_array: <Alloc as Allocator<u64>>::AllocatedMemory;
while new_size < cur_match_pos.wrapping_add(128).wrapping_add(shadow_matches) {
new_size = new_size.wrapping_mul(2);
}
new_array = if new_size > 0usize {
<Alloc as Allocator<u64>>::alloc_cell(alloc, new_size)
} else {
<Alloc as Allocator<u64>>::AllocatedMemory::default()
};
let mut new_array = alloc_or_default::<u64, _>(alloc, new_size);
if matches_size != 0 {
for (dst, src) in new_array
.slice_mut()
Expand Down Expand Up @@ -1380,11 +1361,8 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
*i = *j;
}
let orig_num_commands: usize = *num_commands;
nodes = if num_bytes.wrapping_add(1) > 0usize {
<Alloc as Allocator<ZopfliNode>>::alloc_cell(alloc, num_bytes.wrapping_add(1))
} else {
<Alloc as Allocator<ZopfliNode>>::AllocatedMemory::default()
};
// FIXME: makes little sense to test if N+1 > 0 -- always true unless wrapping. Perhaps use allocate() instead?
let mut nodes = alloc_or_default::<ZopfliNode, _>(alloc, num_bytes + 1);
if !(0i32 == 0) {
return;
}
Expand Down
19 changes: 10 additions & 9 deletions src/enc/backward_references/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use super::static_dict::{
BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64,
};
use super::util::{floatX, Log2FloorNonZero};
use crate::enc::combined_alloc::allocate;

static kBrotliMinWindowBits: i32 = 10;
static kBrotliMaxWindowBits: i32 = 24;
Expand Down Expand Up @@ -1958,8 +1959,8 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
fn clone_with_alloc(&self, m: &mut Alloc) -> Self {
let mut ret = BasicHasher::<H2Sub<Alloc>> {
GetHasherCommon: self.GetHasherCommon.clone(),
buckets_: H2Sub::<Alloc> {
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.buckets_.len()),
buckets_: H2Sub {
buckets_: allocate::<u32, _>(m, self.buckets_.buckets_.len()),
},
h9_opts: self.h9_opts,
};
Expand All @@ -1977,7 +1978,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
let mut ret = BasicHasher::<H3Sub<Alloc>> {
GetHasherCommon: self.GetHasherCommon.clone(),
buckets_: H3Sub::<Alloc> {
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.buckets_.len()),
buckets_: allocate::<u32, _>(m, self.buckets_.buckets_.len()),
},
h9_opts: self.h9_opts,
};
Expand All @@ -1995,7 +1996,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
let mut ret = BasicHasher::<H4Sub<Alloc>> {
GetHasherCommon: self.GetHasherCommon.clone(),
buckets_: H4Sub::<Alloc> {
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.buckets_.len()),
buckets_: allocate::<u32, _>(m, self.buckets_.buckets_.len()),
},
h9_opts: self.h9_opts,
};
Expand All @@ -2013,7 +2014,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
let mut ret = BasicHasher::<H54Sub<Alloc>> {
GetHasherCommon: self.GetHasherCommon.clone(),
buckets_: H54Sub::<Alloc> {
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.len()),
buckets_: allocate::<u32, _>(m, self.buckets_.len()),
},
h9_opts: self.h9_opts,
};
Expand All @@ -2026,9 +2027,9 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
}
impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc> for H9<Alloc> {
fn clone_with_alloc(&self, m: &mut Alloc) -> Self {
let mut num = <Alloc as Allocator<u16>>::alloc_cell(m, self.num_.len());
let mut num = allocate::<u16, _>(m, self.num_.len());
num.slice_mut().clone_from_slice(self.num_.slice());
let mut buckets = <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.len());
let mut buckets = allocate::<u32, _>(m, self.buckets_.len());
buckets.slice_mut().clone_from_slice(self.buckets_.slice());
H9::<Alloc> {
num_: num,
Expand All @@ -2044,9 +2045,9 @@ impl<
> CloneWithAlloc<Alloc> for AdvHasher<Special, Alloc>
{
fn clone_with_alloc(&self, m: &mut Alloc) -> Self {
let mut num = <Alloc as Allocator<u16>>::alloc_cell(m, self.num.len());
let mut num = allocate::<u16, _>(m, self.num.len());
num.slice_mut().clone_from_slice(self.num.slice());
let mut buckets = <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets.len());
let mut buckets = allocate::<u32, _>(m, self.buckets.len());
buckets.slice_mut().clone_from_slice(self.buckets.slice());
AdvHasher::<Special, Alloc> {
GetHasherCommon: self.GetHasherCommon.clone(),
Expand Down
43 changes: 13 additions & 30 deletions src/enc/backward_references/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use alloc_stdlib::StandardAlloc;
use super::{
AdvHasher, AnyHasher, BrotliHasherParams, CloneWithAlloc, H5Sub, H9Opts, HQ7Sub, Struct1,
};
use crate::enc::combined_alloc::allocate;
use crate::enc::{Allocator, SliceWrapper};

static RANDOM_THEN_UNICODE: &[u8] = include_bytes!("../../../testdata/random_then_unicode"); //&[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55];
Expand All @@ -24,11 +25,8 @@ fn test_bulk_store_range() {
let block_size = 1u64 << params_hasher.block_bits;
let bucket_size = 1u64 << params_hasher.bucket_bits;
let mut alloc = StandardAlloc::default();
let mut buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
let mut num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
let mut buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
let mut num = alloc.alloc_cell(bucket_size as usize);

let mut hasher_a = AdvHasher::<H5Sub, StandardAlloc> {
buckets,
Expand All @@ -47,11 +45,8 @@ fn test_bulk_store_range() {
block_mask_: block_size.wrapping_sub(1) as u32,
},
};
buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
num = alloc.alloc_cell(bucket_size as usize);
let mut hasher_b = hasher_a.clone_with_alloc(&mut alloc);
assert!(hasher_a == hasher_b);
let mut hasher_e = hasher_a.clone_with_alloc(&mut alloc);
Expand Down Expand Up @@ -130,11 +125,8 @@ fn test_bulk_store_range_off_spec() {
let block_size = 1u64 << params_hasher.block_bits;
let bucket_size = 1u64 << params_hasher.bucket_bits;
let mut alloc = StandardAlloc::default();
let mut buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
let mut num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
let mut buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
let mut num = alloc.alloc_cell(bucket_size as usize);

let mut hasher_a = AdvHasher::<H5Sub, StandardAlloc> {
buckets,
Expand All @@ -153,11 +145,8 @@ fn test_bulk_store_range_off_spec() {
block_mask_: block_size.wrapping_sub(1) as u32,
},
};
buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
num = alloc.alloc_cell(bucket_size as usize);
let mut hasher_b = hasher_a.clone_with_alloc(&mut alloc);
assert!(hasher_a == hasher_b);
let mut hasher_c = AdvHasher::<HQ7Sub, StandardAlloc> {
Expand Down Expand Up @@ -218,11 +207,8 @@ fn test_bulk_store_range_pow2() {
let block_size = 1u64 << params_hasher.block_bits;
let bucket_size = 1u64 << params_hasher.bucket_bits;
let mut alloc = StandardAlloc::default();
let mut buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
let mut num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
let mut buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
let mut num = alloc.alloc_cell(bucket_size as usize);

let mut hasher_a = AdvHasher::<H5Sub, StandardAlloc> {
buckets,
Expand All @@ -241,11 +227,8 @@ fn test_bulk_store_range_pow2() {
block_mask_: block_size.wrapping_sub(1) as u32,
},
};
buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
&mut alloc,
(bucket_size * block_size) as usize,
);
num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
num = alloc.alloc_cell(bucket_size as usize);
let mut hasher_b = hasher_a.clone_with_alloc(&mut alloc);
assert!(hasher_a == hasher_b);
let mut hasher_e = hasher_a.clone_with_alloc(&mut alloc);
Expand Down
5 changes: 3 additions & 2 deletions src/enc/block_split.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

use super::super::alloc;
use super::super::alloc::{Allocator, SliceWrapper};
use crate::enc::combined_alloc::alloc_default;

pub struct BlockSplit<Alloc: alloc::Allocator<u8> + alloc::Allocator<u32>> {
pub num_types: usize,
Expand All @@ -15,8 +16,8 @@ impl<Alloc: alloc::Allocator<u8> + alloc::Allocator<u32>> Default for BlockSplit
Self {
num_types: 0,
num_blocks: 0,
types: <Alloc as Allocator<u8>>::AllocatedMemory::default(),
lengths: <Alloc as Allocator<u32>>::AllocatedMemory::default(),
types: alloc_default::<u8, Alloc>(),
lengths: alloc_default::<u32, Alloc>(),
}
}
}
Expand Down
Loading

0 comments on commit 1730d38

Please sign in to comment.