Skip to content

Commit 1730d38

Browse files
danielrhdanielrh
authored
Merge pull request #207 from nyurik/alloc-util
Simplify mem cell alloc usage
2 parents 859f722 + 6b01faf commit 1730d38

20 files changed

+260
-413
lines changed

‎src/bin/validate.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ pub fn compress_validate<InputType: Read, OutputType: Write>(
135135
num_threads: usize,
136136
) -> Result<(), io::Error> {
137137
let mut m8 = HeapAllocator::default();
138-
let buffer = <HeapAllocator as Allocator<u8>>::alloc_cell(&mut m8, buffer_size);
138+
let buffer = m8.alloc_cell(buffer_size);
139139
// FIXME: could reuse the dictionary to seed the compressor, but that violates the abstraction right now
140140
// also dictionaries are not very popular since they are mostly an internal concept, given their deprecation in
141141
// the standard brotli spec

‎src/enc/backward_references/benchmark.rs‎

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,11 @@ fn make_generic_hasher() -> AdvHasher<H5Sub, StandardAlloc> {
2929
let block_size = 1u64 << params_hasher.block_bits;
3030
let bucket_size = 1u64 << params_hasher.bucket_bits;
3131
let mut alloc = StandardAlloc::default();
32-
let buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
33-
&mut alloc,
34-
(bucket_size * block_size) as usize,
35-
);
36-
let num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
3732

3833
AdvHasher::<H5Sub, StandardAlloc> {
39-
buckets: buckets,
34+
buckets: alloc.alloc_cell((bucket_size * block_size) as usize),
4035
h9_opts: H9Opts::new(&params_hasher),
41-
num: num,
36+
num: alloc.alloc_cell(bucket_size as usize),
4237
GetHasherCommon: Struct1 {
4338
params: params_hasher,
4439
is_prepared_: 1,
@@ -65,16 +60,11 @@ fn make_specialized_hasher() -> AdvHasher<HQ7Sub, StandardAlloc> {
6560
let block_size = 1u64 << params_hasher.block_bits;
6661
let bucket_size = 1u64 << params_hasher.bucket_bits;
6762
let mut alloc = StandardAlloc::default();
68-
let buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
69-
&mut alloc,
70-
(bucket_size * block_size) as usize,
71-
);
72-
let num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
7363

7464
AdvHasher::<HQ7Sub, StandardAlloc> {
75-
buckets: buckets,
65+
buckets: alloc.alloc_cell((bucket_size * block_size) as usize),
7666
h9_opts: H9Opts::new(&params_hasher),
77-
num: num,
67+
num: alloc.alloc_cell(bucket_size as usize),
7868
GetHasherCommon: Struct1 {
7969
params: params_hasher,
8070
is_prepared_: 1,

‎src/enc/backward_references/hash_to_binary_tree.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use super::{
88
kHashMul32, AnyHasher, BrotliEncoderParams, CloneWithAlloc, H9Opts, HasherSearchResult,
99
HowPrepared, Struct1,
1010
};
11+
use crate::enc::combined_alloc::allocate;
1112
use crate::enc::static_dict::{
1213
BrotliDictionary, FindMatchLengthWithLimit, BROTLI_UNALIGNED_LOAD32,
1314
};
@@ -206,7 +207,7 @@ where
206207
common: self.common.clone(),
207208
buckets_: Buckets::new_uninit(m),
208209
invalid_pos_: self.invalid_pos_,
209-
forest: <Alloc as Allocator<u32>>::alloc_cell(m, self.forest.len()),
210+
forest: allocate::<u32, _>(m, self.forest.len()),
210211
_params: core::marker::PhantomData::<Params>,
211212
};
212213
ret.buckets_

‎src/enc/backward_references/hq.rs‎

Lines changed: 16 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use super::{
1212
kDistanceCacheIndex, kDistanceCacheOffset, kHashMul32, kInvalidMatch, AnyHasher,
1313
BrotliEncoderParams,
1414
};
15+
use crate::enc::combined_alloc::{alloc_if, alloc_or_default};
1516
use crate::enc::command::{
1617
combine_length_codes, BrotliDistanceParams, Command, GetCopyLengthCode, GetInsertLengthCode,
1718
PrefixEncodeCopyDistance,
@@ -209,16 +210,14 @@ impl<AllocF: Allocator<floatX>> ZopfliCostModel<AllocF> {
209210
num_bytes_: num_bytes,
210211
cost_cmd_: [0.0; 704],
211212
min_cost_cmd_: 0.0,
212-
literal_costs_: if num_bytes.wrapping_add(2) > 0usize {
213-
m.alloc_cell(num_bytes.wrapping_add(2))
214-
} else {
215-
AllocF::AllocatedMemory::default()
216-
},
217-
cost_dist_: if dist.alphabet_size > 0u32 {
218-
m.alloc_cell(num_bytes.wrapping_add(dist.alphabet_size as usize))
219-
} else {
220-
AllocF::AllocatedMemory::default()
221-
},
213+
// FIXME: makes little sense to test if N+2 > 0 -- always true unless wrapping. Perhaps use allocate() instead?
214+
literal_costs_: alloc_or_default::<floatX, _>(m, num_bytes + 2),
215+
// FIXME: possible bug because allocation size is different from the condition
216+
cost_dist_: alloc_if::<floatX, _>(
217+
dist.alphabet_size > 0,
218+
m,
219+
num_bytes + dist.alphabet_size as usize,
220+
),
222221
distance_histogram_size: min(dist.alphabet_size, 544),
223222
}
224223
}
@@ -988,12 +987,8 @@ pub fn BrotliCreateZopfliBackwardReferences<
988987
Buckets: PartialEq<Buckets>,
989988
{
990989
let max_backward_limit: usize = (1usize << params.lgwin).wrapping_sub(16);
991-
let mut nodes: <Alloc as Allocator<ZopfliNode>>::AllocatedMemory;
992-
nodes = if num_bytes.wrapping_add(1) > 0usize {
993-
<Alloc as Allocator<ZopfliNode>>::alloc_cell(alloc, num_bytes.wrapping_add(1))
994-
} else {
995-
<Alloc as Allocator<ZopfliNode>>::AllocatedMemory::default()
996-
};
990+
// FIXME: makes little sense to test if N+1 > 0 -- always true unless wrapping. Perhaps use allocate() instead?
991+
let mut nodes = alloc_or_default::<ZopfliNode, _>(alloc, num_bytes + 1);
997992
if !(0i32 == 0) {
998993
return;
999994
}
@@ -1244,11 +1239,7 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
12441239
Buckets: PartialEq<Buckets>,
12451240
{
12461241
let max_backward_limit: usize = (1usize << params.lgwin).wrapping_sub(16);
1247-
let mut num_matches: <Alloc as Allocator<u32>>::AllocatedMemory = if num_bytes > 0usize {
1248-
<Alloc as Allocator<u32>>::alloc_cell(alloc, num_bytes)
1249-
} else {
1250-
<Alloc as Allocator<u32>>::AllocatedMemory::default()
1251-
};
1242+
let mut num_matches = alloc_or_default::<u32, _>(alloc, num_bytes);
12521243
let mut matches_size: usize = (4usize).wrapping_mul(num_bytes);
12531244
let store_end: usize = if num_bytes >= STORE_LOOKAHEAD_H_10 {
12541245
position
@@ -1264,12 +1255,7 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
12641255
let mut orig_dist_cache = [0i32; 4];
12651256

12661257
let mut model: ZopfliCostModel<Alloc>;
1267-
let mut nodes: <Alloc as Allocator<ZopfliNode>>::AllocatedMemory;
1268-
let mut matches: <Alloc as Allocator<u64>>::AllocatedMemory = if matches_size > 0usize {
1269-
<Alloc as Allocator<u64>>::alloc_cell(alloc, matches_size)
1270-
} else {
1271-
<Alloc as Allocator<u64>>::AllocatedMemory::default()
1272-
};
1258+
let mut matches = alloc_or_default::<u64, _>(alloc, matches_size);
12731259
let gap: usize = 0usize;
12741260
let shadow_matches: usize = 0usize;
12751261
i = 0usize;
@@ -1287,15 +1273,10 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
12871273
} else {
12881274
matches_size
12891275
};
1290-
let mut new_array: <Alloc as Allocator<u64>>::AllocatedMemory;
12911276
while new_size < cur_match_pos.wrapping_add(128).wrapping_add(shadow_matches) {
12921277
new_size = new_size.wrapping_mul(2);
12931278
}
1294-
new_array = if new_size > 0usize {
1295-
<Alloc as Allocator<u64>>::alloc_cell(alloc, new_size)
1296-
} else {
1297-
<Alloc as Allocator<u64>>::AllocatedMemory::default()
1298-
};
1279+
let mut new_array = alloc_or_default::<u64, _>(alloc, new_size);
12991280
if matches_size != 0 {
13001281
for (dst, src) in new_array
13011282
.slice_mut()
@@ -1380,11 +1361,8 @@ pub fn BrotliCreateHqZopfliBackwardReferences<
13801361
*i = *j;
13811362
}
13821363
let orig_num_commands: usize = *num_commands;
1383-
nodes = if num_bytes.wrapping_add(1) > 0usize {
1384-
<Alloc as Allocator<ZopfliNode>>::alloc_cell(alloc, num_bytes.wrapping_add(1))
1385-
} else {
1386-
<Alloc as Allocator<ZopfliNode>>::AllocatedMemory::default()
1387-
};
1364+
// FIXME: makes little sense to test if N+1 > 0 -- always true unless wrapping. Perhaps use allocate() instead?
1365+
let mut nodes = alloc_or_default::<ZopfliNode, _>(alloc, num_bytes + 1);
13881366
if !(0i32 == 0) {
13891367
return;
13901368
}

‎src/enc/backward_references/mod.rs‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ use super::static_dict::{
1616
BROTLI_UNALIGNED_LOAD32, BROTLI_UNALIGNED_LOAD64,
1717
};
1818
use super::util::{floatX, Log2FloorNonZero};
19+
use crate::enc::combined_alloc::allocate;
1920

2021
static kBrotliMinWindowBits: i32 = 10;
2122
static kBrotliMaxWindowBits: i32 = 24;
@@ -1958,8 +1959,8 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
19581959
fn clone_with_alloc(&self, m: &mut Alloc) -> Self {
19591960
let mut ret = BasicHasher::<H2Sub<Alloc>> {
19601961
GetHasherCommon: self.GetHasherCommon.clone(),
1961-
buckets_: H2Sub::<Alloc> {
1962-
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.buckets_.len()),
1962+
buckets_: H2Sub {
1963+
buckets_: allocate::<u32, _>(m, self.buckets_.buckets_.len()),
19631964
},
19641965
h9_opts: self.h9_opts,
19651966
};
@@ -1977,7 +1978,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
19771978
let mut ret = BasicHasher::<H3Sub<Alloc>> {
19781979
GetHasherCommon: self.GetHasherCommon.clone(),
19791980
buckets_: H3Sub::<Alloc> {
1980-
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.buckets_.len()),
1981+
buckets_: allocate::<u32, _>(m, self.buckets_.buckets_.len()),
19811982
},
19821983
h9_opts: self.h9_opts,
19831984
};
@@ -1995,7 +1996,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
19951996
let mut ret = BasicHasher::<H4Sub<Alloc>> {
19961997
GetHasherCommon: self.GetHasherCommon.clone(),
19971998
buckets_: H4Sub::<Alloc> {
1998-
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.buckets_.len()),
1999+
buckets_: allocate::<u32, _>(m, self.buckets_.buckets_.len()),
19992000
},
20002001
h9_opts: self.h9_opts,
20012002
};
@@ -2013,7 +2014,7 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
20132014
let mut ret = BasicHasher::<H54Sub<Alloc>> {
20142015
GetHasherCommon: self.GetHasherCommon.clone(),
20152016
buckets_: H54Sub::<Alloc> {
2016-
buckets_: <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.len()),
2017+
buckets_: allocate::<u32, _>(m, self.buckets_.len()),
20172018
},
20182019
h9_opts: self.h9_opts,
20192020
};
@@ -2026,9 +2027,9 @@ impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc>
20262027
}
20272028
impl<Alloc: alloc::Allocator<u16> + alloc::Allocator<u32>> CloneWithAlloc<Alloc> for H9<Alloc> {
20282029
fn clone_with_alloc(&self, m: &mut Alloc) -> Self {
2029-
let mut num = <Alloc as Allocator<u16>>::alloc_cell(m, self.num_.len());
2030+
let mut num = allocate::<u16, _>(m, self.num_.len());
20302031
num.slice_mut().clone_from_slice(self.num_.slice());
2031-
let mut buckets = <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets_.len());
2032+
let mut buckets = allocate::<u32, _>(m, self.buckets_.len());
20322033
buckets.slice_mut().clone_from_slice(self.buckets_.slice());
20332034
H9::<Alloc> {
20342035
num_: num,
@@ -2044,9 +2045,9 @@ impl<
20442045
> CloneWithAlloc<Alloc> for AdvHasher<Special, Alloc>
20452046
{
20462047
fn clone_with_alloc(&self, m: &mut Alloc) -> Self {
2047-
let mut num = <Alloc as Allocator<u16>>::alloc_cell(m, self.num.len());
2048+
let mut num = allocate::<u16, _>(m, self.num.len());
20482049
num.slice_mut().clone_from_slice(self.num.slice());
2049-
let mut buckets = <Alloc as Allocator<u32>>::alloc_cell(m, self.buckets.len());
2050+
let mut buckets = allocate::<u32, _>(m, self.buckets.len());
20502051
buckets.slice_mut().clone_from_slice(self.buckets.slice());
20512052
AdvHasher::<Special, Alloc> {
20522053
GetHasherCommon: self.GetHasherCommon.clone(),

‎src/enc/backward_references/test.rs‎

Lines changed: 13 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use alloc_stdlib::StandardAlloc;
66
use super::{
77
AdvHasher, AnyHasher, BrotliHasherParams, CloneWithAlloc, H5Sub, H9Opts, HQ7Sub, Struct1,
88
};
9+
use crate::enc::combined_alloc::allocate;
910
use crate::enc::{Allocator, SliceWrapper};
1011

1112
static RANDOM_THEN_UNICODE: &[u8] = include_bytes!("../../../testdata/random_then_unicode"); //&[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55];
@@ -24,11 +25,8 @@ fn test_bulk_store_range() {
2425
let block_size = 1u64 << params_hasher.block_bits;
2526
let bucket_size = 1u64 << params_hasher.bucket_bits;
2627
let mut alloc = StandardAlloc::default();
27-
let mut buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
28-
&mut alloc,
29-
(bucket_size * block_size) as usize,
30-
);
31-
let mut num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
28+
let mut buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
29+
let mut num = alloc.alloc_cell(bucket_size as usize);
3230

3331
let mut hasher_a = AdvHasher::<H5Sub, StandardAlloc> {
3432
buckets,
@@ -47,11 +45,8 @@ fn test_bulk_store_range() {
4745
block_mask_: block_size.wrapping_sub(1) as u32,
4846
},
4947
};
50-
buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
51-
&mut alloc,
52-
(bucket_size * block_size) as usize,
53-
);
54-
num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
48+
buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
49+
num = alloc.alloc_cell(bucket_size as usize);
5550
let mut hasher_b = hasher_a.clone_with_alloc(&mut alloc);
5651
assert!(hasher_a == hasher_b);
5752
let mut hasher_e = hasher_a.clone_with_alloc(&mut alloc);
@@ -130,11 +125,8 @@ fn test_bulk_store_range_off_spec() {
130125
let block_size = 1u64 << params_hasher.block_bits;
131126
let bucket_size = 1u64 << params_hasher.bucket_bits;
132127
let mut alloc = StandardAlloc::default();
133-
let mut buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
134-
&mut alloc,
135-
(bucket_size * block_size) as usize,
136-
);
137-
let mut num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
128+
let mut buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
129+
let mut num = alloc.alloc_cell(bucket_size as usize);
138130

139131
let mut hasher_a = AdvHasher::<H5Sub, StandardAlloc> {
140132
buckets,
@@ -153,11 +145,8 @@ fn test_bulk_store_range_off_spec() {
153145
block_mask_: block_size.wrapping_sub(1) as u32,
154146
},
155147
};
156-
buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
157-
&mut alloc,
158-
(bucket_size * block_size) as usize,
159-
);
160-
num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
148+
buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
149+
num = alloc.alloc_cell(bucket_size as usize);
161150
let mut hasher_b = hasher_a.clone_with_alloc(&mut alloc);
162151
assert!(hasher_a == hasher_b);
163152
let mut hasher_c = AdvHasher::<HQ7Sub, StandardAlloc> {
@@ -218,11 +207,8 @@ fn test_bulk_store_range_pow2() {
218207
let block_size = 1u64 << params_hasher.block_bits;
219208
let bucket_size = 1u64 << params_hasher.bucket_bits;
220209
let mut alloc = StandardAlloc::default();
221-
let mut buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
222-
&mut alloc,
223-
(bucket_size * block_size) as usize,
224-
);
225-
let mut num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
210+
let mut buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
211+
let mut num = alloc.alloc_cell(bucket_size as usize);
226212

227213
let mut hasher_a = AdvHasher::<H5Sub, StandardAlloc> {
228214
buckets,
@@ -241,11 +227,8 @@ fn test_bulk_store_range_pow2() {
241227
block_mask_: block_size.wrapping_sub(1) as u32,
242228
},
243229
};
244-
buckets = <StandardAlloc as Allocator<u32>>::alloc_cell(
245-
&mut alloc,
246-
(bucket_size * block_size) as usize,
247-
);
248-
num = <StandardAlloc as Allocator<u16>>::alloc_cell(&mut alloc, bucket_size as usize);
230+
buckets = allocate::<u32, _>(&mut alloc, (bucket_size * block_size) as usize);
231+
num = alloc.alloc_cell(bucket_size as usize);
249232
let mut hasher_b = hasher_a.clone_with_alloc(&mut alloc);
250233
assert!(hasher_a == hasher_b);
251234
let mut hasher_e = hasher_a.clone_with_alloc(&mut alloc);

‎src/enc/block_split.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
use super::super::alloc;
44
use super::super::alloc::{Allocator, SliceWrapper};
5+
use crate::enc::combined_alloc::alloc_default;
56

67
pub struct BlockSplit<Alloc: alloc::Allocator<u8> + alloc::Allocator<u32>> {
78
pub num_types: usize,
@@ -15,8 +16,8 @@ impl<Alloc: alloc::Allocator<u8> + alloc::Allocator<u32>> Default for BlockSplit
1516
Self {
1617
num_types: 0,
1718
num_blocks: 0,
18-
types: <Alloc as Allocator<u8>>::AllocatedMemory::default(),
19-
lengths: <Alloc as Allocator<u32>>::AllocatedMemory::default(),
19+
types: alloc_default::<u8, Alloc>(),
20+
lengths: alloc_default::<u32, Alloc>(),
2021
}
2122
}
2223
}

0 commit comments

Comments
 (0)