Source code

Revision control

Other Tools

1
/* This Source Code Form is subject to the terms of the Mozilla Public
2
* License, v. 2.0. If a copy of the MPL was not distributed with this
3
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5
use api::{DirtyRect, DocumentId, ExternalImageType, ImageFormat};
6
use api::{DebugFlags, ImageDescriptor};
7
use api::units::*;
8
#[cfg(test)]
9
use api::IdNamespace;
10
use crate::device::{TextureFilter, TextureFormatPair, total_gpu_bytes_allocated};
11
use crate::freelist::{FreeList, FreeListHandle, UpsertResult, WeakFreeListHandle};
12
use crate::gpu_cache::{GpuCache, GpuCacheHandle};
13
use crate::gpu_types::{ImageSource, UvRectKind};
14
use crate::internal_types::{
15
CacheTextureId, FastHashMap, LayerIndex, Swizzle, SwizzleSettings,
16
TextureUpdateList, TextureUpdateSource, TextureSource,
17
TextureCacheAllocInfo, TextureCacheUpdate,
18
};
19
use crate::profiler::{ResourceProfileCounter, TextureCacheProfileCounters};
20
use crate::render_backend::{FrameId, FrameStamp};
21
use crate::resource_cache::{CacheItem, CachedImageData};
22
use smallvec::SmallVec;
23
use std::cell::Cell;
24
use std::cmp;
25
use std::mem;
26
use std::time::{Duration, SystemTime};
27
use std::rc::Rc;
28
29
/// The size of each region/layer in shared cache texture arrays.
30
pub const TEXTURE_REGION_DIMENSIONS: i32 = 512;
31
32
const PICTURE_TEXTURE_ADD_SLICES: usize = 4;
33
34
/// The chosen image format for picture tiles.
35
const PICTURE_TILE_FORMAT: ImageFormat = ImageFormat::RGBA8;
36
37
/// The number of pixels in a region. Derived from the above.
38
const TEXTURE_REGION_PIXELS: usize =
39
(TEXTURE_REGION_DIMENSIONS as usize) * (TEXTURE_REGION_DIMENSIONS as usize);
40
41
// The minimum number of bytes that we must be able to reclaim in order
42
// to justify clearing the entire shared cache in order to shrink it.
43
const RECLAIM_THRESHOLD_BYTES: usize = 16 * 512 * 512 * 4;
44
45
/// Items in the texture cache can either be standalone textures,
46
/// or a sub-rect inside the shared cache.
47
#[derive(Debug)]
48
#[cfg_attr(feature = "capture", derive(Serialize))]
49
#[cfg_attr(feature = "replay", derive(Deserialize))]
50
enum EntryDetails {
51
Standalone,
52
Picture {
53
// Index in the picture_textures array
54
texture_index: usize,
55
// Slice in the texture array
56
layer_index: usize,
57
},
58
Cache {
59
/// Origin within the texture layer where this item exists.
60
origin: DeviceIntPoint,
61
/// The layer index of the texture array.
62
layer_index: usize,
63
},
64
}
65
66
impl EntryDetails {
67
fn describe(&self) -> (LayerIndex, DeviceIntPoint) {
68
match *self {
69
EntryDetails::Standalone => (0, DeviceIntPoint::zero()),
70
EntryDetails::Picture { layer_index, .. } => (layer_index, DeviceIntPoint::zero()),
71
EntryDetails::Cache { origin, layer_index } => (layer_index, origin),
72
}
73
}
74
}
75
76
impl EntryDetails {
77
/// Returns the kind associated with the details.
78
fn kind(&self) -> EntryKind {
79
match *self {
80
EntryDetails::Standalone => EntryKind::Standalone,
81
EntryDetails::Picture { .. } => EntryKind::Picture,
82
EntryDetails::Cache { .. } => EntryKind::Shared,
83
}
84
}
85
}
86
87
/// Tag identifying standalone-versus-shared, without the details.
88
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
89
enum EntryKind {
90
Standalone,
91
Picture,
92
Shared,
93
}
94
95
#[derive(Debug)]
96
pub enum CacheEntryMarker {}
97
98
// Stores information related to a single entry in the texture
99
// cache. This is stored for each item whether it's in the shared
100
// cache or a standalone texture.
101
#[derive(Debug)]
102
#[cfg_attr(feature = "capture", derive(Serialize))]
103
#[cfg_attr(feature = "replay", derive(Deserialize))]
104
struct CacheEntry {
105
/// Size the requested item, in device pixels.
106
size: DeviceIntSize,
107
/// Details specific to standalone or shared items.
108
details: EntryDetails,
109
/// Arbitrary user data associated with this item.
110
user_data: [f32; 3],
111
/// The last frame this item was requested for rendering.
112
last_access: FrameStamp,
113
/// Handle to the resource rect in the GPU cache.
114
uv_rect_handle: GpuCacheHandle,
115
/// Image format of the data that the entry expects.
116
input_format: ImageFormat,
117
filter: TextureFilter,
118
swizzle: Swizzle,
119
/// The actual device texture ID this is part of.
120
texture_id: CacheTextureId,
121
/// Optional notice when the entry is evicted from the cache.
122
eviction_notice: Option<EvictionNotice>,
123
/// The type of UV rect this entry specifies.
124
uv_rect_kind: UvRectKind,
125
/// If set to `Auto` the cache entry may be evicted if unused for a number of frames.
126
eviction: Eviction,
127
}
128
129
impl CacheEntry {
130
// Create a new entry for a standalone texture.
131
fn new_standalone(
132
texture_id: CacheTextureId,
133
last_access: FrameStamp,
134
params: &CacheAllocParams,
135
swizzle: Swizzle,
136
) -> Self {
137
CacheEntry {
138
size: params.descriptor.size,
139
user_data: params.user_data,
140
last_access,
141
details: EntryDetails::Standalone,
142
texture_id,
143
input_format: params.descriptor.format,
144
filter: params.filter,
145
swizzle,
146
uv_rect_handle: GpuCacheHandle::new(),
147
eviction_notice: None,
148
uv_rect_kind: params.uv_rect_kind,
149
eviction: Eviction::Auto,
150
}
151
}
152
153
// Update the GPU cache for this texture cache entry.
154
// This ensures that the UV rect, and texture layer index
155
// are up to date in the GPU cache for vertex shaders
156
// to fetch from.
157
fn update_gpu_cache(&mut self, gpu_cache: &mut GpuCache) {
158
if let Some(mut request) = gpu_cache.request(&mut self.uv_rect_handle) {
159
let (layer_index, origin) = self.details.describe();
160
let image_source = ImageSource {
161
p0: origin.to_f32(),
162
p1: (origin + self.size).to_f32(),
163
texture_layer: layer_index as f32,
164
user_data: self.user_data,
165
uv_rect_kind: self.uv_rect_kind,
166
};
167
image_source.write_gpu_blocks(&mut request);
168
}
169
}
170
171
fn evict(&self) {
172
if let Some(eviction_notice) = self.eviction_notice.as_ref() {
173
eviction_notice.notify();
174
}
175
}
176
177
fn alternative_input_format(&self) -> ImageFormat {
178
match self.input_format {
179
ImageFormat::RGBA8 => ImageFormat::BGRA8,
180
ImageFormat::BGRA8 => ImageFormat::RGBA8,
181
other => other,
182
}
183
}
184
}
185
186
187
/// A texture cache handle is a weak reference to a cache entry.
188
///
189
/// If the handle has not been inserted into the cache yet, or if the entry was
190
/// previously inserted and then evicted, lookup of the handle will fail, and
191
/// the cache handle needs to re-upload this item to the texture cache (see
192
/// request() below).
193
pub type TextureCacheHandle = WeakFreeListHandle<CacheEntryMarker>;
194
195
/// Describes the eviction policy for a given entry in the texture cache.
196
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
197
#[cfg_attr(feature = "capture", derive(Serialize))]
198
#[cfg_attr(feature = "replay", derive(Deserialize))]
199
pub enum Eviction {
200
/// The entry will be evicted under the normal rules (which differ between
201
/// standalone and shared entries).
202
Auto,
203
/// The entry will not be evicted until the policy is explicitly set to a
204
/// different value.
205
Manual,
206
/// The entry will be evicted if it was not used in the last frame.
207
///
208
/// FIXME(bholley): Currently this only applies to the standalone case.
209
Eager,
210
}
211
212
// An eviction notice is a shared condition useful for detecting
213
// when a TextureCacheHandle gets evicted from the TextureCache.
214
// It is optionally installed to the TextureCache when an update()
215
// is scheduled. A single notice may be shared among any number of
216
// TextureCacheHandle updates. The notice may then be subsequently
217
// checked to see if any of the updates using it have been evicted.
218
#[derive(Clone, Debug, Default)]
219
#[cfg_attr(feature = "capture", derive(Serialize))]
220
#[cfg_attr(feature = "replay", derive(Deserialize))]
221
pub struct EvictionNotice {
222
evicted: Rc<Cell<bool>>,
223
}
224
225
impl EvictionNotice {
226
fn notify(&self) {
227
self.evicted.set(true);
228
}
229
230
pub fn check(&self) -> bool {
231
if self.evicted.get() {
232
self.evicted.set(false);
233
true
234
} else {
235
false
236
}
237
}
238
}
239
240
/// A set of lazily allocated, fixed size, texture arrays for each format the
241
/// texture cache supports.
242
#[cfg_attr(feature = "capture", derive(Serialize))]
243
#[cfg_attr(feature = "replay", derive(Deserialize))]
244
struct SharedTextures {
245
array_color8_nearest: TextureArray,
246
array_alpha8_linear: TextureArray,
247
array_alpha16_linear: TextureArray,
248
array_color8_linear: TextureArray,
249
}
250
251
impl SharedTextures {
252
/// Mints a new set of shared textures.
253
fn new(color_formats: TextureFormatPair<ImageFormat>) -> Self {
254
Self {
255
// Used primarily for cached shadow masks. There can be lots of
256
// these on some pages like francine, but most pages don't use it
257
// much.
258
array_alpha8_linear: TextureArray::new(
259
TextureFormatPair::from(ImageFormat::R8),
260
TextureFilter::Linear,
261
4,
262
),
263
// Used for experimental hdr yuv texture support, but not used in
264
// production Firefox.
265
array_alpha16_linear: TextureArray::new(
266
TextureFormatPair::from(ImageFormat::R16),
267
TextureFilter::Linear,
268
1,
269
),
270
// The primary cache for images, glyphs, etc.
271
array_color8_linear: TextureArray::new(
272
color_formats.clone(),
273
TextureFilter::Linear,
274
16,
275
),
276
// Used for image-rendering: crisp. This is mostly favicons, which
277
// are small. Some other images use it too, but those tend to be
278
// larger than 512x512 and thus don't use the shared cache anyway.
279
array_color8_nearest: TextureArray::new(
280
color_formats,
281
TextureFilter::Nearest,
282
1,
283
),
284
}
285
}
286
287
/// Returns the cumulative number of GPU bytes consumed by all the shared textures.
288
fn size_in_bytes(&self) -> usize {
289
self.array_alpha8_linear.size_in_bytes() +
290
self.array_alpha16_linear.size_in_bytes() +
291
self.array_color8_linear.size_in_bytes() +
292
self.array_color8_nearest.size_in_bytes()
293
}
294
295
/// Returns the cumulative number of GPU bytes consumed by empty regions.
296
fn reclaimable_region_bytes(&self) -> usize {
297
self.array_alpha8_linear.reclaimable_region_bytes() +
298
self.array_alpha16_linear.reclaimable_region_bytes() +
299
self.array_color8_linear.reclaimable_region_bytes() +
300
self.array_color8_nearest.reclaimable_region_bytes()
301
}
302
303
/// Clears each texture in the set, with the given set of pending updates.
304
fn clear(&mut self, updates: &mut TextureUpdateList) {
305
self.array_alpha8_linear.clear(updates);
306
self.array_alpha16_linear.clear(updates);
307
self.array_color8_linear.clear(updates);
308
self.array_color8_nearest.clear(updates);
309
}
310
311
/// Returns a mutable borrow for the shared texture array matching the parameters.
312
fn select(
313
&mut self, external_format: ImageFormat, filter: TextureFilter
314
) -> &mut TextureArray {
315
match external_format {
316
ImageFormat::R8 => {
317
assert_eq!(filter, TextureFilter::Linear);
318
&mut self.array_alpha8_linear
319
}
320
ImageFormat::R16 => {
321
assert_eq!(filter, TextureFilter::Linear);
322
&mut self.array_alpha16_linear
323
}
324
ImageFormat::RGBA8 |
325
ImageFormat::BGRA8 => {
326
match filter {
327
TextureFilter::Linear => &mut self.array_color8_linear,
328
TextureFilter::Nearest => &mut self.array_color8_nearest,
329
_ => panic!("Unexpexcted filter {:?}", filter),
330
}
331
}
332
_ => panic!("Unexpected format {:?}", external_format),
333
}
334
}
335
}
336
337
/// The texture arrays used to hold picture cache tiles.
338
#[cfg_attr(feature = "capture", derive(Serialize))]
339
#[cfg_attr(feature = "replay", derive(Deserialize))]
340
struct PictureTextures {
341
textures: Vec<WholeTextureArray>,
342
}
343
344
impl PictureTextures {
345
fn new(
346
initial_window_size: DeviceIntSize,
347
picture_tile_sizes: &[DeviceIntSize],
348
next_texture_id: &mut CacheTextureId,
349
pending_updates: &mut TextureUpdateList,
350
) -> Self {
351
let mut textures = Vec::new();
352
for tile_size in picture_tile_sizes {
353
// TODO(gw): The way initial size is used here may allocate a lot of memory once
354
// we are using multiple slice sizes. Do some measurements once we
355
// have multiple slices here and adjust the calculations as required.
356
let num_x = (initial_window_size.width + tile_size.width - 1) / tile_size.width;
357
let num_y = (initial_window_size.height + tile_size.height - 1) / tile_size.height;
358
let mut slice_count = (num_x * num_y).max(1).min(16) as usize;
359
if slice_count < 4 {
360
// On some platforms we get bogus (1x1) initial window size. The first real frame will then
361
// reallocate many more picture cache slices. Don't bother preallocating in that case.
362
slice_count = 0;
363
}
364
365
if slice_count == 0 {
366
continue;
367
}
368
369
let texture = WholeTextureArray {
370
size: *tile_size,
371
filter: TextureFilter::Nearest,
372
format: PICTURE_TILE_FORMAT,
373
texture_id: *next_texture_id,
374
slices: vec![WholeTextureSlice { uv_rect_handle: None }; slice_count],
375
has_depth: true,
376
};
377
378
next_texture_id.0 += 1;
379
380
pending_updates.push_alloc(texture.texture_id, texture.to_info());
381
382
textures.push(texture);
383
}
384
385
PictureTextures { textures }
386
}
387
388
fn get_or_allocate_tile(
389
&mut self,
390
tile_size: DeviceIntSize,
391
now: FrameStamp,
392
next_texture_id: &mut CacheTextureId,
393
pending_updates: &mut TextureUpdateList,
394
) -> CacheEntry {
395
let texture_index = self.textures
396
.iter()
397
.position(|texture| { texture.size == tile_size })
398
.unwrap_or(self.textures.len());
399
400
if texture_index == self.textures.len() {
401
self.textures.push(WholeTextureArray {
402
size: tile_size,
403
filter: TextureFilter::Nearest,
404
format: PICTURE_TILE_FORMAT,
405
texture_id: *next_texture_id,
406
slices: Vec::new(),
407
has_depth: true,
408
});
409
next_texture_id.0 += 1;
410
}
411
412
let texture = &mut self.textures[texture_index];
413
414
let layer_index = match texture.find_free() {
415
Some(index) => index,
416
None => {
417
let was_empty = texture.slices.is_empty();
418
let index = texture.grow(PICTURE_TEXTURE_ADD_SLICES);
419
let info = texture.to_info();
420
if was_empty {
421
pending_updates.push_alloc(texture.texture_id, info);
422
} else {
423
pending_updates.push_realloc(texture.texture_id, info);
424
}
425
426
index
427
},
428
};
429
430
texture.occupy(texture_index, layer_index, now)
431
}
432
433
fn get(&mut self, index: usize) -> &mut WholeTextureArray {
434
&mut self.textures[index]
435
}
436
437
fn clear(&mut self, pending_updates: &mut TextureUpdateList) {
438
for texture in &mut self.textures {
439
if texture.slices.is_empty() {
440
continue;
441
}
442
443
if let Some(texture_id) = texture.reset(PICTURE_TEXTURE_ADD_SLICES) {
444
pending_updates.push_reset(texture_id, texture.to_info());
445
}
446
}
447
}
448
449
fn update_profile(&self, profile: &mut ResourceProfileCounter) {
450
// For now, this profile counter just accumulates the slices and bytes
451
// from all picture cache texture arrays.
452
let mut picture_slices = 0;
453
let mut picture_bytes = 0;
454
for texture in &self.textures {
455
picture_slices += texture.slices.len();
456
picture_bytes += texture.size_in_bytes();
457
}
458
profile.set(picture_slices, picture_bytes);
459
}
460
461
#[cfg(feature = "replay")]
462
fn tile_sizes(&self) -> Vec<DeviceIntSize> {
463
self.textures.iter().map(|pt| pt.size).collect()
464
}
465
}
466
467
/// Lists of strong handles owned by the texture cache. There is only one strong
468
/// handle for each entry, but unlimited weak handles. Consumers receive the weak
469
/// handles, and `TextureCache` owns the strong handles internally.
470
#[derive(Default, Debug)]
471
#[cfg_attr(feature = "capture", derive(Serialize))]
472
#[cfg_attr(feature = "replay", derive(Deserialize))]
473
struct EntryHandles {
474
/// Handles for each standalone texture cache entry.
475
standalone: Vec<FreeListHandle<CacheEntryMarker>>,
476
/// Handles for each picture cache entry.
477
picture: Vec<FreeListHandle<CacheEntryMarker>>,
478
/// Handles for each shared texture cache entry.
479
shared: Vec<FreeListHandle<CacheEntryMarker>>,
480
}
481
482
impl EntryHandles {
483
/// Mutably borrows the requested handle list.
484
fn select(&mut self, kind: EntryKind) -> &mut Vec<FreeListHandle<CacheEntryMarker>> {
485
match kind {
486
EntryKind::Standalone => &mut self.standalone,
487
EntryKind::Picture => &mut self.picture,
488
EntryKind::Shared => &mut self.shared,
489
}
490
}
491
}
492
493
/// Container struct for the various parameters used in cache allocation.
494
struct CacheAllocParams {
495
descriptor: ImageDescriptor,
496
filter: TextureFilter,
497
user_data: [f32; 3],
498
uv_rect_kind: UvRectKind,
499
}
500
501
/// Criterion to determine whether a cache entry should be evicted. Generated
502
/// with `EvictionThresholdBuilder`.
503
///
504
/// Our eviction scheme is based on the age of the entry, both in terms of
505
/// number of frames and ellapsed time. It does not directly consider the size
506
/// of the entry, but may consider overall memory usage by WebRender, by making
507
/// eviction increasingly aggressive as overall memory usage increases.
508
///
509
/// Note that we don't just wrap a `FrameStamp` here, because `FrameStamp`
510
/// requires that if the id fields are the same, the time fields will be as
511
/// well. The pair of values in our eviction threshold generally do not match
512
/// the stamp of any actual frame, and the comparison semantics are also
513
/// different - so it's best to use a distinct type.
514
#[derive(Clone, Copy)]
515
struct EvictionThreshold {
516
id: FrameId,
517
time: SystemTime,
518
}
519
520
impl EvictionThreshold {
521
/// Returns true if the entry with the given access record should be evicted
522
/// under this threshold.
523
fn should_evict(&self, last_access: FrameStamp) -> bool {
524
last_access.frame_id() < self.id &&
525
last_access.time() < self.time
526
}
527
}
528
529
/// Helper to generate an `EvictionThreshold` with the desired policy.
530
///
531
/// Without any constraints, the builder will generate a threshold that evicts
532
/// all frames other than the current one. Constraints are additive, i.e. setting
533
/// a frame limit and a time limit only evicts frames with an id and time each
534
/// less than the respective limits.
535
struct EvictionThresholdBuilder {
536
now: FrameStamp,
537
max_frames: Option<usize>,
538
max_time_ms: Option<usize>,
539
scale_by_pressure: bool,
540
}
541
542
impl EvictionThresholdBuilder {
543
fn new(now: FrameStamp) -> Self {
544
Self {
545
now,
546
max_frames: None,
547
max_time_ms: None,
548
scale_by_pressure: false,
549
}
550
}
551
552
fn max_frames(mut self, frames: usize) -> Self {
553
self.max_frames = Some(frames);
554
self
555
}
556
557
fn max_time_s(mut self, seconds: usize) -> Self {
558
self.max_time_ms = Some(seconds * 1000);
559
self
560
}
561
562
fn scale_by_pressure(mut self) -> Self {
563
self.scale_by_pressure = true;
564
self
565
}
566
567
fn build(self) -> EvictionThreshold {
568
const MAX_MEMORY_PRESSURE_BYTES: f64 = (300 * 512 * 512 * 4) as f64;
569
// Compute the memory pressure factor in the range of [0, 1.0].
570
let pressure_factor = if self.scale_by_pressure {
571
let bytes_allocated = total_gpu_bytes_allocated() as f64;
572
1.0 - (bytes_allocated / MAX_MEMORY_PRESSURE_BYTES).min(0.98)
573
} else {
574
1.0
575
};
576
577
// Compute the maximum period an entry can go unused before eviction.
578
// If a category (frame or time) wasn't specified, we set the
579
// threshold for that category to |now|, which lets the other category
580
// be the deciding factor. If neither category is specified, we'll evict
581
// everything but the current frame.
582
//
583
// Note that we need to clamp the frame id to avoid it going negative or
584
// matching FrameId::INVALID early in execution. We don't need to clamp
585
// the time because it's unix-epoch-relative.
586
let max_frames = self.max_frames
587
.map(|f| (f as f64 * pressure_factor) as usize)
588
.unwrap_or(0)
589
.min(self.now.frame_id().as_usize() - 1);
590
let max_time_ms = self.max_time_ms
591
.map(|f| (f as f64 * pressure_factor) as usize)
592
.unwrap_or(0) as u64;
593
594
EvictionThreshold {
595
id: self.now.frame_id() - max_frames,
596
time: self.now.time() - Duration::from_millis(max_time_ms),
597
}
598
}
599
}
600
601
#[cfg_attr(feature = "capture", derive(Serialize))]
602
#[cfg_attr(feature = "replay", derive(Deserialize))]
603
pub struct PerDocumentData {
604
/// The last `FrameStamp` in which we expired the shared cache for
605
/// this document.
606
last_shared_cache_expiration: FrameStamp,
607
608
/// Strong handles for all entries that this document has allocated
609
/// from the shared FreeList.
610
handles: EntryHandles,
611
}
612
613
impl PerDocumentData {
614
pub fn new() -> Self {
615
PerDocumentData {
616
last_shared_cache_expiration: FrameStamp::INVALID,
617
handles: EntryHandles::default(),
618
}
619
}
620
}
621
622
/// General-purpose manager for images in GPU memory. This includes images,
623
/// rasterized glyphs, rasterized blobs, cached render tasks, etc.
624
///
625
/// The texture cache is owned and managed by the RenderBackend thread, and
626
/// produces a series of commands to manipulate the textures on the Renderer
627
/// thread. These commands are executed before any rendering is performed for
628
/// a given frame.
629
///
630
/// Entries in the texture cache are not guaranteed to live past the end of the
631
/// frame in which they are requested, and may be evicted. The API supports
632
/// querying whether an entry is still available.
633
///
634
/// The TextureCache is different from the GpuCache in that the former stores
635
/// images, whereas the latter stores data and parameters for use in the shaders.
636
/// This means that the texture cache can be visualized, which is a good way to
637
/// understand how it works. Enabling gfx.webrender.debug.texture-cache shows a
638
/// live view of its contents in Firefox.
639
#[cfg_attr(feature = "capture", derive(Serialize))]
640
#[cfg_attr(feature = "replay", derive(Deserialize))]
641
pub struct TextureCache {
642
/// Set of texture arrays in different formats used for the shared cache.
643
shared_textures: SharedTextures,
644
645
/// A texture array per tile size for picture caching.
646
picture_textures: PictureTextures,
647
648
/// Maximum texture size supported by hardware.
649
max_texture_size: i32,
650
651
/// Maximum number of texture layers supported by hardware.
652
max_texture_layers: usize,
653
654
/// Settings on using texture unit swizzling.
655
swizzle: Option<SwizzleSettings>,
656
657
/// The current set of debug flags.
658
debug_flags: DebugFlags,
659
660
/// The next unused virtual texture ID. Monotonically increasing.
661
next_id: CacheTextureId,
662
663
/// A list of allocations and updates that need to be applied to the texture
664
/// cache in the rendering thread this frame.
665
#[cfg_attr(all(feature = "serde", any(feature = "capture", feature = "replay")), serde(skip))]
666
pending_updates: TextureUpdateList,
667
668
/// The current `FrameStamp`. Used for cache eviction policies.
669
now: FrameStamp,
670
671
/// The time at which we first reached the byte threshold for reclaiming
672
/// cache memory. `None if we haven't reached the threshold.
673
reached_reclaim_threshold: Option<SystemTime>,
674
675
/// Maintains the list of all current items in the texture cache.
676
entries: FreeList<CacheEntry, CacheEntryMarker>,
677
678
/// Holds items that need to be maintained on a per-document basis. If we
679
/// modify this data for a document without also building a frame for that
680
/// document, then we might end up erroneously evicting items out from
681
/// under that document.
682
per_doc_data: FastHashMap<DocumentId, PerDocumentData>,
683
684
/// The current document's data. This is moved out of per_doc_data in
685
/// begin_frame and moved back in end_frame to solve borrow checker issues.
686
/// We should try removing this when we require a rustc with NLL.
687
doc_data: PerDocumentData,
688
689
/// This indicates that we performed a cleanup operation which requires all
690
/// documents to build a frame.
691
require_frame_build: bool,
692
}
693
694
impl TextureCache {
695
pub fn new(
696
max_texture_size: i32,
697
mut max_texture_layers: usize,
698
picture_tile_sizes: &[DeviceIntSize],
699
initial_size: DeviceIntSize,
700
color_formats: TextureFormatPair<ImageFormat>,
701
swizzle: Option<SwizzleSettings>,
702
) -> Self {
703
// On MBP integrated Intel GPUs, texture arrays appear to be
704
// implemented as a single texture of stacked layers, and that
705
// texture appears to be subject to the texture size limit. As such,
706
// allocating more than 32 512x512 regions results in a dimension
707
// longer than 16k (the max texture size), causing incorrect behavior.
708
//
709
// So we clamp the number of layers on mac. This results in maximum
710
// texture array size of 32MB, which isn't ideal but isn't terrible
711
// either. OpenGL on mac is not long for this earth, so this may be
712
// good enough until we have WebRender on gfx-rs (on Metal).
713
//
714
// On all platforms, we also clamp the number of textures per layer to 16
715
// to avoid the cost of resizing large texture arrays (at the expense
716
// of batching efficiency).
717
//
718
// Note that we could also define this more generally in terms of
719
// |max_texture_size / TEXTURE_REGION_DIMENSION|, except:
720
// * max_texture_size is actually clamped beyond the device limit
721
// by Gecko to 8192, so we'd need to thread the raw device value
722
// here, and:
723
// * The bug we're working around is likely specific to a single
724
// driver family, and those drivers are also likely to share
725
// the same max texture size of 16k. If we do encounter a driver
726
// with the same bug but a lower max texture size, we might need
727
// to rethink our strategy anyway, since a limit below 32MB might
728
// start to introduce performance issues.
729
max_texture_layers = max_texture_layers.min(16);
730
731
let mut pending_updates = TextureUpdateList::new();
732
733
// Shared texture cache controls swizzling on a per-entry basis, assuming that
734
// the texture as a whole doesn't need to be swizzled (but only some entries do).
735
// It would be possible to support this, but not needed at the moment.
736
assert!(color_formats.internal != ImageFormat::BGRA8 ||
737
swizzle.map_or(true, |s| s.bgra8_sampling_swizzle == Swizzle::default())
738
);
739
740
let mut next_texture_id = CacheTextureId(1);
741
742
TextureCache {
743
shared_textures: SharedTextures::new(color_formats),
744
picture_textures: PictureTextures::new(
745
initial_size,
746
picture_tile_sizes,
747
&mut next_texture_id,
748
&mut pending_updates,
749
),
750
reached_reclaim_threshold: None,
751
entries: FreeList::new(),
752
max_texture_size,
753
max_texture_layers,
754
swizzle,
755
debug_flags: DebugFlags::empty(),
756
next_id: next_texture_id,
757
pending_updates,
758
now: FrameStamp::INVALID,
759
per_doc_data: FastHashMap::default(),
760
doc_data: PerDocumentData::new(),
761
require_frame_build: false,
762
}
763
}
764
765
/// Creates a TextureCache and sets it up with a valid `FrameStamp`, which
766
/// is useful for avoiding panics when instantiating the `TextureCache`
767
/// directly from unit test code.
768
#[cfg(test)]
769
pub fn new_for_testing(
770
max_texture_size: i32,
771
max_texture_layers: usize,
772
image_format: ImageFormat,
773
) -> Self {
774
let mut cache = Self::new(
775
max_texture_size,
776
max_texture_layers,
777
&[],
778
DeviceIntSize::zero(),
779
TextureFormatPair::from(image_format),
780
None,
781
);
782
let mut now = FrameStamp::first(DocumentId::new(IdNamespace(1), 1));
783
now.advance();
784
cache.begin_frame(now);
785
cache
786
}
787
788
pub fn set_debug_flags(&mut self, flags: DebugFlags) {
789
self.debug_flags = flags;
790
}
791
792
/// Clear all entries of the specified kind.
793
fn clear_kind(&mut self, kind: EntryKind) {
794
let mut per_doc_data = mem::replace(&mut self.per_doc_data, FastHashMap::default());
795
for (&_, doc_data) in per_doc_data.iter_mut() {
796
let entry_handles = mem::replace(
797
doc_data.handles.select(kind),
798
Vec::new(),
799
);
800
801
for handle in entry_handles {
802
let entry = self.entries.free(handle);
803
entry.evict();
804
self.free(&entry);
805
}
806
}
807
808
self.pending_updates.note_clear();
809
self.per_doc_data = per_doc_data;
810
self.require_frame_build = true;
811
}
812
813
fn clear_standalone(&mut self) {
814
debug_assert!(!self.now.is_valid());
815
self.clear_kind(EntryKind::Standalone);
816
}
817
818
fn clear_picture(&mut self) {
819
self.clear_kind(EntryKind::Picture);
820
self.picture_textures.clear(&mut self.pending_updates);
821
}
822
823
fn clear_shared(&mut self) {
824
self.unset_doc_data();
825
self.clear_kind(EntryKind::Shared);
826
self.shared_textures.clear(&mut self.pending_updates);
827
self.set_doc_data();
828
}
829
830
/// Clear all entries in the texture cache. This is a fairly drastic
831
/// step that should only be called very rarely.
832
pub fn clear_all(&mut self) {
833
self.clear_standalone();
834
self.clear_picture();
835
self.clear_shared();
836
}
837
838
fn set_doc_data(&mut self) {
839
let document_id = self.now.document_id();
840
self.doc_data = self.per_doc_data
841
.remove(&document_id)
842
.unwrap_or_else(PerDocumentData::new);
843
}
844
845
fn unset_doc_data(&mut self) {
846
self.per_doc_data.insert(self.now.document_id(),
847
mem::replace(&mut self.doc_data, PerDocumentData::new()));
848
}
849
850
pub fn prepare_for_frames(&mut self, time: SystemTime) {
851
self.maybe_reclaim_shared_memory(time);
852
}
853
854
pub fn bookkeep_after_frames(&mut self) {
855
self.require_frame_build = false;
856
}
857
858
pub fn requires_frame_build(&self) -> bool {
859
self.require_frame_build
860
}
861
862
/// Called at the beginning of each frame.
863
pub fn begin_frame(&mut self, stamp: FrameStamp) {
864
debug_assert!(!self.now.is_valid());
865
self.now = stamp;
866
self.set_doc_data();
867
self.maybe_do_periodic_gc();
868
}
869
870
fn maybe_reclaim_shared_memory(&mut self, time: SystemTime) {
871
// If we've had a sufficient number of unused layers for a sufficiently
872
// long time, just blow the whole cache away to shrink it.
873
//
874
// We could do this more intelligently with a resize+blit, but that would
875
// add complexity for a rare case.
876
//
877
// This function must be called before the first begin_frame() for a group
878
// of documents, otherwise documents could end up ignoring the
879
// self.require_frame_build flag which is set if we end up calling
880
// clear_shared.
881
debug_assert!(!self.now.is_valid());
882
if self.shared_textures.reclaimable_region_bytes() >= RECLAIM_THRESHOLD_BYTES {
883
self.reached_reclaim_threshold.get_or_insert(time);
884
} else {
885
self.reached_reclaim_threshold = None;
886
}
887
if let Some(t) = self.reached_reclaim_threshold {
888
let dur = time.duration_since(t).unwrap_or_default();
889
if dur >= Duration::from_secs(5) {
890
self.clear_shared();
891
self.reached_reclaim_threshold = None;
892
}
893
}
894
}
895
896
/// Called at the beginning of each frame to periodically GC by expiring
897
/// old shared entries. If necessary, the shared memory opened up as a
898
/// result of expiring these entries will be reclaimed before the next
899
/// group of document frames.
900
fn maybe_do_periodic_gc(&mut self) {
901
debug_assert!(self.now.is_valid());
902
903
// Normally the shared cache only gets GCed when we fail to allocate.
904
// However, we also perform a periodic, conservative GC to ensure that
905
// we recover unused memory in bounded time, rather than having it
906
// depend on allocation patterns of subsequent content.
907
let time_since_last_gc = self.now.time()
908
.duration_since(self.doc_data.last_shared_cache_expiration.time())
909
.unwrap_or_default();
910
let do_periodic_gc = time_since_last_gc >= Duration::from_secs(5) &&
911
self.shared_textures.size_in_bytes() >= RECLAIM_THRESHOLD_BYTES * 2;
912
if do_periodic_gc {
913
let threshold = EvictionThresholdBuilder::new(self.now)
914
.max_frames(1)
915
.max_time_s(10)
916
.build();
917
self.maybe_expire_old_shared_entries(threshold);
918
}
919
}
920
921
pub fn end_frame(&mut self, texture_cache_profile: &mut TextureCacheProfileCounters) {
922
debug_assert!(self.now.is_valid());
923
// Expire standalone entries.
924
//
925
// Most of the time, standalone cache entries correspond to images whose
926
// width or height is greater than the region size in the shared cache, i.e.
927
// 512 pixels. Cached render tasks also frequently get standalone entries,
928
// but those use the Eviction::Eager policy (for now). So the tradeoff there
929
// is largely around reducing texture upload jank while keeping memory usage
930
// at an acceptable level.
931
let threshold = self.default_eviction();
932
self.expire_old_entries(EntryKind::Standalone, threshold);
933
self.expire_old_entries(EntryKind::Picture, threshold);
934
935
self.shared_textures.array_alpha8_linear.release_empty_textures(&mut self.pending_updates);
936
self.shared_textures.array_alpha16_linear.release_empty_textures(&mut self.pending_updates);
937
self.shared_textures.array_color8_linear.release_empty_textures(&mut self.pending_updates);
938
self.shared_textures.array_color8_nearest.release_empty_textures(&mut self.pending_updates);
939
940
self.shared_textures.array_alpha8_linear
941
.update_profile(&mut texture_cache_profile.pages_alpha8_linear);
942
self.shared_textures.array_alpha16_linear
943
.update_profile(&mut texture_cache_profile.pages_alpha16_linear);
944
self.shared_textures.array_color8_linear
945
.update_profile(&mut texture_cache_profile.pages_color8_linear);
946
self.shared_textures.array_color8_nearest
947
.update_profile(&mut texture_cache_profile.pages_color8_nearest);
948
self.picture_textures
949
.update_profile(&mut texture_cache_profile.pages_picture);
950
951
self.unset_doc_data();
952
self.now = FrameStamp::INVALID;
953
}
954
955
// Request an item in the texture cache. All images that will
956
// be used on a frame *must* have request() called on their
957
// handle, to update the last used timestamp and ensure
958
// that resources are not flushed from the cache too early.
959
//
960
// Returns true if the image needs to be uploaded to the
961
// texture cache (either never uploaded, or has been
962
// evicted on a previous frame).
963
pub fn request(&mut self, handle: &TextureCacheHandle, gpu_cache: &mut GpuCache) -> bool {
964
match self.entries.get_opt_mut(handle) {
965
// If an image is requested that is already in the cache,
966
// refresh the GPU cache data associated with this item.
967
Some(entry) => {
968
entry.last_access = self.now;
969
entry.update_gpu_cache(gpu_cache);
970
false
971
}
972
None => true,
973
}
974
}
975
976
// Returns true if the image needs to be uploaded to the
977
// texture cache (either never uploaded, or has been
978
// evicted on a previous frame).
979
pub fn needs_upload(&self, handle: &TextureCacheHandle) -> bool {
980
self.entries.get_opt(handle).is_none()
981
}
982
983
pub fn max_texture_size(&self) -> i32 {
984
self.max_texture_size
985
}
986
987
#[cfg(feature = "replay")]
988
pub fn max_texture_layers(&self) -> usize {
989
self.max_texture_layers
990
}
991
992
#[cfg(feature = "replay")]
993
pub fn picture_tile_sizes(&self) -> Vec<DeviceIntSize> {
994
self.picture_textures.tile_sizes()
995
}
996
997
#[cfg(feature = "replay")]
998
pub fn color_formats(&self) -> TextureFormatPair<ImageFormat> {
999
self.shared_textures.array_color8_linear.formats.clone()
1000
}
1001
1002
#[cfg(feature = "replay")]
1003
pub fn swizzle_settings(&self) -> Option<SwizzleSettings> {
1004
self.swizzle
1005
}
1006
1007
pub fn pending_updates(&mut self) -> TextureUpdateList {
1008
mem::replace(&mut self.pending_updates, TextureUpdateList::new())
1009
}
1010
1011
// Update the data stored by a given texture cache handle.
1012
pub fn update(
1013
&mut self,
1014
handle: &mut TextureCacheHandle,
1015
descriptor: ImageDescriptor,
1016
filter: TextureFilter,
1017
data: Option<CachedImageData>,
1018
user_data: [f32; 3],
1019
mut dirty_rect: ImageDirtyRect,
1020
gpu_cache: &mut GpuCache,
1021
eviction_notice: Option<&EvictionNotice>,
1022
uv_rect_kind: UvRectKind,
1023
eviction: Eviction,
1024
) {
1025
debug_assert!(self.now.is_valid());
1026
1027
// Determine if we need to allocate texture cache memory
1028
// for this item. We need to reallocate if any of the following
1029
// is true:
1030
// - Never been in the cache
1031
// - Has been in the cache but was evicted.
1032
// - Exists in the cache but dimensions / format have changed.
1033
let realloc = match self.entries.get_opt(handle) {
1034
Some(entry) => {
1035
entry.size != descriptor.size || (entry.input_format != descriptor.format &&
1036
entry.alternative_input_format() != descriptor.format)
1037
}
1038
None => {
1039
// Not allocated, or was previously allocated but has been evicted.
1040
true
1041
}
1042
};
1043
1044
if realloc {
1045
let params = CacheAllocParams { descriptor, filter, user_data, uv_rect_kind };
1046
self.allocate(&params, handle);
1047
1048
// If we reallocated, we need to upload the whole item again.
1049
dirty_rect = DirtyRect::All;
1050
}
1051
1052
let entry = self.entries.get_opt_mut(handle)
1053
.expect("BUG: handle must be valid now");
1054
1055
// Install the new eviction notice for this update, if applicable.
1056
entry.eviction_notice = eviction_notice.cloned();
1057
entry.uv_rect_kind = uv_rect_kind;
1058
1059
// Invalidate the contents of the resource rect in the GPU cache.
1060
// This ensures that the update_gpu_cache below will add
1061
// the new information to the GPU cache.
1062
//TODO: only invalidate if the parameters change?
1063
gpu_cache.invalidate(&entry.uv_rect_handle);
1064
1065
// Upload the resource rect and texture array layer.
1066
entry.update_gpu_cache(gpu_cache);
1067
1068
entry.eviction = eviction;
1069
1070
// Create an update command, which the render thread processes
1071
// to upload the new image data into the correct location
1072
// in GPU memory.
1073
if let Some(data) = data {
1074
// If the swizzling is supported, we always upload in the internal
1075
// texture format (thus avoiding the conversion by the driver).
1076
// Otherwise, pass the external format to the driver.
1077
let use_upload_format = self.swizzle.is_none();
1078
let (layer_index, origin) = entry.details.describe();
1079
let op = TextureCacheUpdate::new_update(
1080
data,
1081
&descriptor,
1082
origin,
1083
entry.size,
1084
layer_index as i32,
1085
use_upload_format,
1086
&dirty_rect,
1087
);
1088
self.pending_updates.push_update(entry.texture_id, op);
1089
}
1090
}
1091
1092
// Check if a given texture handle has a valid allocation
1093
// in the texture cache.
1094
pub fn is_allocated(&self, handle: &TextureCacheHandle) -> bool {
1095
self.entries.get_opt(handle).is_some()
1096
}
1097
1098
// Check if a given texture handle was last used as recently
1099
// as the specified number of previous frames.
1100
pub fn is_recently_used(&self, handle: &TextureCacheHandle, margin: usize) -> bool {
1101
self.entries.get_opt(handle).map_or(false, |entry| {
1102
entry.last_access.frame_id() + margin >= self.now.frame_id()
1103
})
1104
}
1105
1106
// Return the allocated size of the texture handle's associated data,
1107
// or otherwise indicate the handle is invalid.
1108
pub fn get_allocated_size(&self, handle: &TextureCacheHandle) -> Option<usize> {
1109
self.entries.get_opt(handle).map(|entry| {
1110
(entry.input_format.bytes_per_pixel() * entry.size.area()) as usize
1111
})
1112
}
1113
1114
// Retrieve the details of an item in the cache. This is used
1115
// during batch creation to provide the resource rect address
1116
// to the shaders and texture ID to the batching logic.
1117
// This function will assert in debug modes if the caller
1118
// tries to get a handle that was not requested this frame.
1119
pub fn get(&self, handle: &TextureCacheHandle) -> CacheItem {
1120
let (texture_id, layer_index, uv_rect, swizzle, uv_rect_handle) = self.get_cache_location(handle);
1121
CacheItem {
1122
uv_rect_handle,
1123
texture_id: TextureSource::TextureCache(texture_id, swizzle),
1124
uv_rect,
1125
texture_layer: layer_index as i32,
1126
}
1127
}
1128
1129
/// A more detailed version of get(). This allows access to the actual
1130
/// device rect of the cache allocation.
1131
///
1132
/// Returns a tuple identifying the texture, the layer, the region,
1133
/// and its GPU handle.
1134
pub fn get_cache_location(
1135
&self,
1136
handle: &TextureCacheHandle,
1137
) -> (CacheTextureId, LayerIndex, DeviceIntRect, Swizzle, GpuCacheHandle) {
1138
let entry = self.entries
1139
.get_opt(handle)
1140
.expect("BUG: was dropped from cache or not updated!");
1141
debug_assert_eq!(entry.last_access, self.now);
1142
let (layer_index, origin) = entry.details.describe();
1143
(entry.texture_id,
1144
layer_index as usize,
1145
DeviceIntRect::new(origin, entry.size),
1146
entry.swizzle,
1147
entry.uv_rect_handle)
1148
}
1149
1150
pub fn mark_unused(&mut self, handle: &TextureCacheHandle) {
1151
if let Some(entry) = self.entries.get_opt_mut(handle) {
1152
// Set last accessed stamp invalid to ensure it gets cleaned up
1153
// next time we expire entries.
1154
entry.last_access = FrameStamp::INVALID;
1155
entry.eviction = Eviction::Auto;
1156
}
1157
}
1158
1159
/// Returns the default eviction policy.
1160
///
1161
/// These parameters come from very rough instrumentation of hits in the
1162
/// shared cache, with simple browsing on a few pages. In rough terms, more
1163
/// than 99.5% of cache hits occur for entries that were used in the previous
1164
/// frame. This is obviously the dominant case, but we still want good behavior
1165
/// in long-tail cases (i.e. a large image is scrolled off-screen and on again).
1166
/// If we exclude immediately-reused (first frame) entries, 70% of the remaining
1167
/// hits happen within the first 200 frames. So we can be relatively agressive
1168
/// about eviction without sacrificing much in terms of cache performance.
1169
/// The one wrinkle is that animation-heavy pages do tend to extend the
1170
/// distribution, presumably because they churn through FrameIds faster than
1171
/// their more-static counterparts. As such, we _also_ provide a time floor
1172
/// (which was not measured with the same degree of rigour).
1173
fn default_eviction(&self) -> EvictionThreshold {
1174
EvictionThresholdBuilder::new(self.now)
1175
.max_frames(200)
1176
.max_time_s(2)
1177
.scale_by_pressure()
1178
.build()
1179
}
1180
1181
/// Shared eviction code for standalone and shared entries.
1182
///
1183
/// See `EvictionThreshold` for more details on policy.
1184
fn expire_old_entries(&mut self, kind: EntryKind, threshold: EvictionThreshold) {
1185
debug_assert!(self.now.is_valid());
1186
// Iterate over the entries in reverse order, evicting the ones older than
1187
// the frame age threshold. Reverse order avoids iterator invalidation when
1188
// removing entries.
1189
for i in (0..self.doc_data.handles.select(kind).len()).rev() {
1190
let evict = {
1191
let entry = self.entries.get(&self.doc_data.handles.select(kind)[i]);
1192
match entry.eviction {
1193
Eviction::Manual => false,
1194
Eviction::Auto => threshold.should_evict(entry.last_access),
1195
Eviction::Eager => {
1196
// Texture cache entries can be evicted at the start of
1197
// a frame, or at any time during the frame when a cache
1198
// allocation is occurring. This means that entries tagged
1199
// with eager eviction may get evicted before they have a
1200
// chance to be requested on the current frame. Instead,
1201
// advance the frame id of the entry by one before
1202
// comparison. This ensures that an eager entry will
1203
// not be evicted until it is not used for at least
1204
// one complete frame.
1205
let mut entry_frame_id = entry.last_access.frame_id();
1206
entry_frame_id.advance();
1207
1208
entry_frame_id < self.now.frame_id()
1209
}
1210
}
1211
};
1212
if evict {
1213
let handle = self.doc_data.handles.select(kind).swap_remove(i);
1214
let entry = self.entries.free(handle);
1215
entry.evict();
1216
self.free(&entry);
1217
}
1218
}
1219
}
1220
1221
/// Expires old shared entries, if we haven't done so this frame.
1222
///
1223
/// Returns true if any entries were expired.
1224
fn maybe_expire_old_shared_entries(&mut self, threshold: EvictionThreshold) -> bool {
1225
debug_assert!(self.now.is_valid());
1226
let old_len = self.doc_data.handles.shared.len();
1227
if self.doc_data.last_shared_cache_expiration.frame_id() < self.now.frame_id() {
1228
self.expire_old_entries(EntryKind::Shared, threshold);
1229
self.doc_data.last_shared_cache_expiration = self.now;
1230
}
1231
self.doc_data.handles.shared.len() != old_len
1232
}
1233
1234
// Free a cache entry from the standalone list or shared cache.
1235
fn free(&mut self, entry: &CacheEntry) {
1236
match entry.details {
1237
EntryDetails::Picture { texture_index, layer_index } => {
1238
let picture_texture = self.picture_textures.get(texture_index);
1239
picture_texture.slices[layer_index].uv_rect_handle = None;
1240
if self.debug_flags.contains(
1241
DebugFlags::TEXTURE_CACHE_DBG |
1242
DebugFlags::TEXTURE_CACHE_DBG_CLEAR_EVICTED)
1243
{
1244
self.pending_updates.push_debug_clear(
1245
entry.texture_id,
1246
DeviceIntPoint::zero(),
1247
picture_texture.size.width,
1248
picture_texture.size.height,
1249
layer_index,
1250
);
1251
}
1252
}
1253
EntryDetails::Standalone => {
1254
// This is a standalone texture allocation. Free it directly.
1255
self.pending_updates.push_free(entry.texture_id);
1256
}
1257
EntryDetails::Cache { origin, layer_index } => {
1258
// Free the block in the given region.
1259
let texture_array = self.shared_textures.select(entry.input_format, entry.filter);
1260
let unit = texture_array.units
1261
.iter_mut()
1262
.find(|unit| unit.texture_id == entry.texture_id)
1263
.expect("Unable to find the associated texture array unit");
1264
let region = &mut unit.regions[layer_index];
1265
1266
if self.debug_flags.contains(
1267
DebugFlags::TEXTURE_CACHE_DBG |
1268
DebugFlags::TEXTURE_CACHE_DBG_CLEAR_EVICTED)
1269
{
1270
self.pending_updates.push_debug_clear(
1271
entry.texture_id,
1272
origin,
1273
region.slab_size.width,
1274
region.slab_size.height,
1275
layer_index,
1276
);
1277
}
1278
region.free(origin, &mut unit.empty_regions);
1279
}
1280
}
1281
}
1282
1283
/// Check if we can allocate this entry without growing any of the texture cache arrays.
1284
fn has_space_in_shared_cache(
1285
&mut self,
1286
params: &CacheAllocParams,
1287
) -> bool {
1288
let texture_array = self.shared_textures.select(
1289
params.descriptor.format,
1290
params.filter,
1291
);
1292
let slab_size = SlabSize::new(params.descriptor.size);
1293
texture_array.units
1294
.iter()
1295
.any(|unit| unit.can_alloc(slab_size))
1296
}
1297
1298
/// Allocate a block from the shared cache.
1299
fn allocate_from_shared_cache(
1300
&mut self,
1301
params: &CacheAllocParams,
1302
) -> CacheEntry {
1303
// Mutably borrow the correct texture.
1304
let texture_array = self.shared_textures.select(
1305
params.descriptor.format,
1306
params.filter,
1307
);
1308
let swizzle = if texture_array.formats.external == params.descriptor.format {
1309
Swizzle::default()
1310
} else {
1311
match self.swizzle {
1312
Some(_) => Swizzle::Bgra,
1313
None => Swizzle::default(),
1314
}
1315
};
1316
1317
let max_texture_layers = self.max_texture_layers;
1318
let slab_size = SlabSize::new(params.descriptor.size);
1319
1320
let mut info = TextureCacheAllocInfo {
1321
width: TEXTURE_REGION_DIMENSIONS,
1322
height: TEXTURE_REGION_DIMENSIONS,
1323
format: texture_array.formats.internal,
1324
filter: texture_array.filter,
1325
layer_count: 1,
1326
is_shared_cache: true,
1327
has_depth: false,
1328
};
1329
1330
let unit_index = if let Some(index) = texture_array.units
1331
.iter()
1332
.position(|unit| unit.can_alloc(slab_size))
1333
{
1334
index
1335
} else if let Some(index) = texture_array.units
1336
.iter()
1337
.position(|unit| unit.regions.len() < max_texture_layers)
1338
{
1339
let unit = &mut texture_array.units[index];
1340
1341
unit.push_regions(texture_array.layers_per_allocation);
1342
1343
info.layer_count = unit.regions.len() as i32;
1344
self.pending_updates.push_realloc(unit.texture_id, info);
1345
1346
index
1347
} else {
1348
let index = texture_array.units.len();
1349
texture_array.units.push(TextureArrayUnit {
1350
texture_id: self.next_id,
1351
regions: Vec::new(),
1352
empty_regions: 0,
1353
});
1354
1355
let unit = &mut texture_array.units[index];
1356
1357
unit.push_regions(texture_array.layers_per_allocation);
1358
1359
info.layer_count = unit.regions.len() as i32;
1360
self.pending_updates.push_alloc(self.next_id, info);
1361
self.next_id.0 += 1;
1362
index
1363
};
1364
1365
// Do the allocation. This can fail and return None
1366
// if there are no free slots or regions available.
1367
texture_array.alloc(params, unit_index, self.now, swizzle)
1368
}
1369
1370
// Returns true if the given image descriptor *may* be
1371
// placed in the shared texture cache.
1372
pub fn is_allowed_in_shared_cache(
1373
&self,
1374
filter: TextureFilter,
1375
descriptor: &ImageDescriptor,
1376
) -> bool {
1377
let mut allowed_in_shared_cache = true;
1378
1379
// Anything larger than TEXTURE_REGION_DIMENSIONS goes in a standalone texture.
1380
// TODO(gw): If we find pages that suffer from batch breaks in this
1381
// case, add support for storing these in a standalone
1382
// texture array.
1383
if descriptor.size.width > TEXTURE_REGION_DIMENSIONS ||
1384
descriptor.size.height > TEXTURE_REGION_DIMENSIONS
1385
{
1386
allowed_in_shared_cache = false;
1387
}
1388
1389
// TODO(gw): For now, alpha formats of the texture cache can only be linearly sampled.
1390
// Nearest sampling gets a standalone texture.
1391
// This is probably rare enough that it can be fixed up later.
1392
if filter == TextureFilter::Nearest &&
1393
descriptor.format.bytes_per_pixel() <= 2
1394
{
1395
allowed_in_shared_cache = false;
1396
}
1397
1398
allowed_in_shared_cache
1399
}
1400
1401
/// Allocates a new standalone cache entry.
1402
fn allocate_standalone_entry(
1403
&mut self,
1404
params: &CacheAllocParams,
1405
) -> CacheEntry {
1406
let texture_id = self.next_id;
1407
self.next_id.0 += 1;
1408
1409
// Push a command to allocate device storage of the right size / format.
1410
let info = TextureCacheAllocInfo {
1411
width: params.descriptor.size.width,
1412
height: params.descriptor.size.height,
1413
format: params.descriptor.format,
1414
filter: params.filter,
1415
layer_count: 1,
1416
is_shared_cache: false,
1417
has_depth: false,
1418
};
1419
self.pending_updates.push_alloc(texture_id, info);
1420
1421
// Special handing for BGRA8 textures that may need to be swizzled.
1422
let swizzle = if params.descriptor.format == ImageFormat::BGRA8 {
1423
self.swizzle.map(|s| s.bgra8_sampling_swizzle)
1424
} else {
1425
None
1426
};
1427
1428
CacheEntry::new_standalone(
1429
texture_id,
1430
self.now,
1431
params,
1432
swizzle.unwrap_or_default(),
1433
)
1434
}
1435
1436
/// Allocates a cache entry appropriate for the given parameters.
1437
///
1438
/// This allocates from the shared cache unless the parameters do not meet
1439
/// the shared cache requirements, in which case a standalone texture is
1440
/// used.
1441
fn allocate_cache_entry(
1442
&mut self,
1443
params: &CacheAllocParams,
1444
) -> CacheEntry {
1445
assert!(!params.descriptor.size.is_empty_or_negative());
1446
1447
// If this image doesn't qualify to go in the shared (batching) cache,
1448
// allocate a standalone entry.
1449
if self.is_allowed_in_shared_cache(params.filter, &params.descriptor) {
1450
if !self.has_space_in_shared_cache(params) {
1451
// If we don't have extra space and haven't GCed this frame, do so.
1452
let threshold = self.default_eviction();
1453
self.maybe_expire_old_shared_entries(threshold);
1454
}
1455
self.allocate_from_shared_cache(params)
1456
} else {
1457
self.allocate_standalone_entry(params)
1458
}
1459
}
1460
1461
fn upsert_entry(
1462
&mut self,
1463
cache_entry: CacheEntry,
1464
handle: &mut TextureCacheHandle,
1465
) {
1466
let new_kind = cache_entry.details.kind();
1467
// If the handle points to a valid cache entry, we want to replace the
1468
// cache entry with our newly updated location. We also need to ensure
1469
// that the storage (region or standalone) associated with the previous
1470
// entry here gets freed.
1471
//
1472
// If the handle is invalid, we need to insert the data, and append the
1473
// result to the corresponding vector.
1474
//
1475
// This is managed with a database style upsert operation.
1476
match self.entries.upsert(handle, cache_entry) {
1477
UpsertResult::Updated(old_entry) => {
1478
if new_kind != old_entry.details.kind() {
1479
// Handle the rare case than an update moves an entry from
1480
// shared to standalone or vice versa. This involves a linear
1481
// search, but should be rare enough not to matter.
1482
let (from, to) = match new_kind {
1483
EntryKind::Standalone =>
1484
(&mut self.doc_data.handles.shared, &mut self.doc_data.handles.standalone),
1485
EntryKind::Picture => unreachable!(),
1486
EntryKind::Shared =>
1487
(&mut self.doc_data.handles.standalone, &mut self.doc_data.handles.shared),
1488
};
1489
let idx = from.iter().position(|h| h.weak() == *handle).unwrap();
1490
to.push(from.remove(idx));
1491
}
1492
self.free(&old_entry);
1493
}
1494
UpsertResult::Inserted(new_handle) => {
1495
*handle = new_handle.weak();
1496
self.doc_data.handles.select(new_kind).push(new_handle);
1497
}
1498
}
1499
}
1500
1501
/// Allocates a cache entry for the given parameters, and updates the
1502
/// provided handle to point to the new entry.
1503
fn allocate(&mut self, params: &CacheAllocParams, handle: &mut TextureCacheHandle) {
1504
debug_assert!(self.now.is_valid());
1505
let new_cache_entry = self.allocate_cache_entry(params);
1506
self.upsert_entry(new_cache_entry, handle)
1507
}
1508
1509
// Update the data stored by a given texture cache handle for picture caching specifically.
1510
pub fn update_picture_cache(
1511
&mut self,
1512
tile_size: DeviceIntSize,
1513
handle: &mut TextureCacheHandle,
1514
gpu_cache: &mut GpuCache,
1515
) {
1516
debug_assert!(self.now.is_valid());
1517
debug_assert!(tile_size.width > 0 && tile_size.height > 0);
1518
1519
if self.entries.get_opt(handle).is_none() {
1520
let cache_entry = self.picture_textures.get_or_allocate_tile(
1521
tile_size,
1522
self.now,
1523
&mut self.next_id,
1524
&mut self.pending_updates,
1525
);
1526
1527
self.upsert_entry(cache_entry, handle)
1528
}
1529
1530
// Upload the resource rect and texture array layer.
1531
self.entries
1532
.get_opt_mut(handle)
1533
.expect("BUG: handle must be valid now")
1534
.update_gpu_cache(gpu_cache);
1535
}
1536
1537
pub fn shared_alpha_expected_format(&self) -> ImageFormat {
1538
self.shared_textures.array_alpha8_linear.formats.external
1539
}
1540
1541
pub fn shared_color_expected_format(&self) -> ImageFormat {
1542
self.shared_textures.array_color8_linear.formats.external
1543
}
1544
}
1545
1546
#[cfg_attr(feature = "capture", derive(Serialize))]
1547
#[cfg_attr(feature = "replay", derive(Deserialize))]
1548
#[derive(Copy, Clone, PartialEq)]
1549
struct SlabSize {
1550
width: i32,
1551
height: i32,
1552
}
1553
1554
impl SlabSize {
1555
fn new(size: DeviceIntSize) -> Self {
1556
let x_size = quantize_dimension(size.width);
1557
let y_size = quantize_dimension(size.height);
1558
1559
assert!(x_size > 0 && x_size <= TEXTURE_REGION_DIMENSIONS);
1560
assert!(y_size > 0 && y_size <= TEXTURE_REGION_DIMENSIONS);
1561
1562
let (width, height) = match (x_size, y_size) {
1563
// Special cased rectangular slab pages.
1564
(512, 0..=64) => (512, 64),
1565
(512, 128) => (512, 128),
1566
(512, 256) => (512, 256),
1567
(0..=64, 512) => (64, 512),
1568
(128, 512) => (128, 512),
1569
(256, 512) => (256, 512),
1570
1571
// If none of those fit, use a square slab size.
1572
(x_size, y_size) => {
1573
let square_size = cmp::max(x_size, y_size);
1574
(square_size, square_size)
1575
}
1576
};
1577
1578
SlabSize {
1579
width,
1580
height,
1581
}
1582
}
1583
1584
fn invalid() -> SlabSize {
1585
SlabSize {
1586
width: 0,
1587
height: 0,
1588
}
1589
}
1590
}
1591
1592
// The x/y location within a texture region of an allocation.
1593
#[cfg_attr(feature = "capture", derive(Serialize))]
1594
#[cfg_attr(feature = "replay", derive(Deserialize))]
1595
struct TextureLocation(u8, u8);
1596
1597
impl TextureLocation {
1598
fn new(x: i32, y: i32) -> Self {
1599
debug_assert!(x >= 0 && y >= 0 && x < 0x100 && y < 0x100);
1600
TextureLocation(x as u8, y as u8)
1601
}
1602
}
1603
1604
/// A region corresponds to a layer in a shared cache texture.
1605
///
1606
/// All allocations within a region are of the same size.
1607
#[cfg_attr(feature = "capture", derive(Serialize))]
1608
#[cfg_attr(feature = "replay", derive(Deserialize))]
1609
struct TextureRegion {
1610
layer_index: usize,
1611
slab_size: SlabSize,
1612
free_slots: Vec<TextureLocation>,
1613
total_slot_count: usize,
1614
}
1615
1616
impl TextureRegion {
1617
fn new(layer_index: usize) -> Self {
1618
TextureRegion {
1619
layer_index,
1620
slab_size: SlabSize::invalid(),
1621
free_slots: Vec::new(),
1622
total_slot_count: 0,
1623
}
1624
}
1625
1626
// Initialize a region to be an allocator for a specific slab size.
1627
fn init(&mut self, slab_size: SlabSize, empty_regions: &mut usize) {
1628
debug_assert!(self.slab_size == SlabSize::invalid());
1629
debug_assert!(self.free_slots.is_empty());
1630
1631
self.slab_size = slab_size;
1632
let slots_per_x_axis = TEXTURE_REGION_DIMENSIONS / self.slab_size.width;
1633
let slots_per_y_axis = TEXTURE_REGION_DIMENSIONS / self.slab_size.height;
1634
1635
// Add each block to a freelist.
1636
for y in 0 .. slots_per_y_axis {
1637
for x in 0 .. slots_per_x_axis {
1638
self.free_slots.push(TextureLocation::new(x, y));
1639
}
1640
}
1641
1642
self.total_slot_count = self.free_slots.len();
1643
*empty_regions -= 1;
1644
}
1645
1646
// Deinit a region, allowing it to become a region with
1647
// a different allocator size.
1648
fn deinit(&mut self, empty_regions: &mut usize) {
1649
self.slab_size = SlabSize::invalid();
1650
self.free_slots.clear();
1651
self.total_slot_count = 0;
1652
*empty_regions += 1;
1653
}
1654
1655
fn is_empty(&self) -> bool {
1656
self.slab_size == SlabSize::invalid()
1657
}
1658
1659
// Attempt to allocate a fixed size block from this region.
1660
fn alloc(&mut self) -> Option<DeviceIntPoint> {
1661
debug_assert!(self.slab_size != SlabSize::invalid());
1662
1663
self.free_slots.pop().map(|location| {
1664
DeviceIntPoint::new(
1665
self.slab_size.width * location.0 as i32,
1666
self.slab_size.height * location.1 as i32,
1667
)
1668
})
1669
}
1670
1671
// Free a block in this region.
1672
fn free(&mut self, point: DeviceIntPoint, empty_regions: &mut usize) {
1673
let x = point.x / self.slab_size.width;
1674
let y = point.y / self.slab_size.height;
1675
self.free_slots.push(TextureLocation::new(x, y));
1676
1677
// If this region is completely unused, deinit it
1678
// so that it can become a different slab size
1679
// as required.
1680
if self.free_slots.len() == self.total_slot_count {
1681
self.deinit(empty_regions);
1682
}
1683
}
1684
}
1685
1686
#[cfg_attr(feature = "capture", derive(Serialize))]
1687
#[cfg_attr(feature = "replay", derive(Deserialize))]
1688
struct TextureArrayUnit {
1689
texture_id: CacheTextureId,
1690
regions: Vec<TextureRegion>,
1691
empty_regions: usize,
1692
}
1693
1694
impl TextureArrayUnit {
1695
/// Adds a new empty region to the array.
1696
fn push_regions(&mut self, count: i32) {
1697
assert!(self.empty_regions <= self.regions.len());
1698
for _ in 0..count {
1699
let index = self.regions.len();
1700
self.regions.push(TextureRegion::new(index));
1701
self.empty_regions += 1;
1702
}
1703
}
1704
1705
/// Returns true if we can allocate the given entry.
1706
fn can_alloc(&self, slab_size: SlabSize) -> bool {
1707
self.empty_regions != 0 || self.regions.iter().any(|region| {
1708
region.slab_size == slab_size && !region.free_slots.is_empty()
1709
})
1710
}
1711
1712
fn is_empty(&self) -> bool {
1713
self.empty_regions == self.regions.len()
1714
}
1715
}
1716
1717
/// A texture array contains a number of textures, each with a number of
1718
/// layers, where each layer contains a region that can act as a slab allocator.
1719
#[cfg_attr(feature = "capture", derive(Serialize))]
1720
#[cfg_attr(feature = "replay", derive(Deserialize))]
1721
struct TextureArray {
1722
filter: TextureFilter,
1723
formats: TextureFormatPair<ImageFormat>,
1724
units: SmallVec<[TextureArrayUnit; 1]>,
1725
layers_per_allocation: i32,
1726
}
1727
1728
impl TextureArray {
1729
fn new(
1730
formats: TextureFormatPair<ImageFormat>,
1731
filter: TextureFilter,
1732
layers_per_allocation: i32,
1733
) -> Self {
1734
TextureArray {
1735
formats,
1736
filter,
1737
units: SmallVec::new(),
1738
layers_per_allocation,
1739
}
1740
}
1741
1742
/// Returns the number of GPU bytes consumed by this texture array.
1743
fn size_in_bytes(&self) -> usize {
1744
let bpp = self.formats.internal.bytes_per_pixel() as usize;
1745
let num_regions: usize = self.units.iter().map(|u| u.regions.len()).sum();
1746
num_regions * TEXTURE_REGION_PIXELS * bpp
1747
}
1748
1749
/// Returns the number of GPU bytes consumed by empty regions.
1750
fn reclaimable_region_bytes(&self) -> usize {
1751
let bpp = self.formats.internal.bytes_per_pixel() as usize;
1752
let empty_regions: usize = self.units.iter().map(|u| u.empty_regions).sum();
1753
empty_regions * TEXTURE_REGION_PIXELS * bpp
1754
}
1755
1756
fn clear(&mut self, updates: &mut TextureUpdateList) {
1757
for unit in self.units.drain() {
1758
updates.push_free(unit.texture_id);
1759
}
1760
}
1761
1762
fn release_empty_textures(&mut self, updates: &mut TextureUpdateList) {
1763
self.units.retain(|unit| {
1764
if unit.is_empty() {
1765
updates.push_free(unit.texture_id);
1766
1767
false
1768
} else {
1769
true
1770
}
1771
});
1772
}
1773
1774
fn update_profile(&self, counter: &mut ResourceProfileCounter) {
1775
let num_regions: usize = self.units.iter().map(|u| u.regions.len()).sum();
1776
counter.set(num_regions, self.size_in_bytes());
1777
}
1778
1779
/// Allocate space in this texture array.
1780
fn alloc(
1781
&mut self,
1782
params: &CacheAllocParams,
1783
unit_index: usize,
1784
now: FrameStamp,
1785
swizzle: Swizzle,
1786
) -> CacheEntry {
1787
// Quantize the size of the allocation to select a region to
1788
// allocate from.
1789
let slab_size = SlabSize::new(params.descriptor.size);
1790
let unit = &mut self.units[unit_index];
1791
1792
// TODO(gw): For simplicity, the initial implementation just
1793
// has a single vec<> of regions. We could easily
1794
// make this more efficient by storing a list of
1795
// regions for each slab size specifically...
1796
1797
// Keep track of the location of an empty region,
1798
// in case we need to select a new empty region
1799
// after the loop.
1800
let mut empty_region_index = None;
1801
let mut entry_details = None;
1802
1803
// Run through the existing regions of this size, and see if
1804
// we can find a free block in any of them.
1805
for (i, region) in unit.regions.iter_mut().enumerate() {
1806
if region.is_empty() {
1807
empty_region_index = Some(i);
1808
} else if region.slab_size == slab_size {
1809
if let Some(location) = region.alloc() {
1810
entry_details = Some(EntryDetails::Cache {
1811
layer_index: region.layer_index,
1812
origin: location,
1813
});
1814
break;
1815
}