1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
//! Track which values are live in a block with instruction granularity.
//!
//! The `LiveValueTracker` keeps track of the set of live SSA values at each instruction in a block.
//! The sets of live values are computed on the fly as the tracker is moved from instruction to
//! instruction, starting at the block header.

use crate::dominator_tree::DominatorTree;
use crate::entity::{EntityList, ListPool};
use crate::fx::FxHashMap;
use crate::ir::{Block, DataFlowGraph, ExpandedProgramPoint, Inst, Layout, Value};
use crate::partition_slice::partition_slice;
use crate::regalloc::affinity::Affinity;
use crate::regalloc::liveness::Liveness;
use crate::regalloc::liverange::LiveRange;
use alloc::vec::Vec;

type ValueList = EntityList<Value>;

/// Compute and track live values throughout a block.
pub struct LiveValueTracker {
    /// The set of values that are live at the current program point.
    live: LiveValueVec,

    /// Saved set of live values for every jump and branch that can potentially be an immediate
    /// dominator of a block.
    ///
    /// This is the set of values that are live *before* the branch.
    idom_sets: FxHashMap<Inst, ValueList>,

    /// Memory pool for the live sets.
    idom_pool: ListPool<Value>,
}

/// Information about a value that is live at the current program point.
#[derive(Debug)]
pub struct LiveValue {
    /// The live value.
    pub value: Value,

    /// The local ending point of the live range in the current block, as returned by
    /// `LiveRange::def_local_end()` or `LiveRange::livein_local_end()`.
    pub endpoint: Inst,

    /// The affinity of the value as represented in its `LiveRange`.
    ///
    /// This value is simply a copy of the affinity stored in the live range. We copy it because
    /// almost all users of `LiveValue` need to look at it.
    pub affinity: Affinity,

    /// The live range for this value never leaves its block.
    pub is_local: bool,

    /// This value is dead - the live range ends immediately.
    pub is_dead: bool,
}

struct LiveValueVec {
    /// The set of values that are live at the current program point.
    values: Vec<LiveValue>,

    /// How many values at the front of `values` are known to be live after `inst`?
    ///
    /// This is used to pass a much smaller slice to `partition_slice` when its called a second
    /// time for the same instruction.
    live_prefix: Option<(Inst, usize)>,
}

impl LiveValueVec {
    fn new() -> Self {
        Self {
            values: Vec::new(),
            live_prefix: None,
        }
    }

    /// Add a new live value to `values`. Copy some properties from `lr`.
    fn push(&mut self, value: Value, endpoint: Inst, lr: &LiveRange) {
        self.values.push(LiveValue {
            value,
            endpoint,
            affinity: lr.affinity,
            is_local: lr.is_local(),
            is_dead: lr.is_dead(),
        });
    }

    /// Remove all elements.
    fn clear(&mut self) {
        self.values.clear();
        self.live_prefix = None;
    }

    /// Make sure that the values killed by `next_inst` are moved to the end of the `values`
    /// vector.
    ///
    /// Returns the number of values that will be live after `next_inst`.
    fn live_after(&mut self, next_inst: Inst) -> usize {
        // How many values at the front of the vector are already known to survive `next_inst`?
        // We don't need to pass this prefix to `partition_slice()`
        let keep = match self.live_prefix {
            Some((i, prefix)) if i == next_inst => prefix,
            _ => 0,
        };

        // Move the remaining surviving values to the front partition of the vector.
        let prefix = keep + partition_slice(&mut self.values[keep..], |v| v.endpoint != next_inst);

        // Remember the new prefix length in case we get called again for the same `next_inst`.
        self.live_prefix = Some((next_inst, prefix));
        prefix
    }

    /// Remove the values killed by `next_inst`.
    fn remove_kill_values(&mut self, next_inst: Inst) {
        let keep = self.live_after(next_inst);
        self.values.truncate(keep);
    }

    /// Remove any dead values.
    fn remove_dead_values(&mut self) {
        self.values.retain(|v| !v.is_dead);
        self.live_prefix = None;
    }
}

impl LiveValueTracker {
    /// Create a new blank tracker.
    pub fn new() -> Self {
        Self {
            live: LiveValueVec::new(),
            idom_sets: FxHashMap(),
            idom_pool: ListPool::new(),
        }
    }

    /// Clear all cached information.
    pub fn clear(&mut self) {
        self.live.clear();
        self.idom_sets.clear();
        self.idom_pool.clear();
    }

    /// Get the set of currently live values.
    ///
    /// Between calls to `process_inst()` and `drop_dead()`, this includes both values killed and
    /// defined by the current instruction.
    pub fn live(&self) -> &[LiveValue] {
        &self.live.values
    }

    /// Get a mutable set of currently live values.
    ///
    /// Use with care and don't move entries around.
    pub fn live_mut(&mut self) -> &mut [LiveValue] {
        &mut self.live.values
    }

    /// Move the current position to the top of `block`.
    ///
    /// This depends on the stored live value set at `block`'s immediate dominator, so that must have
    /// been visited first.
    ///
    /// Returns `(liveins, args)` as a pair of slices. The first slice is the set of live-in values
    /// from the immediate dominator. The second slice is the set of `block` parameters.
    ///
    /// Dead parameters with no uses are included in `args`. Call `drop_dead_args()` to remove them.
    pub fn block_top(
        &mut self,
        block: Block,
        dfg: &DataFlowGraph,
        liveness: &Liveness,
        layout: &Layout,
        domtree: &DominatorTree,
    ) -> (&[LiveValue], &[LiveValue]) {
        // Start over, compute the set of live values at the top of the block from two sources:
        //
        // 1. Values that were live before `block`'s immediate dominator, filtered for those that are
        //    actually live-in.
        // 2. Arguments to `block` that are not dead.
        //
        self.live.clear();

        // Compute the live-in values. Start by filtering the set of values that were live before
        // the immediate dominator. Just use the empty set if there's no immediate dominator (i.e.,
        // the entry block or an unreachable block).
        if let Some(idom) = domtree.idom(block) {
            // If the immediate dominator exits, we must have a stored list for it. This is a
            // requirement to the order blocks are visited: All dominators must have been processed
            // before the current block.
            let idom_live_list = self
                .idom_sets
                .get(&idom)
                .expect("No stored live set for dominator");
            // Get just the values that are live-in to `block`.
            for &value in idom_live_list.as_slice(&self.idom_pool) {
                let lr = liveness
                    .get(value)
                    .expect("Immediate dominator value has no live range");

                // Check if this value is live-in here.
                if let Some(endpoint) = lr.livein_local_end(block, layout) {
                    self.live.push(value, endpoint, lr);
                }
            }
        }

        // Now add all the live parameters to `block`.
        let first_arg = self.live.values.len();
        for &value in dfg.block_params(block) {
            let lr = &liveness[value];
            debug_assert_eq!(lr.def(), block.into());
            match lr.def_local_end().into() {
                ExpandedProgramPoint::Inst(endpoint) => {
                    self.live.push(value, endpoint, lr);
                }
                ExpandedProgramPoint::Block(local_block) => {
                    // This is a dead block parameter which is not even live into the first
                    // instruction in the block.
                    debug_assert_eq!(
                        local_block, block,
                        "block parameter live range ends at wrong block header"
                    );
                    // Give this value a fake endpoint that is the first instruction in the block.
                    // We expect it to be removed by calling `drop_dead_args()`.
                    self.live
                        .push(value, layout.first_inst(block).expect("Empty block"), lr);
                }
            }
        }

        self.live.values.split_at(first_arg)
    }

    /// Prepare to move past `inst`.
    ///
    /// Determine the set of already live values that are killed by `inst`, and add the new defined
    /// values to the tracked set.
    ///
    /// Returns `(throughs, kills, defs)` as a tuple of slices:
    ///
    /// 1. The `throughs` slice is the set of live-through values that are neither defined nor
    ///    killed by the instruction.
    /// 2. The `kills` slice is the set of values that were live before the instruction and are
    ///    killed at the instruction. This does not include dead defs.
    /// 3. The `defs` slice is guaranteed to be in the same order as `inst`'s results, and includes
    ///    dead defines.
    ///
    /// The order of `throughs` and `kills` is arbitrary.
    ///
    /// The `drop_dead()` method must be called next to actually remove the dead values from the
    /// tracked set after the two returned slices are no longer needed.
    pub fn process_inst(
        &mut self,
        inst: Inst,
        dfg: &DataFlowGraph,
        liveness: &Liveness,
    ) -> (&[LiveValue], &[LiveValue], &[LiveValue]) {
        // Save a copy of the live values before any branches or jumps that could be somebody's
        // immediate dominator.
        if dfg[inst].opcode().is_branch() {
            self.save_idom_live_set(inst);
        }

        // Move killed values to the end of the vector.
        // Don't remove them yet, `drop_dead()` will do that.
        let first_kill = self.live.live_after(inst);

        // Add the values defined by `inst`.
        let first_def = self.live.values.len();
        for &value in dfg.inst_results(inst) {
            let lr = &liveness[value];
            debug_assert_eq!(lr.def(), inst.into());
            match lr.def_local_end().into() {
                ExpandedProgramPoint::Inst(endpoint) => {
                    self.live.push(value, endpoint, lr);
                }
                ExpandedProgramPoint::Block(block) => {
                    panic!("Instruction result live range can't end at {}", block);
                }
            }
        }

        (
            &self.live.values[0..first_kill],
            &self.live.values[first_kill..first_def],
            &self.live.values[first_def..],
        )
    }

    /// Prepare to move past a ghost instruction.
    ///
    /// This is like `process_inst`, except any defs are ignored.
    ///
    /// Returns `(throughs, kills)`.
    pub fn process_ghost(&mut self, inst: Inst) -> (&[LiveValue], &[LiveValue]) {
        let first_kill = self.live.live_after(inst);
        self.live.values.as_slice().split_at(first_kill)
    }

    /// Drop the values that are now dead after moving past `inst`.
    ///
    /// This removes both live values that were killed by `inst` and dead defines on `inst` itself.
    ///
    /// This must be called after `process_inst(inst)` and before proceeding to the next
    /// instruction.
    pub fn drop_dead(&mut self, inst: Inst) {
        // Remove both live values that were killed by `inst` and dead defines from `inst`.
        self.live.remove_kill_values(inst);
    }

    /// Drop any values that are marked as `is_dead`.
    ///
    /// Use this after calling `block_top` to clean out dead block parameters.
    pub fn drop_dead_params(&mut self) {
        self.live.remove_dead_values();
    }

    /// Process new spills.
    ///
    /// Any values where `f` returns true are spilled and will be treated as if their affinity was
    /// `Stack`.
    pub fn process_spills<F>(&mut self, mut f: F)
    where
        F: FnMut(Value) -> bool,
    {
        for lv in &mut self.live.values {
            if f(lv.value) {
                lv.affinity = Affinity::Stack;
            }
        }
    }

    /// Save the current set of live values so it is associated with `idom`.
    fn save_idom_live_set(&mut self, idom: Inst) {
        let values = self.live.values.iter().map(|lv| lv.value);
        let pool = &mut self.idom_pool;
        // If there already is a set saved for `idom`, just keep it.
        self.idom_sets.entry(idom).or_insert_with(|| {
            let mut list = ValueList::default();
            list.extend(values, pool);
            list
        });
    }
}