1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
//! Value splitting.
//!
//! Some value types are too large to fit in registers, so they need to be split into smaller parts
//! that the ISA can operate on. There's two dimensions of splitting, represented by two
//! complementary instruction pairs:
//!
//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
//!   lane types.
//!
//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
//! This breakdown is handled by the ABI lowering.
//!
//! When legalizing a single instruction, it is wrapped in splits and concatenations:
//!
//! ```clif
//!     v1 = bxor.i64 v2, v3
//! ```
//!
//! becomes:
//!
//! ```clif
//!     v20, v21 = isplit v2
//!     v30, v31 = isplit v3
//!     v10 = bxor.i32 v20, v30
//!     v11 = bxor.i32 v21, v31
//!     v1 = iconcat v10, v11
//! ```
//!
//! This local expansion approach still leaves the original `i64` values in the code as operands on
//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
//! values are constantly split and concatenated.
//!
//! # Optimized splitting
//!
//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
//! first check if the value is defined by the corresponding concatenation. If so, then just use
//! the two concatenation inputs directly:
//!
//! ```clif
//!     v4 = iadd_imm.i64 v1, 1
//! ```
//!
//! becomes, using the expanded code from above:
//!
//! ```clif
//!     v40, v5 = iadd_imm_cout.i32 v10, 1
//!     v6 = bint.i32
//!     v41 = iadd.i32 v11, v6
//!     v4 = iconcat v40, v41
//! ```
//!
//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
//! can be trivially deleted by a dead code elimination pass.
//!
//! # block arguments
//!
//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
//! up with no `i64` values anywhere, except for block arguments. We can work around this by
//! iteratively splitting block arguments too. That should leave us with no illegal value types
//! anywhere.
//!
//! It is possible to have circular dependencies of block arguments that are never used by any real
//! instructions. These loops will remain in the program.

use crate::cursor::{Cursor, CursorPosition, FuncCursor};
use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
use crate::ir::{self, Block, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
use alloc::vec::Vec;
use core::iter;
use smallvec::SmallVec;

/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
/// if possible.
pub fn isplit(
    func: &mut ir::Function,
    cfg: &ControlFlowGraph,
    pos: CursorPosition,
    srcloc: ir::SourceLoc,
    value: Value,
) -> (Value, Value) {
    split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
}

/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
/// possible.
pub fn vsplit(
    func: &mut ir::Function,
    cfg: &ControlFlowGraph,
    pos: CursorPosition,
    srcloc: ir::SourceLoc,
    value: Value,
) -> (Value, Value) {
    split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
}

/// After splitting a block argument, we need to go back and fix up all of the predecessor
/// instructions. This is potentially a recursive operation, but we don't implement it recursively
/// since that could use up too muck stack.
///
/// Instead, the repairs are deferred and placed on a work list in stack form.
struct Repair {
    concat: Opcode,
    // The argument type after splitting.
    split_type: Type,
    // The destination block whose arguments have been split.
    block: Block,
    // Number of the original block argument which has been replaced by the low part.
    num: usize,
    // Number of the new block argument which represents the high part after the split.
    hi_num: usize,
}

/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
fn split_any(
    func: &mut ir::Function,
    cfg: &ControlFlowGraph,
    pos: CursorPosition,
    srcloc: ir::SourceLoc,
    value: Value,
    concat: Opcode,
) -> (Value, Value) {
    let mut repairs = Vec::new();
    let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
    let result = split_value(pos, value, concat, &mut repairs);

    perform_repairs(pos, cfg, repairs);

    result
}

pub fn split_block_params(func: &mut ir::Function, cfg: &ControlFlowGraph, block: Block) {
    let pos = &mut FuncCursor::new(func).at_top(block);
    let block_params = pos.func.dfg.block_params(block);

    // Add further splittable types here.
    fn type_requires_splitting(ty: Type) -> bool {
        ty == ir::types::I128
    }

    // A shortcut.  If none of the param types require splitting, exit now.  This helps because
    // the loop below necessarily has to copy the block params into a new vector, so it's better to
    // avoid doing so when possible.
    if !block_params
        .iter()
        .any(|block_param| type_requires_splitting(pos.func.dfg.value_type(*block_param)))
    {
        return;
    }

    let mut repairs = Vec::new();
    for (num, block_param) in block_params.to_vec().into_iter().enumerate() {
        if !type_requires_splitting(pos.func.dfg.value_type(block_param)) {
            continue;
        }

        split_block_param(pos, block, num, block_param, Opcode::Iconcat, &mut repairs);
    }

    perform_repairs(pos, cfg, repairs);
}

fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec<Repair>) {
    // We have split the value requested, and now we may need to fix some block predecessors.
    while let Some(repair) = repairs.pop() {
        for BlockPredecessor { inst, .. } in cfg.pred_iter(repair.block) {
            let branch_opc = pos.func.dfg[inst].opcode();
            debug_assert!(
                branch_opc.is_branch(),
                "Predecessor not a branch: {}",
                pos.func.dfg.display_inst(inst, None)
            );
            let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments();
            let mut args = pos.func.dfg[inst]
                .take_value_list()
                .expect("Branches must have value lists.");
            let num_args = args.len(&pos.func.dfg.value_lists);
            // Get the old value passed to the block argument we're repairing.
            let old_arg = args
                .get(num_fixed_args + repair.num, &pos.func.dfg.value_lists)
                .expect("Too few branch arguments");

            // It's possible that the CFG's predecessor list has duplicates. Detect them here.
            if pos.func.dfg.value_type(old_arg) == repair.split_type {
                pos.func.dfg[inst].put_value_list(args);
                continue;
            }

            // Split the old argument, possibly causing more repairs to be scheduled.
            pos.goto_inst(inst);

            let inst_block = pos.func.layout.inst_block(inst).expect("inst in block");

            // Insert split values prior to the terminal branch group.
            let canonical = pos
                .func
                .layout
                .canonical_branch_inst(&pos.func.dfg, inst_block);
            if let Some(first_branch) = canonical {
                pos.goto_inst(first_branch);
            }

            let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);

            // The `lo` part replaces the original argument.
            *args
                .get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists)
                .unwrap() = lo;

            // The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
            // same block, there could be multiple arguments missing.
            if num_args > num_fixed_args + repair.hi_num {
                *args
                    .get_mut(
                        num_fixed_args + repair.hi_num,
                        &mut pos.func.dfg.value_lists,
                    )
                    .unwrap() = hi;
            } else {
                // We need to append one or more arguments. If we're adding more than one argument,
                // there must be pending repairs on the stack that will fill in the correct values
                // instead of `hi`.
                args.extend(
                    iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args),
                    &mut pos.func.dfg.value_lists,
                );
            }

            // Put the value list back after manipulating it.
            pos.func.dfg[inst].put_value_list(args);
        }
    }
}

/// Split a single value using the integer or vector semantics given by the `concat` opcode.
///
/// If the value is defined by a `concat` instruction, just reuse the operand values of that
/// instruction.
///
/// Return the two new values representing the parts of `value`.
fn split_value(
    pos: &mut FuncCursor,
    value: Value,
    concat: Opcode,
    repairs: &mut Vec<Repair>,
) -> (Value, Value) {
    let value = pos.func.dfg.resolve_aliases(value);
    let mut reuse = None;

    match pos.func.dfg.value_def(value) {
        ValueDef::Result(inst, num) => {
            // This is an instruction result. See if the value was created by a `concat`
            // instruction.
            if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
                debug_assert_eq!(num, 0);
                if opcode == concat {
                    reuse = Some((args[0], args[1]));
                }
            }
        }
        ValueDef::Param(block, num) => {
            // This is a block parameter.
            // We can split the parameter value unless this is the entry block.
            if pos.func.layout.entry_block() != Some(block) {
                reuse = Some(split_block_param(pos, block, num, value, concat, repairs));
            }
        }
    }

    // Did the code above succeed in finding values we can reuse?
    if let Some(pair) = reuse {
        pair
    } else {
        // No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
        // has not been moved by the block argument code above when `reuse` is `None`.
        match concat {
            Opcode::Iconcat => pos.ins().isplit(value),
            Opcode::Vconcat => pos.ins().vsplit(value),
            _ => panic!("Unhandled concat opcode: {}", concat),
        }
    }
}

fn split_block_param(
    pos: &mut FuncCursor,
    block: Block,
    param_num: usize,
    value: Value,
    concat: Opcode,
    repairs: &mut Vec<Repair>,
) -> (Value, Value) {
    // We are going to replace the parameter at `num` with two new arguments.
    // Determine the new value types.
    let ty = pos.func.dfg.value_type(value);
    let split_type = match concat {
        Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
        Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
        _ => panic!("Unhandled concat opcode: {}", concat),
    };

    // Since the `repairs` stack potentially contains other parameter numbers for
    // `block`, avoid shifting and renumbering block parameters. It could invalidate other
    // `repairs` entries.
    //
    // Replace the original `value` with the low part, and append the high part at the
    // end of the argument list.
    let lo = pos.func.dfg.replace_block_param(value, split_type);
    let hi_num = pos.func.dfg.num_block_params(block);
    let hi = pos.func.dfg.append_block_param(block, split_type);

    // Now the original value is dangling. Insert a concatenation instruction that can
    // compute it from the two new parameters. This also serves as a record of what we
    // did so a future call to this function doesn't have to redo the work.
    //
    // Note that it is safe to move `pos` here since `reuse` was set above, so we don't
    // need to insert a split instruction before returning.
    pos.goto_first_inst(block);
    pos.ins()
        .with_result(value)
        .Binary(concat, split_type, lo, hi);

    // Finally, splitting the block parameter is not enough. We also have to repair all
    // of the predecessor instructions that branch here.
    add_repair(concat, split_type, block, param_num, hi_num, repairs);

    (lo, hi)
}

// Add a repair entry to the work list.
fn add_repair(
    concat: Opcode,
    split_type: Type,
    block: Block,
    num: usize,
    hi_num: usize,
    repairs: &mut Vec<Repair>,
) {
    repairs.push(Repair {
        concat,
        split_type,
        block,
        num,
        hi_num,
    });
}

/// Strip concat-split chains. Return a simpler way of computing the same value.
///
/// Given this input:
///
/// ```clif
///     v10 = iconcat v1, v2
///     v11, v12 = isplit v10
/// ```
///
/// This function resolves `v11` to `v1` and `v12` to `v2`.
fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
    let value = dfg.resolve_aliases(value);

    // Deconstruct a split instruction.
    let split_res;
    let concat_opc;
    let split_arg;
    if let ValueDef::Result(inst, num) = dfg.value_def(value) {
        split_res = num;
        concat_opc = match dfg[inst].opcode() {
            Opcode::Isplit => Opcode::Iconcat,
            Opcode::Vsplit => Opcode::Vconcat,
            _ => return value,
        };
        split_arg = dfg.inst_args(inst)[0];
    } else {
        return value;
    }

    // See if split_arg is defined by a concatenation instruction.
    if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
        if dfg[inst].opcode() == concat_opc {
            return dfg.inst_args(inst)[split_res];
        }
    }

    value
}

/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
/// legalized.
///
/// The branch argument repairs performed by `split_any()` above may be performed on branches that
/// have not yet been legalized. The repaired arguments can be defined by actual split
/// instructions in that case.
///
/// After legalizing the instructions computing the value that was split, it is likely that we can
/// avoid depending on the split instruction. Its input probably comes from a concatenation.
pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
    let mut new_args = SmallVec::<[Value; 32]>::new();

    for &arg in dfg.inst_args(branch) {
        let new_arg = resolve_splits(dfg, arg);
        new_args.push(new_arg);
    }

    dfg.inst_args_mut(branch).copy_from_slice(&new_args);
}