1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
//! Value splitting.
//!
//! Some value types are too large to fit in registers, so they need to be split into smaller parts
//! that the ISA can operate on. There's two dimensions of splitting, represented by two
//! complementary instruction pairs:
//!
//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
//! lane types.
//!
//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
//! This breakdown is handled by the ABI lowering.
//!
//! When legalizing a single instruction, it is wrapped in splits and concatenations:
//!
//! ```clif
//! v1 = bxor.i64 v2, v3
//! ```
//!
//! becomes:
//!
//! ```clif
//! v20, v21 = isplit v2
//! v30, v31 = isplit v3
//! v10 = bxor.i32 v20, v30
//! v11 = bxor.i32 v21, v31
//! v1 = iconcat v10, v11
//! ```
//!
//! This local expansion approach still leaves the original `i64` values in the code as operands on
//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
//! values are constantly split and concatenated.
//!
//! # Optimized splitting
//!
//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
//! first check if the value is defined by the corresponding concatenation. If so, then just use
//! the two concatenation inputs directly:
//!
//! ```clif
//! v4 = iadd_imm.i64 v1, 1
//! ```
//!
//! becomes, using the expanded code from above:
//!
//! ```clif
//! v40, v5 = iadd_imm_cout.i32 v10, 1
//! v6 = bint.i32
//! v41 = iadd.i32 v11, v6
//! v4 = iconcat v40, v41
//! ```
//!
//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
//! can be trivially deleted by a dead code elimination pass.
//!
//! # block arguments
//!
//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
//! up with no `i64` values anywhere, except for block arguments. We can work around this by
//! iteratively splitting block arguments too. That should leave us with no illegal value types
//! anywhere.
//!
//! It is possible to have circular dependencies of block arguments that are never used by any real
//! instructions. These loops will remain in the program.
use crate::cursor::{Cursor, CursorPosition, FuncCursor};
use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
use crate::ir::{self, Block, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
use alloc::vec::Vec;
use core::iter;
use smallvec::SmallVec;
/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
/// if possible.
pub fn isplit(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
) -> (Value, Value) {
split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
}
/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
/// possible.
pub fn vsplit(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
) -> (Value, Value) {
split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
}
/// After splitting a block argument, we need to go back and fix up all of the predecessor
/// instructions. This is potentially a recursive operation, but we don't implement it recursively
/// since that could use up too muck stack.
///
/// Instead, the repairs are deferred and placed on a work list in stack form.
struct Repair {
concat: Opcode,
// The argument type after splitting.
split_type: Type,
// The destination block whose arguments have been split.
block: Block,
// Number of the original block argument which has been replaced by the low part.
num: usize,
// Number of the new block argument which represents the high part after the split.
hi_num: usize,
}
/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
fn split_any(
func: &mut ir::Function,
cfg: &ControlFlowGraph,
pos: CursorPosition,
srcloc: ir::SourceLoc,
value: Value,
concat: Opcode,
) -> (Value, Value) {
let mut repairs = Vec::new();
let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
let result = split_value(pos, value, concat, &mut repairs);
perform_repairs(pos, cfg, repairs);
result
}
pub fn split_block_params(func: &mut ir::Function, cfg: &ControlFlowGraph, block: Block) {
let pos = &mut FuncCursor::new(func).at_top(block);
let block_params = pos.func.dfg.block_params(block);
// Add further splittable types here.
fn type_requires_splitting(ty: Type) -> bool {
ty == ir::types::I128
}
// A shortcut. If none of the param types require splitting, exit now. This helps because
// the loop below necessarily has to copy the block params into a new vector, so it's better to
// avoid doing so when possible.
if !block_params
.iter()
.any(|block_param| type_requires_splitting(pos.func.dfg.value_type(*block_param)))
{
return;
}
let mut repairs = Vec::new();
for (num, block_param) in block_params.to_vec().into_iter().enumerate() {
if !type_requires_splitting(pos.func.dfg.value_type(block_param)) {
continue;
}
split_block_param(pos, block, num, block_param, Opcode::Iconcat, &mut repairs);
}
perform_repairs(pos, cfg, repairs);
}
fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec<Repair>) {
// We have split the value requested, and now we may need to fix some block predecessors.
while let Some(repair) = repairs.pop() {
for BlockPredecessor { inst, .. } in cfg.pred_iter(repair.block) {
let branch_opc = pos.func.dfg[inst].opcode();
debug_assert!(
branch_opc.is_branch(),
"Predecessor not a branch: {}",
pos.func.dfg.display_inst(inst, None)
);
let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments();
let mut args = pos.func.dfg[inst]
.take_value_list()
.expect("Branches must have value lists.");
let num_args = args.len(&pos.func.dfg.value_lists);
// Get the old value passed to the block argument we're repairing.
let old_arg = args
.get(num_fixed_args + repair.num, &pos.func.dfg.value_lists)
.expect("Too few branch arguments");
// It's possible that the CFG's predecessor list has duplicates. Detect them here.
if pos.func.dfg.value_type(old_arg) == repair.split_type {
pos.func.dfg[inst].put_value_list(args);
continue;
}
// Split the old argument, possibly causing more repairs to be scheduled.
pos.goto_inst(inst);
let inst_block = pos.func.layout.inst_block(inst).expect("inst in block");
// Insert split values prior to the terminal branch group.
let canonical = pos
.func
.layout
.canonical_branch_inst(&pos.func.dfg, inst_block);
if let Some(first_branch) = canonical {
pos.goto_inst(first_branch);
}
let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);
// The `lo` part replaces the original argument.
*args
.get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists)
.unwrap() = lo;
// The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
// same block, there could be multiple arguments missing.
if num_args > num_fixed_args + repair.hi_num {
*args
.get_mut(
num_fixed_args + repair.hi_num,
&mut pos.func.dfg.value_lists,
)
.unwrap() = hi;
} else {
// We need to append one or more arguments. If we're adding more than one argument,
// there must be pending repairs on the stack that will fill in the correct values
// instead of `hi`.
args.extend(
iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args),
&mut pos.func.dfg.value_lists,
);
}
// Put the value list back after manipulating it.
pos.func.dfg[inst].put_value_list(args);
}
}
}
/// Split a single value using the integer or vector semantics given by the `concat` opcode.
///
/// If the value is defined by a `concat` instruction, just reuse the operand values of that
/// instruction.
///
/// Return the two new values representing the parts of `value`.
fn split_value(
pos: &mut FuncCursor,
value: Value,
concat: Opcode,
repairs: &mut Vec<Repair>,
) -> (Value, Value) {
let value = pos.func.dfg.resolve_aliases(value);
let mut reuse = None;
match pos.func.dfg.value_def(value) {
ValueDef::Result(inst, num) => {
// This is an instruction result. See if the value was created by a `concat`
// instruction.
if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
debug_assert_eq!(num, 0);
if opcode == concat {
reuse = Some((args[0], args[1]));
}
}
}
ValueDef::Param(block, num) => {
// This is a block parameter.
// We can split the parameter value unless this is the entry block.
if pos.func.layout.entry_block() != Some(block) {
reuse = Some(split_block_param(pos, block, num, value, concat, repairs));
}
}
}
// Did the code above succeed in finding values we can reuse?
if let Some(pair) = reuse {
pair
} else {
// No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
// has not been moved by the block argument code above when `reuse` is `None`.
match concat {
Opcode::Iconcat => pos.ins().isplit(value),
Opcode::Vconcat => pos.ins().vsplit(value),
_ => panic!("Unhandled concat opcode: {}", concat),
}
}
}
fn split_block_param(
pos: &mut FuncCursor,
block: Block,
param_num: usize,
value: Value,
concat: Opcode,
repairs: &mut Vec<Repair>,
) -> (Value, Value) {
// We are going to replace the parameter at `num` with two new arguments.
// Determine the new value types.
let ty = pos.func.dfg.value_type(value);
let split_type = match concat {
Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
_ => panic!("Unhandled concat opcode: {}", concat),
};
// Since the `repairs` stack potentially contains other parameter numbers for
// `block`, avoid shifting and renumbering block parameters. It could invalidate other
// `repairs` entries.
//
// Replace the original `value` with the low part, and append the high part at the
// end of the argument list.
let lo = pos.func.dfg.replace_block_param(value, split_type);
let hi_num = pos.func.dfg.num_block_params(block);
let hi = pos.func.dfg.append_block_param(block, split_type);
// Now the original value is dangling. Insert a concatenation instruction that can
// compute it from the two new parameters. This also serves as a record of what we
// did so a future call to this function doesn't have to redo the work.
//
// Note that it is safe to move `pos` here since `reuse` was set above, so we don't
// need to insert a split instruction before returning.
pos.goto_first_inst(block);
pos.ins()
.with_result(value)
.Binary(concat, split_type, lo, hi);
// Finally, splitting the block parameter is not enough. We also have to repair all
// of the predecessor instructions that branch here.
add_repair(concat, split_type, block, param_num, hi_num, repairs);
(lo, hi)
}
// Add a repair entry to the work list.
fn add_repair(
concat: Opcode,
split_type: Type,
block: Block,
num: usize,
hi_num: usize,
repairs: &mut Vec<Repair>,
) {
repairs.push(Repair {
concat,
split_type,
block,
num,
hi_num,
});
}
/// Strip concat-split chains. Return a simpler way of computing the same value.
///
/// Given this input:
///
/// ```clif
/// v10 = iconcat v1, v2
/// v11, v12 = isplit v10
/// ```
///
/// This function resolves `v11` to `v1` and `v12` to `v2`.
fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
let value = dfg.resolve_aliases(value);
// Deconstruct a split instruction.
let split_res;
let concat_opc;
let split_arg;
if let ValueDef::Result(inst, num) = dfg.value_def(value) {
split_res = num;
concat_opc = match dfg[inst].opcode() {
Opcode::Isplit => Opcode::Iconcat,
Opcode::Vsplit => Opcode::Vconcat,
_ => return value,
};
split_arg = dfg.inst_args(inst)[0];
} else {
return value;
}
// See if split_arg is defined by a concatenation instruction.
if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
if dfg[inst].opcode() == concat_opc {
return dfg.inst_args(inst)[split_res];
}
}
value
}
/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
/// legalized.
///
/// The branch argument repairs performed by `split_any()` above may be performed on branches that
/// have not yet been legalized. The repaired arguments can be defined by actual split
/// instructions in that case.
///
/// After legalizing the instructions computing the value that was split, it is likely that we can
/// avoid depending on the split instruction. Its input probably comes from a concatenation.
pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
let mut new_args = SmallVec::<[Value; 32]>::new();
for &arg in dfg.inst_args(branch) {
let new_arg = resolve_splits(dfg, arg);
new_args.push(new_arg);
}
dfg.inst_args_mut(branch).copy_from_slice(&new_args);
}