From ecbe1a6b5a826c938b9e39ca17f428f7df6305ec Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 9 Apr 2024 21:28:20 +0200 Subject: [PATCH] Generate allocation and release code Instead of allocations and releases calling functions provided by the runtime library (inko_alloc() and inko_free()), the compiler now generates the underlying code directly. Depending on the program used, this can improve performance by 3-5%. While this isn't significant, it's an easy win and help reduces the amount of code provided by the runtime library. See https://github.com/inko-lang/inko/issues/542 for more details. Changelog: performance --- compiler/src/format.rs | 5 - compiler/src/llvm/builder.rs | 54 ++++++ compiler/src/llvm/layouts.rs | 15 +- compiler/src/llvm/passes.rs | 235 ++++++++++++-------------- compiler/src/llvm/runtime_function.rs | 42 +++-- compiler/src/mir/mod.rs | 22 ++- compiler/src/mir/passes.rs | 4 +- compiler/src/mir/specialize.rs | 18 +- rt/src/mem.rs | 19 +-- rt/src/memory_map.rs | 29 ++-- rt/src/process.rs | 9 +- rt/src/runtime/general.rs | 29 +--- rt/src/stack.rs | 2 +- 13 files changed, 269 insertions(+), 214 deletions(-) diff --git a/compiler/src/format.rs b/compiler/src/format.rs index 2fd2d4ab9..3f3f4c6ae 100644 --- a/compiler/src/format.rs +++ b/compiler/src/format.rs @@ -109,11 +109,6 @@ enum Node { Indent(Vec), /// A node of which the width should be reported as zero. - /// - /// TODO: this node causes too many weird formatting issues, such as when a - /// ZeroWidth argument is followed by an array. This is because when - /// rendering, we think we have more space than we actually do, so we wrap - /// too late. ZeroWidth(Box), /// Indent the given nodes recursively, but only starting the next line. diff --git a/compiler/src/llvm/builder.rs b/compiler/src/llvm/builder.rs index 2cae2246c..83c818ff7 100644 --- a/compiler/src/llvm/builder.rs +++ b/compiler/src/llvm/builder.rs @@ -1,4 +1,7 @@ +use crate::llvm::constants::{HEADER_CLASS_INDEX, HEADER_REFS_INDEX}; use crate::llvm::context::Context; +use crate::llvm::module::Module; +use crate::llvm::runtime_function::RuntimeFunction; use inkwell::basic_block::BasicBlock; use inkwell::builder; use inkwell::debug_info::{ @@ -17,6 +20,7 @@ use inkwell::{ AddressSpace, AtomicOrdering, AtomicRMWBinOp, FloatPredicate, IntPredicate, }; use std::path::Path; +use types::{ClassId, Database}; /// A wrapper around an LLVM Builder that provides some additional methods. pub(crate) struct Builder<'ctx> { @@ -652,6 +656,13 @@ impl<'ctx> Builder<'ctx> { .unwrap() } + pub(crate) fn pointer_is_null( + &self, + value: PointerValue<'ctx>, + ) -> IntValue<'ctx> { + self.inner.build_is_null(value, "").unwrap() + } + pub(crate) fn bitcast, T: BasicType<'ctx>>( &self, value: V, @@ -755,6 +766,49 @@ impl<'ctx> Builder<'ctx> { pub(crate) fn set_debug_function(&self, function: DISubprogram) { self.function.set_subprogram(function); } + + pub(crate) fn allocate<'a, 'b>( + &self, + module: &'a mut Module<'b, 'ctx>, + db: &Database, + names: &crate::symbol_names::SymbolNames, + class: ClassId, + ) -> PointerValue<'ctx> { + let atomic = class.is_atomic(db); + let name = &names.classes[&class]; + let global = module.add_class(class, name).as_pointer_value(); + let class_ptr = self.load_untyped_pointer(global); + let size = module.layouts.size_of_class(class); + let err_func = + module.runtime_function(RuntimeFunction::AllocationError); + let alloc_func = module.runtime_function(RuntimeFunction::Allocate); + let size = self.u64_literal(size).into(); + let res = self.call(alloc_func, &[size]).into_pointer_value(); + + let err_block = self.add_block(); + let ok_block = self.add_block(); + let is_null = self.pointer_is_null(res); + let header = module.layouts.header; + + self.branch(is_null, err_block, ok_block); + + // The block to jump to when the allocation failed. + self.switch_to_block(err_block); + self.call_void(err_func, &[class_ptr.into()]); + self.unreachable(); + + // The block to jump to when the allocation succeeds. + self.switch_to_block(ok_block); + + // Atomic values start with a reference count of 1, so atomic decrements + // returns the correct result for a value for which no extra references + // have been created (instead of underflowing). + let refs = self.u32_literal(if atomic { 1 } else { 0 }); + + self.store_field(header, res, HEADER_CLASS_INDEX, class_ptr); + self.store_field(header, res, HEADER_REFS_INDEX, refs); + res + } } /// A wrapper around the LLVM types used for building debugging information. diff --git a/compiler/src/llvm/layouts.rs b/compiler/src/llvm/layouts.rs index 607738164..add4e737c 100644 --- a/compiler/src/llvm/layouts.rs +++ b/compiler/src/llvm/layouts.rs @@ -8,7 +8,8 @@ use inkwell::types::{ }; use inkwell::AddressSpace; use types::{ - CallConvention, BOOL_ID, BYTE_ARRAY_ID, FLOAT_ID, INT_ID, NIL_ID, STRING_ID, + CallConvention, ClassId, BOOL_ID, BYTE_ARRAY_ID, FLOAT_ID, INT_ID, NIL_ID, + STRING_ID, }; /// The size of an object header. @@ -31,6 +32,8 @@ pub(crate) struct Method<'ctx> { /// Types and layout information to expose to all modules. pub(crate) struct Layouts<'ctx> { + pub(crate) target_data: &'ctx TargetData, + /// The layout of an empty class. /// /// This is used for generating dynamic dispatch code, as we don't know the @@ -80,7 +83,7 @@ impl<'ctx> Layouts<'ctx> { state: &State, mir: &Mir, context: &'ctx Context, - target_data: TargetData, + target_data: &'ctx TargetData, ) -> Self { let db = &state.db; let space = AddressSpace::default(); @@ -179,6 +182,7 @@ impl<'ctx> Layouts<'ctx> { }; let mut layouts = Self { + target_data, empty_class: context.class_type(method), method, classes, @@ -408,4 +412,11 @@ impl<'ctx> Layouts<'ctx> { layouts } + + pub(crate) fn size_of_class(&self, class: ClassId) -> u64 { + let layout = &self.instances[class.0 as usize]; + + self.target_data.get_bit_size(layout) + / (self.target_data.get_pointer_byte_size(None) as u64) + } } diff --git a/compiler/src/llvm/passes.rs b/compiler/src/llvm/passes.rs index 41ebdf91d..873d7d887 100644 --- a/compiler/src/llvm/passes.rs +++ b/compiler/src/llvm/passes.rs @@ -518,11 +518,12 @@ impl<'a> Worker<'a> { // to `Context`, preventing it from being moved into `self`. Thus, we // create this data here and pass it as arguments. let context = Context::new(); + let target_data = self.machine.get_target_data(); let layouts = Layouts::new( self.shared.state, self.shared.mir, &context, - self.machine.get_target_data(), + &target_data, ); loop { @@ -562,17 +563,14 @@ impl<'a> Worker<'a> { let mut module = Module::new(context, layouts, name.clone(), &path); LowerModule { - state: self.shared.state, - mir: self.shared.mir, + shared: self.shared, index, - names: self.shared.names, module: &mut module, layouts, - methods: self.shared.methods, } .run(); - let res = self.process_module(&module, obj_path); + let res = self.process_module(&module, layouts, obj_path); self.timings.insert(name.clone(), start.elapsed()); res @@ -599,7 +597,7 @@ impl<'a> Worker<'a> { .run(); let path = object_path(self.shared.directories, &name); - let res = self.process_module(&main, path); + let res = self.process_module(&main, layouts, path); self.timings.insert(name, start.elapsed()); res @@ -608,14 +606,15 @@ impl<'a> Worker<'a> { fn process_module( &self, module: &Module, + layouts: &Layouts, path: PathBuf, ) -> Result { - self.run_passes(module); + self.run_passes(module, layouts); self.write_object_file(module, path) } - fn run_passes(&self, module: &Module) { - let layout = self.machine.get_target_data().get_data_layout(); + fn run_passes(&self, module: &Module, layouts: &Layouts) { + let layout = layouts.target_data.get_data_layout(); let opts = PassBuilderOptions::create(); let passes = if let Opt::Aggressive = self.shared.state.config.opt { &["default"] @@ -667,27 +666,21 @@ impl<'a> Worker<'a> { } /// A pass that lowers a single Inko module into an LLVM module. -pub(crate) struct LowerModule<'a, 'b, 'mir, 'ctx> { - state: &'a State, - mir: &'mir Mir, +pub(crate) struct LowerModule<'shared, 'module, 'ctx> { + shared: &'shared SharedState<'shared>, index: usize, layouts: &'ctx Layouts<'ctx>, - methods: &'a Methods, - names: &'a SymbolNames, - module: &'b mut Module<'a, 'ctx>, + module: &'module mut Module<'shared, 'ctx>, } -impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { +impl<'shared, 'module, 'ctx> LowerModule<'shared, 'module, 'ctx> { pub(crate) fn run(mut self) { - for method in &self.mir.modules[self.index].methods { + for method in &self.shared.mir.modules[self.index].methods { LowerMethod::new( - self.state, - self.mir, + self.shared, self.layouts, - self.methods, - self.names, self.module, - &self.mir.methods[method], + &self.shared.mir.methods[method], ) .run(); } @@ -698,9 +691,9 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { } fn setup_classes(&mut self) { - let mod_id = self.mir.modules[self.index].id; + let mod_id = self.shared.mir.modules[self.index].id; let space = AddressSpace::default(); - let fn_name = &self.names.setup_classes[&mod_id]; + let fn_name = &self.shared.names.setup_classes[&mod_id]; let fn_val = self.module.add_setup_function(fn_name); let builder = Builder::new(self.module.context, fn_val); let entry_block = self.module.context.append_basic_block(fn_val); @@ -715,24 +708,27 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { // Allocate all classes defined in this module, and store them in their // corresponding globals. - for &class_id in &self.mir.modules[self.index].classes { - let raw_name = class_id.name(&self.state.db); + for &class_id in &self.shared.mir.modules[self.index].classes { + let raw_name = class_id.name(&self.shared.state.db); let name_ptr = builder.string_literal(raw_name).0.into(); let methods_len = self .module .context .i16_type() - .const_int(self.methods.counts[class_id.0 as usize] as _, false) + .const_int( + self.shared.methods.counts[class_id.0 as usize] as _, + false, + ) .into(); - let class_new = if class_id.kind(&self.state.db).is_async() { + let class_new = if class_id.kind(&self.shared.state.db).is_async() { self.module.runtime_function(RuntimeFunction::ClassProcess) } else { self.module.runtime_function(RuntimeFunction::ClassObject) }; let layout = self.layouts.classes[class_id.0 as usize]; - let global_name = &self.names.classes[&class_id]; + let global_name = &self.shared.names.classes[&class_id]; let global = self.module.add_class(class_id, global_name); // The class globals must have an initializer, otherwise LLVM treats @@ -765,15 +761,15 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { } }; - for method in &self.mir.classes[&class_id].methods { + for method in &self.shared.mir.classes[&class_id].methods { // Static methods aren't stored in classes, nor can we call them // through dynamic dispatch, so we can skip the rest. - if method.is_static(&self.state.db) { + if method.is_static(&self.shared.state.db) { continue; } - let info = &self.methods.info[method.0 as usize]; - let name = &self.names.methods[method]; + let info = &self.shared.methods.info[method.0 as usize]; + let name = &self.shared.names.methods[method]; let func = self .module .get_function(name) @@ -810,8 +806,8 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { } fn setup_constants(&mut self) { - let mod_id = self.mir.modules[self.index].id; - let fn_name = &self.names.setup_constants[&mod_id]; + let mod_id = self.shared.mir.modules[self.index].id; + let fn_name = &self.shared.names.setup_constants[&mod_id]; let fn_val = self.module.add_setup_function(fn_name); let builder = Builder::new(self.module.context, fn_val); let entry_block = self.module.context.append_basic_block(fn_val); @@ -824,10 +820,10 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { builder.jump(body); builder.switch_to_block(body); - for &cid in &self.mir.modules[self.index].constants { - let name = &self.names.constants[&cid]; + for &cid in &self.shared.mir.modules[self.index].constants { + let name = &self.shared.names.constants[&cid]; let global = self.module.add_constant(name); - let value = &self.mir.constants[&cid]; + let value = &self.shared.mir.constants[&cid]; global.set_initializer( &self @@ -915,20 +911,15 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { ), }; - let class_id = ClassId::array().specializations(&self.state.db) - [&vec![shape]]; - + let class_id = ClassId::array() + .specializations(&self.shared.state.db)[&vec![shape]]; let layout = self.layouts.instances[class_id.0 as usize]; - let class_name = &self.names.classes[&class_id]; - let class_global = self - .module - .add_class(class_id, class_name) - .as_pointer_value(); - let class = builder.load_untyped_pointer(class_global); - let alloc = - self.module.runtime_function(RuntimeFunction::Allocate); - let array = - builder.call(alloc, &[class.into()]).into_pointer_value(); + let array = builder.allocate( + self.module, + &self.shared.state.db, + self.shared.names, + class_id, + ); let buf_typ = val_typ.array_type(values.len() as _); @@ -991,25 +982,18 @@ impl<'a, 'b, 'mir, 'ctx> LowerModule<'a, 'b, 'mir, 'ctx> { } /// A pass that lowers a single Inko method into an LLVM method. -pub struct LowerMethod<'a, 'b, 'mir, 'ctx> { - state: &'a State, - mir: &'mir Mir, +pub struct LowerMethod<'shared, 'module, 'ctx> { + shared: &'shared SharedState<'shared>, layouts: &'ctx Layouts<'ctx>, - methods: &'a Methods, /// The MIR method that we're lowering to LLVM. - method: &'mir Method, - - /// A map of method names to their mangled names. - /// - /// We cache these so we don't have to recalculate them on every reference. - names: &'a SymbolNames, + method: &'shared Method, /// The builder to use for generating instructions. builder: Builder<'ctx>, /// The LLVM module the generated code belongs to. - module: &'b mut Module<'a, 'ctx>, + module: &'module mut Module<'shared, 'ctx>, /// MIR registers and their corresponding LLVM stack variables. variables: HashMap>, @@ -1018,26 +1002,21 @@ pub struct LowerMethod<'a, 'b, 'mir, 'ctx> { variable_types: HashMap>, } -impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { +impl<'shared, 'module, 'ctx> LowerMethod<'shared, 'module, 'ctx> { fn new( - state: &'a State, - mir: &'mir Mir, + shared: &'shared SharedState<'shared>, layouts: &'ctx Layouts<'ctx>, - methods: &'a Methods, - names: &'a SymbolNames, - module: &'b mut Module<'a, 'ctx>, - method: &'mir Method, + module: &'module mut Module<'shared, 'ctx>, + method: &'shared Method, ) -> Self { - let function = module.add_method(&names.methods[&method.id], method.id); + let function = + module.add_method(&shared.names.methods[&method.id], method.id); let builder = Builder::new(module.context, function); LowerMethod { - state, - mir, + shared, layouts, - methods, method, - names, module, builder, variables: HashMap::new(), @@ -1046,7 +1025,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { } fn run(&mut self) { - if self.method.id.is_async(&self.state.db) { + if self.method.id.is_async(&self.shared.state.db) { self.async_method(); } else { self.regular_method(); @@ -1065,13 +1044,14 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { self.builder.store(self.variables[reg], arg); } - let (line, _) = self.mir.location(self.method.location).line_column(); + let (line, _) = + self.shared.mir.location(self.method.location).line_column(); let debug_func = self.module.debug_builder.new_function( - self.method.id.name(&self.state.db), - &self.names.methods[&self.method.id], - &self.method.id.source_file(&self.state.db), + self.method.id.name(&self.shared.state.db), + &self.shared.names.methods[&self.method.id], + &self.method.id.source_file(&self.shared.state.db), line, - self.method.id.is_private(&self.state.db), + self.method.id.is_private(&self.shared.state.db), false, ); @@ -1126,13 +1106,14 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { self.builder.store(var, val); } - let (line, _) = self.mir.location(self.method.location).line_column(); + let (line, _) = + self.shared.mir.location(self.method.location).line_column(); let debug_func = self.module.debug_builder.new_function( - self.method.id.name(&self.state.db), - &self.names.methods[&self.method.id], - &self.method.id.source_file(&self.state.db), + self.method.id.name(&self.shared.state.db), + &self.shared.names.methods[&self.method.id], + &self.method.id.source_file(&self.shared.state.db), line, - self.method.id.is_private(&self.state.db), + self.method.id.is_private(&self.shared.state.db), false, ); @@ -1810,7 +1791,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { Instruction::CallExtern(ins) => { self.set_debug_location(ins.location); - let func_name = ins.method.name(&self.state.db); + let func_name = ins.method.name(&self.shared.state.db); let func = self.module.add_method(func_name, ins.method); let mut args: Vec = Vec::with_capacity(ins.arguments.len() + 1); @@ -1846,7 +1827,9 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { self.builder.store(ret, self.builder.load(typ, temp)); } - if self.register_type(ins.register).is_never(&self.state.db) + if self + .register_type(ins.register) + .is_never(&self.shared.state.db) { self.builder.unreachable(); } @@ -1855,7 +1838,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { Instruction::CallStatic(ins) => { self.set_debug_location(ins.location); - let func_name = &self.names.methods[&ins.method]; + let func_name = &self.shared.names.methods[&ins.method]; let func = self.module.add_method(func_name, ins.method); let mut args: Vec = Vec::with_capacity(ins.arguments.len()); @@ -1874,7 +1857,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; - let func_name = &self.names.methods[&ins.method]; + let func_name = &self.shared.names.methods[&ins.method]; let func = self.module.add_method(func_name, ins.method); let mut args: Vec = vec![self.builder.load(rec_typ, rec_var).into()]; @@ -1903,7 +1886,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; let rec = self.builder.load(rec_typ, rec_var); - let info = &self.methods.info[ins.method.0 as usize]; + let info = &self.shared.methods.info[ins.method.0 as usize]; let fn_typ = self.layouts.methods[ins.method.0 as usize].signature; @@ -2126,7 +2109,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; - let method_name = &self.names.methods[&ins.method]; + let method_name = &self.shared.names.methods[&ins.method]; let method = self .module .add_method(method_name, ins.method) @@ -2172,13 +2155,13 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { ); } Instruction::GetField(ins) - if ins.class.kind(&self.state.db).is_extern() => + if ins.class.kind(&self.shared.state.db).is_extern() => { let reg_var = self.variables[&ins.register]; let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; let layout = self.layouts.instances[ins.class.0 as usize]; - let index = ins.field.index(&self.state.db) as u32; + let index = ins.field.index(&self.shared.state.db) as u32; let field = if rec_typ.is_pointer_type() { let rec = self .builder @@ -2196,13 +2179,13 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { self.builder.store(reg_var, field); } Instruction::SetField(ins) - if ins.class.kind(&self.state.db).is_extern() => + if ins.class.kind(&self.shared.state.db).is_extern() => { let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; let val_var = self.variables[&ins.value]; let layout = self.layouts.instances[ins.class.0 as usize]; - let index = ins.field.index(&self.state.db) as u32; + let index = ins.field.index(&self.shared.state.db) as u32; let val_typ = self.variable_types[&ins.value]; let val = self.builder.load(val_typ, val_var); @@ -2221,13 +2204,14 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let reg_var = self.variables[&ins.register]; let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; - let base = if ins.class.kind(&self.state.db).is_async() { + let base = if ins.class.kind(&self.shared.state.db).is_async() { PROCESS_FIELD_OFFSET } else { FIELD_OFFSET }; - let index = (base + ins.field.index(&self.state.db)) as u32; + let index = + (base + ins.field.index(&self.shared.state.db)) as u32; let layout = self.layouts.instances[ins.class.0 as usize]; let rec = self.builder.load(rec_typ, rec_var); let field = self.builder.load_field( @@ -2242,13 +2226,14 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let reg_var = self.variables[&ins.register]; let rec_var = self.variables[&ins.receiver]; let rec_typ = self.variable_types[&ins.receiver]; - let base = if ins.class.kind(&self.state.db).is_async() { + let base = if ins.class.kind(&self.shared.state.db).is_async() { PROCESS_FIELD_OFFSET } else { FIELD_OFFSET }; - let index = (base + ins.field.index(&self.state.db)) as u32; + let index = + (base + ins.field.index(&self.shared.state.db)) as u32; let layout = self.layouts.instances[ins.class.0 as usize]; let rec = self.builder.load(rec_typ, rec_var); let addr = self.builder.field_address( @@ -2261,7 +2246,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { } Instruction::MethodPointer(ins) => { let reg_var = self.variables[&ins.register]; - let func_name = &self.names.methods[&ins.method]; + let func_name = &self.shared.names.methods[&ins.method]; let func = self.module.add_method(func_name, ins.method); let ptr = func.as_global_value().as_pointer_value(); @@ -2272,13 +2257,14 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let rec_typ = self.variable_types[&ins.receiver]; let val_var = self.variables[&ins.value]; let val_typ = self.variable_types[&ins.value]; - let base = if ins.class.kind(&self.state.db).is_async() { + let base = if ins.class.kind(&self.shared.state.db).is_async() { PROCESS_FIELD_OFFSET } else { FIELD_OFFSET }; - let index = (base + ins.field.index(&self.state.db)) as u32; + let index = + (base + ins.field.index(&self.shared.state.db)) as u32; let val = self.builder.load(val_typ, val_var); let layout = self.layouts.instances[ins.class.0 as usize]; let rec = self.builder.load(rec_typ, rec_var); @@ -2323,10 +2309,10 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { } Instruction::Free(ins) => { let var = self.variables[&ins.register]; - let free = self.builder.load_untyped_pointer(var).into(); + let ptr = self.builder.load_untyped_pointer(var); let func = self.module.runtime_function(RuntimeFunction::Free); - self.builder.call_void(func, &[free]); + self.builder.call_void(func, &[ptr.into()]); } Instruction::Increment(ins) => { let reg_var = self.variables[&ins.register]; @@ -2384,7 +2370,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { self.builder.branch(is_zero, drop_block, after_block); } Instruction::Allocate(ins) - if ins.class.kind(&self.state.db).is_extern() => + if ins.class.kind(&self.shared.state.db).is_extern() => { // Defining the alloca already reserves (uninitialised) memory, // so there's nothing we actually need to do here. Setting the @@ -2392,24 +2378,13 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { } Instruction::Allocate(ins) => { let reg_var = self.variables[&ins.register]; - let name = &self.names.classes[&ins.class]; - let global = - self.module.add_class(ins.class, name).as_pointer_value(); - let class = self.builder.load_untyped_pointer(global); - let func_name = if ins.class.is_atomic(&self.state.db) { - RuntimeFunction::AllocateAtomic - } else { - RuntimeFunction::Allocate - }; - - let func = self.module.runtime_function(func_name); - let ptr = self.builder.call(func, &[class.into()]); + let ptr = self.allocate(ins.class).as_basic_value_enum(); self.builder.store(reg_var, ptr); } Instruction::Spawn(ins) => { let reg_var = self.variables[&ins.register]; - let name = &self.names.classes[&ins.class]; + let name = &self.shared.names.classes[&ins.class]; let global = self.module.add_class(ins.class, name).as_pointer_value(); let class = self.builder.load_untyped_pointer(global).into(); @@ -2423,7 +2398,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { Instruction::GetConstant(ins) => { let var = self.variables[&ins.register]; let typ = self.variable_types[&ins.register]; - let name = &self.names.constants[&ins.id]; + let name = &self.shared.names.constants[&ins.id]; let global = self.module.add_constant(name).as_pointer_value(); let value = self.builder.load(typ, global); @@ -2587,7 +2562,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { let id = RegisterId(index as _); let raw = self.method.registers.value_type(id); let typ = self.builder.context.llvm_type( - &self.state.db, + &self.shared.state.db, self.layouts, raw, ); @@ -2609,7 +2584,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { ) { let var = self.variables[®ister]; - if self.register_type(register).is_never(&self.state.db) { + if self.register_type(register).is_never(&self.shared.state.db) { self.builder.call_void(function, arguments); self.builder.unreachable(); } else { @@ -2626,7 +2601,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { ) { let var = self.variables[®ister]; - if self.register_type(register).is_never(&self.state.db) { + if self.register_type(register).is_never(&self.shared.state.db) { self.builder.indirect_call(function_type, function, arguments); self.builder.unreachable(); } else { @@ -2643,7 +2618,7 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { fn set_debug_location(&self, location_id: LocationId) { let scope = self.builder.debug_scope(); - let (line, col) = self.mir.location(location_id).line_column(); + let (line, col) = self.shared.mir.location(location_id).line_column(); let loc = self.module.debug_builder.new_location(line, col, scope); self.builder.set_debug_location(loc); @@ -2684,7 +2659,8 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { &[self.builder.context.i64_type().into()], ); - let rsp_name = self.state.config.target.stack_pointer_register_name(); + let rsp_name = + self.shared.state.config.target.stack_pointer_register_name(); let mname = self.builder.context.inner.metadata_string(rsp_name); let mnode = self.builder.context.inner.metadata_node(&[mname.into()]); let rsp_addr = @@ -2708,6 +2684,15 @@ impl<'a, 'b, 'mir, 'ctx> LowerMethod<'a, 'b, 'mir, 'ctx> { .load(self.builder.context.i64_type(), var.as_pointer_value()) .into_int_value() } + + fn allocate(&mut self, class: ClassId) -> PointerValue<'ctx> { + self.builder.allocate( + self.module, + &self.shared.state.db, + self.shared.names, + class, + ) + } } /// A pass for generating the entry module and method (i.e. `main()`). diff --git a/compiler/src/llvm/runtime_function.rs b/compiler/src/llvm/runtime_function.rs index 6fc95479f..226eda5af 100644 --- a/compiler/src/llvm/runtime_function.rs +++ b/compiler/src/llvm/runtime_function.rs @@ -7,10 +7,7 @@ pub(crate) enum RuntimeFunction { ReferenceCountError, ClassObject, ClassProcess, - Free, MessageNew, - Allocate, - AllocateAtomic, ProcessFinishMessage, ProcessNew, ProcessPanic, @@ -23,6 +20,9 @@ pub(crate) enum RuntimeFunction { StringConcat, StringNew, RuntimeStackMask, + Allocate, + Free, + AllocationError, } impl RuntimeFunction { @@ -33,10 +33,7 @@ impl RuntimeFunction { } RuntimeFunction::ClassObject => "inko_class_object", RuntimeFunction::ClassProcess => "inko_class_process", - RuntimeFunction::Free => "inko_free", RuntimeFunction::MessageNew => "inko_message_new", - RuntimeFunction::Allocate => "inko_alloc", - RuntimeFunction::AllocateAtomic => "inko_alloc_atomic", RuntimeFunction::ProcessFinishMessage => { "inko_process_finish_message" } @@ -51,6 +48,9 @@ impl RuntimeFunction { RuntimeFunction::StringConcat => "inko_string_concat", RuntimeFunction::StringNew => "inko_string_new", RuntimeFunction::RuntimeStackMask => "inko_runtime_stack_mask", + RuntimeFunction::Allocate => "malloc", + RuntimeFunction::Free => "free", + RuntimeFunction::AllocationError => "inko_alloc_error", } } @@ -68,24 +68,12 @@ impl RuntimeFunction { ret.fn_type(&[proc, val], false) } - RuntimeFunction::Free => { - let val = context.pointer_type().into(); - let ret = context.void_type(); - - ret.fn_type(&[val], false) - } RuntimeFunction::ProcessYield => { let proc = context.pointer_type().into(); let ret = context.void_type(); ret.fn_type(&[proc], false) } - RuntimeFunction::Allocate | RuntimeFunction::AllocateAtomic => { - let class = context.pointer_type().into(); - let ret = context.pointer_type(); - - ret.fn_type(&[class], false) - } RuntimeFunction::ProcessPanic => { let proc = context.pointer_type().into(); let val = context.pointer_type().into(); @@ -182,6 +170,24 @@ impl RuntimeFunction { ret.fn_type(&[state], false) } + RuntimeFunction::Allocate => { + let size = context.i64_type().into(); + let ret = context.pointer_type(); + + ret.fn_type(&[size], false) + } + RuntimeFunction::Free => { + let ptr = context.pointer_type().into(); + let ret = context.void_type(); + + ret.fn_type(&[ptr], false) + } + RuntimeFunction::AllocationError => { + let ptr = context.pointer_type().into(); + let ret = context.void_type(); + + ret.fn_type(&[ptr], false) + } }; module.add_function(self.name(), fn_type, None) diff --git a/compiler/src/mir/mod.rs b/compiler/src/mir/mod.rs index ee9c1c0d9..fbcf793f3 100644 --- a/compiler/src/mir/mod.rs +++ b/compiler/src/mir/mod.rs @@ -372,9 +372,17 @@ impl Block { }))); } - pub(crate) fn free(&mut self, register: RegisterId, location: LocationId) { - self.instructions - .push(Instruction::Free(Box::new(Free { register, location }))); + pub(crate) fn free( + &mut self, + register: RegisterId, + class: types::ClassId, + location: LocationId, + ) { + self.instructions.push(Instruction::Free(Box::new(Free { + register, + class, + location, + }))); } pub(crate) fn check_refs( @@ -790,6 +798,7 @@ pub(crate) struct CallDropper { #[derive(Clone)] pub(crate) struct Free { + pub(crate) class: types::ClassId, pub(crate) register: RegisterId, pub(crate) location: LocationId, } @@ -1218,7 +1227,12 @@ impl Instruction { format!("drop r{}", v.register.0) } Instruction::Free(ref v) => { - format!("free r{}", v.register.0) + format!( + "free r{} {}#{}", + v.register.0, + v.class.name(db), + v.class.0 + ) } Instruction::CheckRefs(ref v) => { format!("check_refs r{}", v.register.0) diff --git a/compiler/src/mir/passes.rs b/compiler/src/mir/passes.rs index 0768b0018..8f6444a2a 100644 --- a/compiler/src/mir/passes.rs +++ b/compiler/src/mir/passes.rs @@ -499,7 +499,7 @@ impl<'a> GenerateDropper<'a> { lower.current_block_mut().check_refs(self_reg, loc); lower.drop_register(tag_reg, loc); - lower.current_block_mut().free(self_reg, loc); + lower.current_block_mut().free(self_reg, class, loc); let nil_reg = lower.get_nil(loc); @@ -563,7 +563,7 @@ impl<'a> GenerateDropper<'a> { lower.current_block_mut().check_refs(self_reg, loc); if free_self { - lower.current_block_mut().free(self_reg, loc); + lower.current_block_mut().free(self_reg, class, loc); } if terminate { diff --git a/compiler/src/mir/specialize.rs b/compiler/src/mir/specialize.rs index c7533ca59..d15eab51b 100644 --- a/compiler/src/mir/specialize.rs +++ b/compiler/src/mir/specialize.rs @@ -415,6 +415,15 @@ impl<'a, 'b> Specialize<'a, 'b> { ins.class = cls; self.schedule_regular_dropper(cls); } + Instruction::Free(ins) => { + let cls = method + .registers + .value_type(ins.register) + .class_id(&self.state.db) + .unwrap(); + + ins.class = cls; + } Instruction::Spawn(ins) => { let cls = method .registers @@ -1200,8 +1209,15 @@ impl<'a, 'b, 'c> ExpandDrop<'a, 'b, 'c> { if dropper { self.call_dropper(before_id, value, location); } else { + let class = self + .method + .registers + .value_type(value) + .class_id(self.db) + .unwrap(); + self.block_mut(before_id).check_refs(value, location); - self.block_mut(before_id).free(value, location); + self.block_mut(before_id).free(value, class, location); } self.block_mut(before_id).goto(after_id, location); diff --git a/rt/src/mem.rs b/rt/src/mem.rs index edb648f44..90651b042 100644 --- a/rt/src/mem.rs +++ b/rt/src/mem.rs @@ -25,12 +25,6 @@ pub(crate) unsafe fn header_of<'a, T>(ptr: *const T) -> &'a mut Header { &mut *(ptr as *mut Header) } -pub(crate) unsafe fn free(ptr: *mut T) { - let layout = header_of(ptr).class.instance_layout(); - - dealloc(ptr as *mut u8, layout); -} - /// The header used by heap allocated objects. /// /// The layout is fixed to ensure we can safely assume certain fields are at @@ -50,6 +44,11 @@ pub struct Header { /// overflowing a u32 is very tiny, but overflowing a u16 is something that /// _could_ happen (i.e. a process reference shared with many other /// processes). + /// + /// For regular objects, this field is initially set to 0, while for atomic + /// values it defaults to 1. The latter is done as atomics always use a + /// checked decrement, so starting with 1 ensures we don't underflow this + /// value. pub references: u32, } @@ -61,16 +60,8 @@ impl Header { pub(crate) fn init_atomic(&mut self, class: ClassPointer) { self.class = class; - - // Atomic values start with a reference count of 1, so - // `decrement_atomic()` returns the correct result for a value for which - // no extra references have been created (instead of overflowing). self.references = 1; } - - pub(crate) fn references(&self) -> u32 { - self.references - } } /// A function bound to an object. diff --git a/rt/src/memory_map.rs b/rt/src/memory_map.rs index 728f6ff48..1bd421535 100644 --- a/rt/src/memory_map.rs +++ b/rt/src/memory_map.rs @@ -4,14 +4,6 @@ use rustix::mm::{ use std::io::{Error, Result as IoResult}; use std::ptr::null_mut; -/// A chunk of memory created using `mmap` and similar functions. -/// -/// The alignment of the memory mapping is equal to its size. -pub(crate) struct MemoryMap { - pub(crate) ptr: *mut u8, - pub(crate) len: usize, -} - fn mmap_options(_stack: bool) -> MapFlags { let base = MapFlags::PRIVATE; @@ -30,15 +22,22 @@ fn mmap_options(_stack: bool) -> MapFlags { base } +/// A chunk of memory created using `mmap` and similar functions. +pub(crate) struct MemoryMap { + pub(crate) ptr: *mut u8, + pub(crate) len: usize, +} + impl MemoryMap { - /// Allocates a new memory mapping. + /// Allocates a new memory mapping suitable for use as stack memory. /// - /// This method expects that `size` is a multiple of the page size. - pub(crate) fn new(size: usize, stack: bool) -> Self { + /// This method expects that `size` is a multiple of the page size. The + /// alignment of the memory mapping is equal to its size. + pub(crate) fn stack(size: usize) -> MemoryMap { // In order to align the desired region to its size, we have to allocate // more and manually align the resulting pointer. let alloc_size = size * 2; - let opts = mmap_options(stack); + let opts = mmap_options(true); let res = unsafe { mmap_anonymous( null_mut(), @@ -109,8 +108,8 @@ mod tests { #[test] fn test_new() { - let map1 = MemoryMap::new(page_size(), false); - let map2 = MemoryMap::new(page_size() * 3, false); + let map1 = MemoryMap::stack(page_size()); + let map2 = MemoryMap::stack(page_size() * 3); assert_eq!(map1.len, page_size()); assert_eq!(map2.len, page_size() * 3); @@ -118,7 +117,7 @@ mod tests { #[test] fn test_protect() { - let mut map = MemoryMap::new(page_size() * 2, false); + let mut map = MemoryMap::stack(page_size() * 2); assert!(map.protect(0, page_size()).is_ok()); } diff --git a/rt/src/process.rs b/rt/src/process.rs index 0b87ebf69..20d91659d 100644 --- a/rt/src/process.rs +++ b/rt/src/process.rs @@ -1,5 +1,5 @@ use crate::arc_without_weak::ArcWithoutWeak; -use crate::mem::{allocate, free, ClassPointer, Header}; +use crate::mem::{allocate, header_of, ClassPointer, Header}; use crate::scheduler::process::Thread; use crate::scheduler::timeouts::Timeout; use crate::stack::Stack; @@ -486,8 +486,11 @@ pub struct Process { impl Process { pub(crate) fn drop_and_deallocate(ptr: ProcessPointer) { unsafe { - drop_in_place(ptr.0.as_ptr()); - free(ptr.0.as_ptr()); + let raw = ptr.as_ptr(); + let layout = header_of(raw).class.instance_layout(); + + drop_in_place(raw); + dealloc(raw as *mut u8, layout); } } diff --git a/rt/src/runtime/general.rs b/rt/src/runtime/general.rs index 7cf2c76a3..31daacbb5 100644 --- a/rt/src/runtime/general.rs +++ b/rt/src/runtime/general.rs @@ -1,8 +1,8 @@ -use crate::mem::{free, header_of, ClassPointer}; +use crate::mem::{header_of, ClassPointer}; use crate::process::ProcessPointer; use crate::runtime::exit; use crate::runtime::process::panic; -use std::alloc::alloc; +use std::alloc::handle_alloc_error; use std::io::Error; // Taken from Rust's standard library, with some removals of platforms we don't @@ -37,38 +37,19 @@ pub unsafe extern "system" fn inko_reference_count_error( pointer: *const u8, ) { let header = header_of(pointer); - let refs = header.references(); panic( process, &format!( "can't drop a value of type '{}' as it still has {} reference(s)", - &header.class.name, refs + &header.class.name, header.references ), ); } #[no_mangle] -pub unsafe extern "system" fn inko_free(pointer: *mut u8) { - free(pointer); -} - -#[no_mangle] -pub unsafe extern "system" fn inko_alloc(class: ClassPointer) -> *mut u8 { - let ptr = alloc(class.instance_layout()); - - header_of(ptr).init(class); - ptr -} - -#[no_mangle] -pub unsafe extern "system" fn inko_alloc_atomic( - class: ClassPointer, -) -> *mut u8 { - let ptr = alloc(class.instance_layout()); - - header_of(ptr).init_atomic(class); - ptr +pub unsafe extern "system" fn inko_alloc_error(class: ClassPointer) -> ! { + handle_alloc_error(class.instance_layout()); } #[no_mangle] diff --git a/rt/src/stack.rs b/rt/src/stack.rs index c3032acb9..2f8e7f1c5 100644 --- a/rt/src/stack.rs +++ b/rt/src/stack.rs @@ -161,7 +161,7 @@ pub struct Stack { impl Stack { pub(crate) fn new(size: usize, page_size: usize) -> Self { let size = total_stack_size(size, page_size); - let mut mem = MemoryMap::new(size, true); + let mut mem = MemoryMap::stack(size); // There's nothing we can do at runtime in response to the guard page // not being set up, so we just terminate if this ever happens.