[vm] New async/async* implementation in JIT mode

The new implementation is based on suspend/resume stubs and doesn't
use desugaring of async functions on kernel AST.

Previously, new implementation of async/async* was only supported in
AOT mode. This change adds all necessary bits for the JIT mode:

 * Suspending variable-length frames (for unoptimized code).
 * Handling of Code and pool pointers in Dart stack frames.
 * OSR.
 * Deoptimization.
 * Hot reload.
 * Debugger.

The new implementation is not enabled in JIT mode yet.

Design doc: go/compact-async-await.
TEST=ci

Issue: #48378
Change-Id: I477d6684bdce7cbc1edb179ae2271ff598b7dcc5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/246081
Reviewed-by: Martin Kustermann <kustermann@google.com>
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Slava Egorov <vegorov@google.com>

pkg/front_end/test/fasta/testing/suite.dart

@@ -66,7 +66,6 @@ import 'package:front_end/src/fasta/util/parser_ast.dart'
 import 'package:front_end/src/fasta/util/parser_ast_helper.dart';
 import 'package:kernel/ast.dart'
     show
-        AwaitExpression,
         BasicLiteral,
         Class,
         Component,
@@ -87,9 +86,7 @@ import 'package:kernel/ast.dart'
         TreeNode,
         UnevaluatedConstant,
         VariableDeclaration,
-        Version,
-        Visitor,
-        VisitorVoidMixin;
+        Version;
 import 'package:kernel/binary/ast_to_binary.dart' show BinaryPrinter;
 import 'package:kernel/class_hierarchy.dart' show ClassHierarchy;
 import 'package:kernel/core_types.dart' show CoreTypes;
@@ -2421,13 +2418,6 @@ class Transform extends Step<ComponentResult, ComponentResult, FastaContext> {
           backendTarget.performModularTransformations = false;
         }
       }
-      List<String> errors = VerifyTransformed.verify(component, backendTarget);
-      if (errors.isNotEmpty) {
-        return new Result<ComponentResult>(
-            result,
-            context.expectationSet["TransformVerificationError"],
-            errors.join('\n'));
-      }
       if (backendTarget is TestTarget &&
           backendTarget.hasGlobalTransformation) {
         component =
@@ -2492,34 +2482,6 @@ class Verify extends Step<ComponentResult, ComponentResult, FastaContext> {
   }
 }
 
-/// Visitor that checks that the component has been transformed properly.
-// TODO(johnniwinther): Add checks for all nodes that are unsupported after
-// transformation.
-class VerifyTransformed extends Visitor<void> with VisitorVoidMixin {
-  final Target target;
-  List<String> errors = [];
-
-  VerifyTransformed(this.target);
-
-  @override
-  void defaultNode(Node node) {
-    node.visitChildren(this);
-  }
-
-  @override
-  void visitAwaitExpression(AwaitExpression node) {
-    if (target is VmTarget) {
-      errors.add("ERROR: Untransformed await expression: $node");
-    }
-  }
-
-  static List<String> verify(Component component, Target target) {
-    VerifyTransformed visitor = new VerifyTransformed(target);
-    component.accept(visitor);
-    return visitor.errors;
-  }
-}
-
 mixin TestTarget on Target {
   bool performModularTransformations = false;
 

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -1587,7 +1587,7 @@ void Assembler::CheckCodePointer() {
                           target::Instructions::HeaderSize() - kHeapObjectTag;
   mov(R0, Operand(PC));
   AddImmediate(R0, -offset);
-  ldr(IP, FieldAddress(CODE_REG, target::Code::saved_instructions_offset()));
+  ldr(IP, FieldAddress(CODE_REG, target::Code::instructions_offset()));
   cmp(R0, Operand(IP));
   b(&instructions_ok, EQ);
   bkpt(1);
@@ -3308,6 +3308,10 @@ void Assembler::Ret(Condition cond /* = AL */) {
   READS_RETURN_ADDRESS_FROM_LR(bx(LR, cond));
 }
 
+void Assembler::SetReturnAddress(Register value) {
+  RESTORES_RETURN_ADDRESS_FROM_REGISTER_TO_LR(MoveRegister(LR, value));
+}
+
 void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
   // Reserve space for arguments and align frame before entering
   // the C++ world.

runtime/vm/compiler/assembler/assembler_arm.h

@@ -1300,6 +1300,11 @@ class Assembler : public AssemblerBase {
   void EnterFrame(RegList regs, intptr_t frame_space);
   void LeaveFrame(RegList regs, bool allow_pop_pc = false);
   void Ret(Condition cond = AL);
+
+  // Sets the return address to [value] as if there was a call.
+  // On ARM sets LR.
+  void SetReturnAddress(Register value);
+
   void ReserveAlignedFrameSpace(intptr_t frame_space);
 
   // In debug mode, this generates code to check that:

runtime/vm/compiler/assembler/assembler_arm64.cc

@@ -1540,7 +1540,7 @@ void Assembler::CheckCodePointer() {
   const intptr_t entry_offset =
       CodeSize() + target::Instructions::HeaderSize() - kHeapObjectTag;
   adr(R0, Immediate(-entry_offset));
-  ldr(TMP, FieldAddress(CODE_REG, target::Code::saved_instructions_offset()));
+  ldr(TMP, FieldAddress(CODE_REG, target::Code::instructions_offset()));
   cmp(R0, Operand(TMP));
   b(&instructions_ok, EQ);
   brk(1);
@@ -1583,6 +1583,10 @@ void Assembler::RestoreCSP() {
   mov(CSP, SP);
 }
 
+void Assembler::SetReturnAddress(Register value) {
+  RESTORES_RETURN_ADDRESS_FROM_REGISTER_TO_LR(MoveRegister(LR, value));
+}
+
 void Assembler::EnterFrame(intptr_t frame_size) {
   SPILLS_LR_TO_FRAME(PushPair(FP, LR));  // low: FP, high: LR.
   mov(FP, SP);

runtime/vm/compiler/assembler/assembler_arm64.h

@@ -2121,6 +2121,10 @@ class Assembler : public AssemblerBase {
   void LeaveFrame();
   void Ret() { ret(); }
 
+  // Sets the return address to [value] as if there was a call.
+  // On ARM64 sets LR.
+  void SetReturnAddress(Register value);
+
   // Emit code to transition between generated mode and native mode.
   //
   // These require and ensure that CSP and SP are equal and aligned and require

runtime/vm/compiler/assembler/assembler_ia32.h

@@ -573,6 +573,13 @@ class Assembler : public AssemblerBase {
    */
 
   void Ret() { ret(); }
+
+  // Sets the return address to [value] as if there was a call.
+  // On IA32 pushes [value].
+  void SetReturnAddress(Register value) {
+    PushRegister(value);
+  }
+
   void CompareRegisters(Register a, Register b);
   void CompareObjectRegisters(Register a, Register b) {
     CompareRegisters(a, b);

runtime/vm/compiler/assembler/assembler_riscv.cc

@@ -2727,6 +2727,9 @@ void Assembler::AddImmediate(Register rd,
                              Register rs1,
                              intx_t imm,
                              OperandSize sz) {
+  if ((imm == 0) && (rd == rs1)) {
+    return;
+  }
   if (IsITypeImm(imm)) {
     addi(rd, rs1, imm);
   } else {
@@ -3646,7 +3649,7 @@ void Assembler::CheckCodePointer() {
   intx_t hi = (imm - lo) << (XLEN - 32) >> (XLEN - 32);
   auipc(TMP, hi);
   addi(TMP, TMP, lo);
-  lx(TMP2, FieldAddress(CODE_REG, target::Code::saved_instructions_offset()));
+  lx(TMP2, FieldAddress(CODE_REG, target::Code::instructions_offset()));
   beq(TMP, TMP2, &instructions_ok, kNearJump);
   ebreak();
   Bind(&instructions_ok);

runtime/vm/compiler/assembler/assembler_riscv.h

@@ -1231,6 +1231,12 @@ class Assembler : public MicroAssembler {
   void LeaveFrame();
   void Ret() { ret(); }
 
+  // Sets the return address to [value] as if there was a call.
+  // On RISC-V sets RA.
+  void SetReturnAddress(Register value) {
+    mv(RA, value);
+  }
+
   // Emit code to transition between generated mode and native mode.
   //
   // These require and ensure that CSP and SP are equal and aligned and require

runtime/vm/compiler/assembler/assembler_x64.cc

@@ -2069,7 +2069,7 @@ void Assembler::CheckCodePointer() {
     leaq(RAX, Address::AddressRIPRelative(-header_to_rip_offset));
     ASSERT(CodeSize() == (header_to_rip_offset - header_to_entry_offset));
   }
-  cmpq(RAX, FieldAddress(CODE_REG, target::Code::saved_instructions_offset()));
+  cmpq(RAX, FieldAddress(CODE_REG, target::Code::instructions_offset()));
   j(EQUAL, &instructions_ok);
   int3();
   Bind(&instructions_ok);

runtime/vm/compiler/assembler/assembler_x64.h

@@ -711,6 +711,13 @@ class Assembler : public AssemblerBase {
 
   // Methods for High-level operations and implemented on all architectures.
   void Ret() { ret(); }
+
+  // Sets the return address to [value] as if there was a call.
+  // On X64 pushes [value].
+  void SetReturnAddress(Register value) {
+    PushRegister(value);
+  }
+
   void CompareRegisters(Register a, Register b);
   void CompareObjectRegisters(Register a, Register b) { OBJ(cmp)(a, b); }
   void BranchIf(Condition condition,

runtime/vm/compiler/backend/constant_propagator.cc

@@ -1456,6 +1456,10 @@ void ConstantPropagator::VisitCall1ArgStub(Call1ArgStubInstr* instr) {
   SetValue(instr, non_constant_);
 }
 
+void ConstantPropagator::VisitSuspend(SuspendInstr* instr) {
+  SetValue(instr, non_constant_);
+}
+
 void ConstantPropagator::VisitLoadThread(LoadThreadInstr* instr) {
   SetValue(instr, non_constant_);
 }

runtime/vm/compiler/backend/flow_graph_compiler_arm.cc

@@ -373,7 +373,8 @@ void FlowGraphCompiler::EmitPrologue() {
       Register value_reg = slot_index == args_desc_slot ? ARGS_DESC_REG : R0;
       __ StoreToOffset(value_reg, FP, slot_index * compiler::target::kWordSize);
     }
-  } else if (parsed_function().suspend_state_var() != nullptr) {
+  } else if (parsed_function().suspend_state_var() != nullptr &&
+             !flow_graph().IsCompiledForOsr()) {
     // Initialize synthetic :suspend_state variable early
     // as it may be accessed by GC and exception handling before
     // InitSuspendableFunction stub is called.

runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc

@@ -363,7 +363,8 @@ void FlowGraphCompiler::EmitPrologue() {
           slot_index == args_desc_slot ? ARGS_DESC_REG : NULL_REG;
       __ StoreToOffset(value_reg, FP, slot_index * kWordSize);
     }
-  } else if (parsed_function().suspend_state_var() != nullptr) {
+  } else if (parsed_function().suspend_state_var() != nullptr &&
+             !flow_graph().IsCompiledForOsr()) {
     // Initialize synthetic :suspend_state variable early
     // as it may be accessed by GC and exception handling before
     // InitSuspendableFunction stub is called.

runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc

@@ -450,7 +450,8 @@ void FlowGraphCompiler::EmitPrologue() {
       Register value_reg = slot_index == args_desc_slot ? ARGS_DESC_REG : EAX;
       __ movl(compiler::Address(EBP, slot_index * kWordSize), value_reg);
     }
-  } else if (parsed_function().suspend_state_var() != nullptr) {
+  } else if (parsed_function().suspend_state_var() != nullptr &&
+             !flow_graph().IsCompiledForOsr()) {
     // Initialize synthetic :suspend_state variable early
     // as it may be accessed by GC and exception handling before
     // InitSuspendableFunction stub is called.

runtime/vm/compiler/backend/flow_graph_compiler_riscv.cc

@@ -357,7 +357,8 @@ void FlowGraphCompiler::EmitPrologue() {
       __ StoreToOffset(value_reg, SP,
                        (slot_index + fp_to_sp_delta) * kWordSize);
     }
-  } else if (parsed_function().suspend_state_var() != nullptr) {
+  } else if (parsed_function().suspend_state_var() != nullptr &&
+             !flow_graph().IsCompiledForOsr()) {
     // Initialize synthetic :suspend_state variable early
     // as it may be accessed by GC and exception handling before
     // InitSuspendableFunction stub is called.

runtime/vm/compiler/backend/flow_graph_compiler_x64.cc

@@ -369,7 +369,8 @@ void FlowGraphCompiler::EmitPrologue() {
       Register value_reg = slot_index == args_desc_slot ? ARGS_DESC_REG : RAX;
       __ movq(compiler::Address(RBP, slot_index * kWordSize), value_reg);
     }
-  } else if (parsed_function().suspend_state_var() != nullptr) {
+  } else if (parsed_function().suspend_state_var() != nullptr &&
+             !flow_graph().IsCompiledForOsr()) {
     // Initialize synthetic :suspend_state variable early
     // as it may be accessed by GC and exception handling before
     // InitSuspendableFunction stub is called.

runtime/vm/compiler/backend/il.cc

@@ -7298,10 +7298,6 @@ LocationSummary* Call1ArgStubInstr::MakeLocationSummary(Zone* zone,
       locs->set_in(0, Location::RegisterLocation(
                           InitSuspendableFunctionStubABI::kTypeArgsReg));
       break;
-    case StubId::kAwait:
-    case StubId::kYieldAsyncStar:
-      locs->set_in(0, Location::RegisterLocation(SuspendStubABI::kArgumentReg));
-      break;
   }
   locs->set_out(0, Location::RegisterLocation(CallingConventions::kReturnReg));
   return locs;
@@ -7314,12 +7310,34 @@ void Call1ArgStubInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
     case StubId::kInitAsync:
       stub = object_store->init_async_stub();
       break;
-    case StubId::kAwait:
-      stub = object_store->await_stub();
-      break;
     case StubId::kInitAsyncStar:
       stub = object_store->init_async_star_stub();
       break;
+  }
+  compiler->GenerateStubCall(source(), stub, UntaggedPcDescriptors::kOther,
+                             locs(), deopt_id(), env());
+}
+
+LocationSummary* SuspendInstr::MakeLocationSummary(Zone* zone, bool opt) const {
+  const intptr_t kNumInputs = 1;
+  const intptr_t kNumTemps = 0;
+  LocationSummary* locs = new (zone)
+      LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kCall);
+  locs->set_in(0, Location::RegisterLocation(SuspendStubABI::kArgumentReg));
+  locs->set_out(0, Location::RegisterLocation(CallingConventions::kReturnReg));
+  return locs;
+}
+
+void SuspendInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  // Use deopt_id as a yield index.
+  compiler->EmitYieldPositionMetadata(source(), deopt_id());
+
+  ObjectStore* object_store = compiler->isolate_group()->object_store();
+  Code& stub = Code::ZoneHandle(compiler->zone());
+  switch (stub_id_) {
+    case StubId::kAwait:
+      stub = object_store->await_stub();
+      break;
     case StubId::kYieldAsyncStar:
       stub = object_store->yield_async_star_stub();
       break;
@@ -7328,18 +7346,18 @@ void Call1ArgStubInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
                              locs(), deopt_id(), env());
 
 #if defined(TARGET_ARCH_X64) || defined(TARGET_ARCH_IA32)
-  if ((stub_id_ == StubId::kAwait) || (stub_id_ == StubId::kYieldAsyncStar)) {
-    // On x86 (X64 and IA32) mismatch between calls and returns
-    // significantly regresses performance. So suspend stub
-    // does not return directly to the caller. Instead, a small
-    // epilogue is generated right after the call to suspend stub,
-    // and resume stub adjusts resume PC to skip this epilogue.
-    const intptr_t start = compiler->assembler()->CodeSize();
-    __ LeaveFrame();
-    __ ret();
-    RELEASE_ASSERT(compiler->assembler()->CodeSize() - start ==
-                   SuspendStubABI::kResumePcDistance);
-  }
+  // On x86 (X64 and IA32) mismatch between calls and returns
+  // significantly regresses performance. So suspend stub
+  // does not return directly to the caller. Instead, a small
+  // epilogue is generated right after the call to suspend stub,
+  // and resume stub adjusts resume PC to skip this epilogue.
+  const intptr_t start = compiler->assembler()->CodeSize();
+  __ LeaveFrame();
+  __ ret();
+  RELEASE_ASSERT(compiler->assembler()->CodeSize() - start ==
+                 SuspendStubABI::kResumePcDistance);
+  compiler->EmitCallsiteMetadata(source(), resume_deopt_id(),
+                                 UntaggedPcDescriptors::kOther, locs(), env());
 #endif
 }