12862147048

Committed 20 Jan 2025 06:06AM UTC coverage: 92.973% (+0.04%) from 92.93%

Build # 12862147048

Build Type

push

github

Committed by

Buristan

Commit Message

Disable FMA by default. Use -Ofma or jit.opt.start("+fma") to enable.

See the discussion in the corresponding ticket for the rationale.

(cherry picked from commit de2e1ca9d)

For the modulo operation, the arm64 VM uses `fmsub` [1] instruction,
which is the fused multiply-add (FMA [2]) operation (more precisely,
multiply-sub). Hence, it may produce different results compared to the
unfused one. This patch fixes the behaviour by using the unfused
instructions by default. However, the new JIT optimization flag (fma) is
introduced to make it possible to take advantage of the FMA
optimizations.

Sergey Kaplun:
* added the description and the test for the problem

[1]: https://developer.arm.com/documentation/dui0801/g/A64-Floating-point-Instructions/FMSUB
[2]: https://en.wikipedia.org/wiki/Multiply%E2%80%93accumulate_operation

Part of tarantool/tarantool#10709

Reviewed-by: Sergey Bronnikov <sergeyb@tarantool.org>
Signed-off-by: Sergey Kaplun <skaplun@tarantool.org>

Run Details

5692 of 6029 branches covered (94.41%)

Branch coverage included in aggregate %.

3 of 3 new or added lines in 1 file covered. (100.0%)

70 existing lines in 6 files now uncovered.

21697 of 23430 relevant lines covered (92.6%)

2948914.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.79

/src/lj_dispatch.c

/*
** Instruction dispatch handling.
** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
*/

#define lj_dispatch_c
#define LUA_CORE

#include "lj_obj.h"
#include "lj_err.h"
#include "lj_buf.h"
#include "lj_func.h"
#include "lj_str.h"
#include "lj_tab.h"
#include "lj_meta.h"
#include "lj_debug.h"
#include "lj_state.h"
#include "lj_frame.h"
#include "lj_bc.h"
#include "lj_ff.h"
#include "lj_strfmt.h"
#if LJ_HASJIT
#include "lj_jit.h"
#endif
#if LJ_HASFFI
#include "lj_ccallback.h"
#endif
#include "lj_trace.h"
#include "lj_dispatch.h"
#if LJ_HASPROFILE
#include "lj_profile.h"
#endif
#include "lj_vm.h"
#include "luajit.h"

/* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);

/* -- Dispatch table management ------------------------------------------- */

#if LJ_TARGET_MIPS
#include <math.h>
LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
                                                          lua_State *co);
#if !LJ_HASJIT
#define lj_dispatch_stitch        lj_dispatch_ins
#endif
#if !LJ_HASPROFILE
#define lj_dispatch_profile        lj_dispatch_ins
#endif

#define GOTFUNC(name)        (ASMFunction)name,
static const ASMFunction dispatch_got[] = {
  GOTDEF(GOTFUNC)
};
#undef GOTFUNC
#endif

/* Initialize instruction dispatch table and hot counters. */
void lj_dispatch_init(GG_State *GG)
{
  uint32_t i;
  ASMFunction *disp = GG->dispatch;
  for (i = 0; i < GG_LEN_SDISP; i++)
    disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
  for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
    disp[i] = makeasmfunc(lj_bc_ofs[i]);
  /* The JIT engine is off by default. luaopen_jit() turns it on. */
  disp[BC_FORL] = disp[BC_IFORL];
  disp[BC_ITERL] = disp[BC_IITERL];
  disp[BC_LOOP] = disp[BC_ILOOP];
  disp[BC_FUNCF] = disp[BC_IFUNCF];
  disp[BC_FUNCV] = disp[BC_IFUNCV];
  GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
  for (i = 0; i < GG_NUM_ASMFF; i++)
    GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
#if LJ_TARGET_MIPS
  memcpy(GG->got, dispatch_got, LJ_GOT__MAX*sizeof(ASMFunction *));
#endif
}

#if LJ_HASJIT
/* Initialize hotcount table. */
void lj_dispatch_init_hotcount(global_State *g)
{
  int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
  HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
  HotCount *hotcount = G2GG(g)->hotcount;
  uint32_t i;
  for (i = 0; i < HOTCOUNT_SIZE; i++)
    hotcount[i] = start;
}
#endif

/* Internal dispatch mode bits. */
#define DISPMODE_CALL        0x01        /* Override call dispatch. */
#define DISPMODE_RET        0x02        /* Override return dispatch. */
#define DISPMODE_INS        0x04        /* Override instruction dispatch. */
#define DISPMODE_JIT        0x10        /* JIT compiler on. */
#define DISPMODE_REC        0x20        /* Recording active. */
#define DISPMODE_PROF        0x40        /* Profiling active. */

/* Update dispatch table depending on various flags. */
void lj_dispatch_update(global_State *g)
{
  uint8_t oldmode = g->dispatchmode;
  uint8_t mode = 0;
#if LJ_HASJIT
  mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
  mode |= G2J(g)->state != LJ_TRACE_IDLE ?
            (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
#endif
#if LJ_HASPROFILE
  mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
#endif
  mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
  mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
  mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
  if (oldmode != mode) {  /* Mode changed? */
    ASMFunction *disp = G2GG(g)->dispatch;
    ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;
    g->dispatchmode = mode;

    /* Hotcount if JIT is on, but not while recording. */
    if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
      f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
      f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
      f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
      f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
      f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
    } else {  /* Otherwise use the non-hotcounting instructions. */
      f_forl = disp[GG_LEN_DDISP+BC_IFORL];
      f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
      f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
      f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
      f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
    }
    /* Init static counting instruction dispatch first (may be copied below). */
    disp[GG_LEN_DDISP+BC_FORL] = f_forl;
    disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
    disp[GG_LEN_DDISP+BC_LOOP] = f_loop;

    /* Set dynamic instruction dispatch. */
    if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
      /* Need to update the whole table. */
      if (!(mode & DISPMODE_INS)) {  /* No ins dispatch? */
        /* Copy static dispatch table to dynamic dispatch table. */
        memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
        /* Overwrite with dynamic return dispatch. */
        if ((mode & DISPMODE_RET)) {
          disp[BC_RETM] = lj_vm_rethook;
          disp[BC_RET] = lj_vm_rethook;
          disp[BC_RET0] = lj_vm_rethook;
          disp[BC_RET1] = lj_vm_rethook;
        }
      } else {
        /* The recording dispatch also checks for hooks. */
        ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
                        (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
        uint32_t i;
        for (i = 0; i < GG_LEN_SDISP; i++)
          disp[i] = f;
      }
    } else if (!(mode & DISPMODE_INS)) {
      /* Otherwise set dynamic counting ins. */
      disp[BC_FORL] = f_forl;
      disp[BC_ITERL] = f_iterl;
      disp[BC_LOOP] = f_loop;
      /* Set dynamic return dispatch. */
      if ((mode & DISPMODE_RET)) {
        disp[BC_RETM] = lj_vm_rethook;
        disp[BC_RET] = lj_vm_rethook;
        disp[BC_RET0] = lj_vm_rethook;
        disp[BC_RET1] = lj_vm_rethook;
      } else {
        disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
        disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
        disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
        disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
      }
    }

    /* Set dynamic call dispatch. */
    if ((oldmode ^ mode) & DISPMODE_CALL) {  /* Update the whole table? */
      uint32_t i;
      if ((mode & DISPMODE_CALL) == 0) {  /* No call hooks? */
        for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
          disp[i] = makeasmfunc(lj_bc_ofs[i]);
      } else {
        for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
          disp[i] = lj_vm_callhook;
      }
    }
    if (!(mode & DISPMODE_CALL)) {  /* Overwrite dynamic counting ins. */
      disp[BC_FUNCF] = f_funcf;
      disp[BC_FUNCV] = f_funcv;
    }

#if LJ_HASJIT
    /* Reset hotcounts for JIT off to on transition. */
    if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
      lj_dispatch_init_hotcount(g);
#endif
  }
}

/* -- JIT mode setting ---------------------------------------------------- */

#if LJ_HASJIT
/* Set JIT mode for a single prototype. */
static void setptmode(global_State *g, GCproto *pt, int mode)
{
  if ((mode & LUAJIT_MODE_ON)) {  /* (Re-)enable JIT compilation. */
    pt->flags &= ~PROTO_NOJIT;
    lj_trace_reenableproto(pt);  /* Unpatch all ILOOP etc. bytecodes. */
  } else {  /* Flush and/or disable JIT compilation. */
    if (!(mode & LUAJIT_MODE_FLUSH))
      pt->flags |= PROTO_NOJIT;
    lj_trace_flushproto(g, pt);  /* Flush all traces of prototype. */
  }
}

/* Recursively set the JIT mode for all children of a prototype. */
static void setptmode_all(global_State *g, GCproto *pt, int mode)
{
  ptrdiff_t i;
  if (!(pt->flags & PROTO_CHILD)) return;
  for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
    GCobj *o = proto_kgc(pt, i);
    if (o->gch.gct == ~LJ_TPROTO) {
      setptmode(g, gco2pt(o), mode);
      setptmode_all(g, gco2pt(o), mode);
    }
  }
}
#endif

/* Public API function: control the JIT engine. */
int luaJIT_setmode(lua_State *L, int idx, int mode)
{
  global_State *g = G(L);
  int mm = mode & LUAJIT_MODE_MASK;
  /* Forbid JIT state change while running the trace */
  if (tvref(g->jit_base)) {
    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITMODE));
    if (g->panic) g->panic(L);
    exit(EXIT_FAILURE);
  }
  lj_trace_abort(g);  /* Abort recording on any state change. */
  /* Avoid pulling the rug from under our own feet. */
  if ((g->hookmask & HOOK_GC))
    lj_err_caller(L, LJ_ERR_NOGCMM);
  switch (mm) {
#if LJ_HASJIT
  case LUAJIT_MODE_ENGINE:
    if ((mode & LUAJIT_MODE_FLUSH)) {
      lj_trace_flushall(L);
    } else {
      if (!(mode & LUAJIT_MODE_ON))
        G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
      else
        G2J(g)->flags |= (uint32_t)JIT_F_ON;
      lj_dispatch_update(g);
    }
    break;
  case LUAJIT_MODE_FUNC:
  case LUAJIT_MODE_ALLFUNC:
  case LUAJIT_MODE_ALLSUBFUNC: {
    cTValue *tv = idx == 0 ? frame_prev(L->base-1)-LJ_FR2 :
                  idx > 0 ? L->base + (idx-1) : L->top + idx;
    GCproto *pt;
    if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
      pt = funcproto(&gcval(tv)->fn);  /* Cannot use funcV() for frame slot. */
    else if (tvisproto(tv))
      pt = protoV(tv);
    else
      return 0;  /* Failed. */
    if (mm != LUAJIT_MODE_ALLSUBFUNC)
      setptmode(g, pt, mode);
    if (mm != LUAJIT_MODE_FUNC)
      setptmode_all(g, pt, mode);
    break;
    }
  case LUAJIT_MODE_TRACE:
    if (!(mode & LUAJIT_MODE_FLUSH))
      return 0;  /* Failed. */
    lj_trace_flush(G2J(g), idx);
    break;
#else
  case LUAJIT_MODE_ENGINE:
  case LUAJIT_MODE_FUNC:
  case LUAJIT_MODE_ALLFUNC:
  case LUAJIT_MODE_ALLSUBFUNC:
    UNUSED(idx);
    if ((mode & LUAJIT_MODE_ON))
      return 0;  /* Failed. */
    break;
#endif
  case LUAJIT_MODE_WRAPCFUNC:
    if ((mode & LUAJIT_MODE_ON)) {
      if (idx != 0) {
        cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
        if (tvislightud(tv))
          g->wrapf = (lua_CFunction)lightudV(g, tv);
        else
          return 0;  /* Failed. */
      } else {
        return 0;  /* Failed. */
      }
      g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
    } else {
      g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
    }
    break;
  default:
    return 0;  /* Failed. */
  }
  return 1;  /* OK. */
}

/* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
LUA_API void LUAJIT_VERSION_SYM(void)
{
}

/* -- Hooks --------------------------------------------------------------- */

/* This function can be called asynchronously (e.g. during a signal). */
LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
{
  global_State *g = G(L);
  mask &= HOOK_EVENTMASK;
  if (func == NULL || mask == 0) { mask = 0; func = NULL; }  /* Consistency. */
  g->hookf = func;
  g->hookcount = g->hookcstart = (int32_t)count;
  g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
  lj_trace_abort(g);  /* Abort recording on any hook change. */
  lj_dispatch_update(g);
  return 1;
}

LUA_API lua_Hook lua_gethook(lua_State *L)
{
  return G(L)->hookf;
}

LUA_API int lua_gethookmask(lua_State *L)
{
  return G(L)->hookmask & HOOK_EVENTMASK;
}

LUA_API int lua_gethookcount(lua_State *L)
{
  return (int)G(L)->hookcstart;
}

/* Call a hook. */
static void callhook(lua_State *L, int event, BCLine line)
{
  global_State *g = G(L);
  lua_Hook hookf = g->hookf;
  if (hookf && !hook_active(g)) {
    lua_Debug ar;
    lj_trace_abort(g);  /* Abort recording on any hook call. */
    ar.event = event;
    ar.currentline = line;
    /* Top frame, nextframe = NULL. */
    ar.i_ci = (int)((L->base-1) - tvref(L->stack));
    lj_state_checkstack(L, 1+LUA_MINSTACK);
#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
    lj_profile_hook_enter(g);
#else
    hook_enter(g);
#endif
    hookf(L, &ar);
    lj_assertG(hook_active(g), "active hook flag removed");
    setgcref(g->cur_L, obj2gco(L));
#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
    lj_profile_hook_leave(g);
#else
    hook_leave(g);
#endif
  }
}

/* -- Dispatch callbacks -------------------------------------------------- */

/* Calculate number of used stack slots in the current frame. */
static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
{
  BCIns ins = pc[-1];
  if (bc_op(ins) == BC_UCLO)
    ins = pc[bc_j(ins)];
  switch (bc_op(ins)) {
  case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
  case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
  case BC_TSETM: return bc_a(ins) + nres-1;
  default: return pt->framesize;
  }
}

/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
{
  ERRNO_SAVE
  GCfunc *fn = curr_func(L);
  GCproto *pt = funcproto(fn);
  void *cf = cframe_raw(L->cframe);
  const BCIns *oldpc = cframe_pc(cf);
  global_State *g = G(L);
  BCReg slots;
  setcframe_pc(cf, pc);
  slots = cur_topslot(pt, pc, cframe_multres_n(cf));
  L->top = L->base + slots;  /* Fix top. */
#if LJ_HASJIT
  {
    jit_State *J = G2J(g);
    if (J->state != LJ_TRACE_IDLE) {
#ifdef LUA_USE_ASSERT
      ptrdiff_t delta = L->top - L->base;
#endif
      J->L = L;
      lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
      lj_assertG(L->top - L->base == delta,
                 "unbalanced stack after tracing of instruction");
    }
  }
#endif
  if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
    g->hookcount = g->hookcstart;
    callhook(L, LUA_HOOKCOUNT, -1);
    L->top = L->base + slots;  /* Fix top again. */
  }
  if ((g->hookmask & LUA_MASKLINE)) {
    BCPos npc = proto_bcpos(pt, pc) - 1;
    BCPos opc = proto_bcpos(pt, oldpc) - 1;
    BCLine line = lj_debug_line(pt, npc);
    if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
      callhook(L, LUA_HOOKLINE, line);
      L->top = L->base + slots;  /* Fix top again. */
    }
  }
  if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
    callhook(L, LUA_HOOKRET, -1);
  ERRNO_RESTORE
}

/* Initialize call. Ensure stack space and return # of missing parameters. */
static int call_init(lua_State *L, GCfunc *fn)
{
  if (isluafunc(fn)) {
    GCproto *pt = funcproto(fn);
    int numparams = pt->numparams;
    int gotparams = (int)(L->top - L->base);
    int need = pt->framesize;
    if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
    lj_state_checkstack(L, (MSize)need);
    numparams -= gotparams;
    return numparams >= 0 ? numparams : 0;
  } else {
    lj_state_checkstack(L, LUA_MINSTACK);
    return 0;
  }
}

/* Call dispatch. Used by call hooks, hot calls or when recording. */
ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
{
  ERRNO_SAVE
  GCfunc *fn = curr_func(L);
  BCOp op;
  global_State *g = G(L);
#if LJ_HASJIT
  jit_State *J = G2J(g);
#endif
  int missing = call_init(L, fn);
#if LJ_HASJIT
  J->L = L;
  if ((uintptr_t)pc & 1) {  /* Marker for hot call. */
#ifdef LUA_USE_ASSERT
    ptrdiff_t delta = L->top - L->base;
#endif
    pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
    lj_trace_hot(J, pc);
    lj_assertG(L->top - L->base == delta,
               "unbalanced stack after hot call");
    goto out;
  } else if (J->state != LJ_TRACE_IDLE &&
             !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
#ifdef LUA_USE_ASSERT
    ptrdiff_t delta = L->top - L->base;
#endif
    /* Record the FUNC* bytecodes, too. */
    lj_trace_ins(J, pc-1);  /* The interpreter bytecode PC is offset by 1. */
    lj_assertG(L->top - L->base == delta,
               "unbalanced stack after hot instruction");
  }
#endif
  if ((g->hookmask & LUA_MASKCALL)) {
    int i;
    for (i = 0; i < missing; i++)  /* Add missing parameters. */
      setnilV(L->top++);
    callhook(L, LUA_HOOKCALL, -1);
    /* Preserve modifications of missing parameters by lua_setlocal(). */
    while (missing-- > 0 && tvisnil(L->top - 1))
      L->top--;
  }
#if LJ_HASJIT
out:
#endif
  op = bc_op(pc[-1]);  /* Get FUNC* op. */
#if LJ_HASJIT
  /* Use the non-hotcounting variants if JIT is off or while recording. */
  if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
      (op == BC_FUNCF || op == BC_FUNCV))
    op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
#endif
  ERRNO_RESTORE
  return makeasmfunc(lj_bc_ofs[op]);  /* Return static dispatch target. */
}

#if LJ_HASJIT
/* Stitch a new trace. */
void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
{
  ERRNO_SAVE
  lua_State *L = J->L;
  void *cf = cframe_raw(L->cframe);
  const BCIns *oldpc = cframe_pc(cf);
  setcframe_pc(cf, pc);
  /* Before dispatch, have to bias PC by 1. */
  L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
  lj_trace_stitch(J, pc-1);  /* Point to the CALL instruction. */
  setcframe_pc(cf, oldpc);
  ERRNO_RESTORE
}
#endif

#if LJ_HASPROFILE
/* Profile dispatch. */
void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
{
  ERRNO_SAVE
  GCfunc *fn = curr_func(L);
  GCproto *pt = funcproto(fn);
  void *cf = cframe_raw(L->cframe);
  const BCIns *oldpc = cframe_pc(cf);
  global_State *g;
  setcframe_pc(cf, pc);
  L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
  lj_profile_interpreter(L);
  setcframe_pc(cf, oldpc);
  g = G(L);
  setgcref(g->cur_L, obj2gco(L));
  setvmstate(g, INTERP);
  ERRNO_RESTORE
}
#endif


1	/*
2	** Instruction dispatch handling.
3	** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
4	*/
5
6	#define lj_dispatch_c
7	#define LUA_CORE
8
9	#include "lj_obj.h"
10	#include "lj_err.h"
11	#include "lj_buf.h"
12	#include "lj_func.h"
13	#include "lj_str.h"
14	#include "lj_tab.h"
15	#include "lj_meta.h"
16	#include "lj_debug.h"
17	#include "lj_state.h"
18	#include "lj_frame.h"
19	#include "lj_bc.h"
20	#include "lj_ff.h"
21	#include "lj_strfmt.h"
22	#if LJ_HASJIT
23	#include "lj_jit.h"
24	#endif
25	#if LJ_HASFFI
26	#include "lj_ccallback.h"
27	#endif
28	#include "lj_trace.h"
29	#include "lj_dispatch.h"
30	#if LJ_HASPROFILE
31	#include "lj_profile.h"
32	#endif
33	#include "lj_vm.h"
34	#include "luajit.h"
35
36	/* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
37	LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
38
39	/* -- Dispatch table management ------------------------------------------- */
40
41	#if LJ_TARGET_MIPS
42	#include <math.h>
43	LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
44	lua_State *co);
45	#if !LJ_HASJIT
46	#define lj_dispatch_stitch lj_dispatch_ins
47	#endif
48	#if !LJ_HASPROFILE
49	#define lj_dispatch_profile lj_dispatch_ins
50	#endif
51
52	#define GOTFUNC(name) (ASMFunction)name,
53	static const ASMFunction dispatch_got[] = {
54	GOTDEF(GOTFUNC)
55	};
56	#undef GOTFUNC
57	#endif
58
59	/* Initialize instruction dispatch table and hot counters. */
60	void lj_dispatch_init(GG_State *GG)	362✔
61	{
62	uint32_t i;	362✔
63	ASMFunction *disp = GG->dispatch;	362✔
64	for (i = 0; i < GG_LEN_SDISP; i++)	32,580✔
65	disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);	32,218✔
66	for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)	23,892✔
67	disp[i] = makeasmfunc(lj_bc_ofs[i]);	23,530✔
68	/* The JIT engine is off by default. luaopen_jit() turns it on. */
69	disp[BC_FORL] = disp[BC_IFORL];	362✔
70	disp[BC_ITERL] = disp[BC_IITERL];	362✔
71	disp[BC_LOOP] = disp[BC_ILOOP];	362✔
72	disp[BC_FUNCF] = disp[BC_IFUNCF];	362✔
73	disp[BC_FUNCV] = disp[BC_IFUNCV];	362✔
74	GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);	362✔
75	for (i = 0; i < GG_NUM_ASMFF; i++)	20,996✔
76	GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);	20,634✔
77	#if LJ_TARGET_MIPS
78	memcpy(GG->got, dispatch_got, LJ_GOT__MAXsizeof(ASMFunction ));
79	#endif
80	}	362✔
81
82	#if LJ_HASJIT
83	/* Initialize hotcount table. */
84	void lj_dispatch_init_hotcount(global_State *g)	893✔
85	{
86	int32_t hotloop = G2J(g)->param[JIT_P_hotloop];	893✔
87	HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);	893✔
88	HotCount *hotcount = G2GG(g)->hotcount;	893✔
89	uint32_t i;	893✔
90	for (i = 0; i < HOTCOUNT_SIZE; i++)	58,045✔
91	hotcount[i] = start;	57,152✔
92	}	161✔
93	#endif
94
95	/* Internal dispatch mode bits. */
96	#define DISPMODE_CALL 0x01 /* Override call dispatch. */
97	#define DISPMODE_RET 0x02 /* Override return dispatch. */
98	#define DISPMODE_INS 0x04 /* Override instruction dispatch. */
99	#define DISPMODE_JIT 0x10 /* JIT compiler on. */
100	#define DISPMODE_REC 0x20 /* Recording active. */
101	#define DISPMODE_PROF 0x40 /* Profiling active. */
102
103	/* Update dispatch table depending on various flags. */
104	void lj_dispatch_update(global_State *g)	44,225✔
105	{
106	uint8_t oldmode = g->dispatchmode;	44,225✔
107	uint8_t mode = 0;	44,225✔
108	#if LJ_HASJIT
109	mode \|= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;	44,225✔
110	mode \|= G2J(g)->state != LJ_TRACE_IDLE ?	44,225✔
111	(DISPMODE_REC\|DISPMODE_INS\|DISPMODE_CALL) : 0;
112	#endif
113	#if LJ_HASPROFILE
114	mode \|= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF\|DISPMODE_INS) : 0;	44,225✔
115	#endif
116	mode \|= (g->hookmask & (LUA_MASKLINE\|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;	44,225✔
117	mode \|= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;	44,225✔
118	mode \|= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;	44,225✔
119	if (oldmode != mode) { /* Mode changed? */	44,225✔
120	ASMFunction *disp = G2GG(g)->dispatch;	44,011✔
121	ASMFunction f_forl, f_iterl, f_loop, f_funcf, f_funcv;	44,011✔
122	g->dispatchmode = mode;	44,011✔
123
124	/* Hotcount if JIT is on, but not while recording. */
125	if ((mode & (DISPMODE_JIT\|DISPMODE_REC)) == DISPMODE_JIT) {	44,011✔
126	f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);	22,240✔
127	f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);	22,240✔
128	f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);	22,240✔
129	f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);	22,240✔
130	f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);	22,240✔
131	} else { /* Otherwise use the non-hotcounting instructions. */
132	f_forl = disp[GG_LEN_DDISP+BC_IFORL];	21,771✔
133	f_iterl = disp[GG_LEN_DDISP+BC_IITERL];	21,771✔
134	f_loop = disp[GG_LEN_DDISP+BC_ILOOP];	21,771✔
135	f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);	21,771✔
136	f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);	21,771✔
137	}
138	/* Init static counting instruction dispatch first (may be copied below). */
139	disp[GG_LEN_DDISP+BC_FORL] = f_forl;	44,011✔
140	disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;	44,011✔
141	disp[GG_LEN_DDISP+BC_LOOP] = f_loop;	44,011✔
142
143	/* Set dynamic instruction dispatch. */
144	if ((oldmode ^ mode) & (DISPMODE_PROF\|DISPMODE_REC\|DISPMODE_INS)) {	44,011✔
145	/* Need to update the whole table. */
146	if (!(mode & DISPMODE_INS)) { /* No ins dispatch? */	42,552✔
147	/* Copy static dispatch table to dynamic dispatch table. */
148	memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));	20,018✔
149	/* Overwrite with dynamic return dispatch. */
150	if ((mode & DISPMODE_RET)) {	20,018✔
151	disp[BC_RETM] = lj_vm_rethook;	1✔
152	disp[BC_RET] = lj_vm_rethook;	1✔
153	disp[BC_RET0] = lj_vm_rethook;	1✔
154	disp[BC_RET1] = lj_vm_rethook;	1✔
155	}
156	} else {
157	/* The recording dispatch also checks for hooks. */
158	ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :	22,534✔
159	(mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;	22,333✔
160	uint32_t i;	22,534✔
161	for (i = 0; i < GG_LEN_SDISP; i++)	2,028,060✔
162	disp[i] = f;	2,005,526✔
163	}
164	} else if (!(mode & DISPMODE_INS)) {	1,459✔
165	/* Otherwise set dynamic counting ins. */
166	disp[BC_FORL] = f_forl;	1,458✔
167	disp[BC_ITERL] = f_iterl;	1,458✔
168	disp[BC_LOOP] = f_loop;	1,458✔
169	/* Set dynamic return dispatch. */
170	if ((mode & DISPMODE_RET)) {	1,458✔
171	disp[BC_RETM] = lj_vm_rethook;	2✔
172	disp[BC_RET] = lj_vm_rethook;	2✔
173	disp[BC_RET0] = lj_vm_rethook;	2✔
174	disp[BC_RET1] = lj_vm_rethook;	2✔
175	} else {
176	disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];	1,456✔
177	disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];	1,456✔
178	disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];	1,456✔
179	disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];	1,456✔
180	}
181	}
182
183	/* Set dynamic call dispatch. */
184	if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */	44,011✔
185	uint32_t i;	42,095✔
186	if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */	42,095✔
187	for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)	1,389,036✔
188	disp[i] = makeasmfunc(lj_bc_ofs[i]);	1,367,990✔
189	} else {
190	for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)	1,389,234✔
191	disp[i] = lj_vm_callhook;	1,368,185✔
192	}
193	}
194	if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */	44,011✔
195	disp[BC_FUNCF] = f_funcf;	22,952✔
196	disp[BC_FUNCV] = f_funcv;	22,952✔
197	}
198
199	#if LJ_HASJIT
200	/* Reset hotcounts for JIT off to on transition. */
201	if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))	44,011✔
202	lj_dispatch_init_hotcount(g);	732✔
203	#endif
204	}
205	}	44,225✔
206
207	/* -- JIT mode setting ---------------------------------------------------- */
208
209	#if LJ_HASJIT
210	/* Set JIT mode for a single prototype. */
211	static void setptmode(global_State g, GCproto pt, int mode)	270✔
212	{
213	if ((mode & LUAJIT_MODE_ON)) { /* (Re-)enable JIT compilation. */	270✔
214	pt->flags &= ~PROTO_NOJIT;	51✔
215	lj_trace_reenableproto(pt); /* Unpatch all ILOOP etc. bytecodes. */	51✔
216	} else { /* Flush and/or disable JIT compilation. */
217	if (!(mode & LUAJIT_MODE_FLUSH))	219✔
218	pt->flags \|= PROTO_NOJIT;	219✔
219	lj_trace_flushproto(g, pt); /* Flush all traces of prototype. */	219✔
220	}
221	}	270✔
222
223	/* Recursively set the JIT mode for all children of a prototype. */
224	static void setptmode_all(global_State g, GCproto pt, int mode)	246✔
225	{
226	ptrdiff_t i;	246✔
227	if (!(pt->flags & PROTO_CHILD)) return;	246✔
228	for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {	1,485✔
229	GCobj *o = proto_kgc(pt, i);	1,442✔
230	if (o->gch.gct == ~LJ_TPROTO) {	1,442✔
231	setptmode(g, gco2pt(o), mode);	169✔
232	setptmode_all(g, gco2pt(o), mode);	169✔
233	}
234	}
235	}
236	#endif
237
238	/* Public API function: control the JIT engine. */
239	int luaJIT_setmode(lua_State *L, int idx, int mode)	1,194✔
240	{
241	global_State *g = G(L);	1,194✔
242	int mm = mode & LUAJIT_MODE_MASK;	1,194✔
243	/* Forbid JIT state change while running the trace */
244	if (tvref(g->jit_base)) {	1,194✔
245	setstrV(L, L->top++, lj_err_str(L, LJ_ERR_JITMODE));	1✔
246	if (g->panic) g->panic(L);	1✔
247	exit(EXIT_FAILURE);	1✔
248	}
249	lj_trace_abort(g); /* Abort recording on any state change. */	1,193✔
250	/* Avoid pulling the rug from under our own feet. */
251	if ((g->hookmask & HOOK_GC))	1,193✔
252	lj_err_caller(L, LJ_ERR_NOGCMM);	×
253	switch (mm) {	1,193✔
254	#if LJ_HASJIT
255	case LUAJIT_MODE_ENGINE:	1,074✔
256	if ((mode & LUAJIT_MODE_FLUSH)) {	1,074✔
257	lj_trace_flushall(L);	298✔
258	} else {
259	if (!(mode & LUAJIT_MODE_ON))	776✔
260	G2J(g)->flags &= ~(uint32_t)JIT_F_ON;	395✔
261	else
262	G2J(g)->flags \|= (uint32_t)JIT_F_ON;	381✔
263	lj_dispatch_update(g);	776✔
264	}
265	break;
266	case LUAJIT_MODE_FUNC:	101✔
267	case LUAJIT_MODE_ALLFUNC:
268	case LUAJIT_MODE_ALLSUBFUNC: {
269	cTValue *tv = idx == 0 ? frame_prev(L->base-1)-LJ_FR2 :	101✔
270	idx > 0 ? L->base + (idx-1) : L->top + idx;	92✔
271	GCproto *pt;	101✔
272	if ((idx == 0 \|\| tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))	101✔
273	pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */	101✔
UNCOV 274	else if (tvisproto(tv))	×
UNCOV 275	pt = protoV(tv);	×
276	else
277	return 0; /* Failed. */
278	if (mm != LUAJIT_MODE_ALLSUBFUNC)	101✔
279	setptmode(g, pt, mode);	101✔
280	if (mm != LUAJIT_MODE_FUNC)	101✔
281	setptmode_all(g, pt, mode);	77✔
282	break;
283	}
284	case LUAJIT_MODE_TRACE:	6✔
285	if (!(mode & LUAJIT_MODE_FLUSH))	6✔
286	return 0; /* Failed. */
287	lj_trace_flush(G2J(g), idx);	6✔
288	break;	6✔
289	#else
290	case LUAJIT_MODE_ENGINE:
291	case LUAJIT_MODE_FUNC:
292	case LUAJIT_MODE_ALLFUNC:
293	case LUAJIT_MODE_ALLSUBFUNC:
294	UNUSED(idx);
295	if ((mode & LUAJIT_MODE_ON))
296	return 0; /* Failed. */
297	break;
298	#endif
299	case LUAJIT_MODE_WRAPCFUNC:	12✔
300	if ((mode & LUAJIT_MODE_ON)) {	12✔
301	if (idx != 0) {	6✔
302	cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;	6✔
303	if (tvislightud(tv))	6✔
304	g->wrapf = (lua_CFunction)lightudV(g, tv);	6✔
305	else
306	return 0; /* Failed. */
307	} else {
308	return 0; /* Failed. */
309	}
310	g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);	6✔
311	} else {
312	g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);	6✔
313	}
314	break;
315	default:
316	return 0; /* Failed. */
317	}
318	return 1; /* OK. */
319	}
320
321	/* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
322	LUA_API void LUAJIT_VERSION_SYM(void)	351✔
323	{
324	}	351✔
325
326	/* -- Hooks --------------------------------------------------------------- */
327
328	/* This function can be called asynchronously (e.g. during a signal). */
329	LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)	67✔
330	{
331	global_State *g = G(L);	67✔
332	mask &= HOOK_EVENTMASK;	67✔
333	if (func == NULL \|\| mask == 0) { mask = 0; func = NULL; } /* Consistency. */	67✔
334	g->hookf = func;	67✔
335	g->hookcount = g->hookcstart = (int32_t)count;	67✔
336	g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) \| mask);	67✔
337	lj_trace_abort(g); /* Abort recording on any hook change. */	67✔
338	lj_dispatch_update(g);	67✔
339	return 1;	67✔
340	}
341
342	LUA_API lua_Hook lua_gethook(lua_State *L)	132✔
343	{
344	return G(L)->hookf;	132✔
345	}
346
347	LUA_API int lua_gethookmask(lua_State *L)	132✔
348	{
349	return G(L)->hookmask & HOOK_EVENTMASK;	132✔
350	}
351
352	LUA_API int lua_gethookcount(lua_State *L)	132✔
353	{
354	return (int)G(L)->hookcstart;	132✔
355	}
356
357	/* Call a hook. */
358	static void callhook(lua_State *L, int event, BCLine line)	4,290,148✔
359	{
360	global_State *g = G(L);	4,290,148✔
361	lua_Hook hookf = g->hookf;	4,290,148✔
362	if (hookf && !hook_active(g)) {	4,290,148✔
363	lua_Debug ar;	4,288,791✔
364	lj_trace_abort(g); /* Abort recording on any hook call. */	4,288,791✔
365	ar.event = event;	4,288,791✔
366	ar.currentline = line;	4,288,791✔
367	/* Top frame, nextframe = NULL. */
368	ar.i_ci = (int)((L->base-1) - tvref(L->stack));	4,288,791✔
369	lj_state_checkstack(L, 1+LUA_MINSTACK);	4,288,791✔
370	#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
371	lj_profile_hook_enter(g);
372	#else
373	hook_enter(g);	4,288,787✔
374	#endif
375	hookf(L, &ar);	4,288,787✔
376	lj_assertG(hook_active(g), "active hook flag removed");	4,288,783✔
377	setgcref(g->cur_L, obj2gco(L));	4,288,783✔
378	#if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
379	lj_profile_hook_leave(g);
380	#else
381	hook_leave(g);	4,288,783✔
382	#endif
383	}
384	}	4,290,140✔
385
386	/* -- Dispatch callbacks -------------------------------------------------- */
387
388	/* Calculate number of used stack slots in the current frame. */
389	static BCReg cur_topslot(GCproto pt, const BCIns pc, uint32_t nres)
390	{
391	BCIns ins = pc[-1];
392	if (bc_op(ins) == BC_UCLO)
393	ins = pc[bc_j(ins)];
394	switch (bc_op(ins)) {
395	case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
396	case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
397	case BC_TSETM: return bc_a(ins) + nres-1;
398	default: return pt->framesize;
399	}
400	}
401
402	/* Instruction dispatch. Used by instr/line/return hooks or when recording. */
403	void LJ_FASTCALL lj_dispatch_ins(lua_State L, const BCIns pc)	24,003,289✔
404	{
405	ERRNO_SAVE	24,003,289✔
406	GCfunc *fn = curr_func(L);	24,003,289✔
407	GCproto *pt = funcproto(fn);	24,003,289✔
408	void *cf = cframe_raw(L->cframe);	24,003,289✔
409	const BCIns *oldpc = cframe_pc(cf);	24,003,289✔
410	global_State *g = G(L);	24,003,289✔
411	BCReg slots;	24,003,289✔
412	setcframe_pc(cf, pc);	24,003,289✔
413	slots = cur_topslot(pt, pc, cframe_multres_n(cf));	24,003,289✔
414	L->top = L->base + slots; /* Fix top. */	24,003,289✔
415	#if LJ_HASJIT
416	{
417	jit_State *J = G2J(g);	24,003,289✔
418	if (J->state != LJ_TRACE_IDLE) {	24,003,289✔
419	#ifdef LUA_USE_ASSERT
420	ptrdiff_t delta = L->top - L->base;
421	#endif
422	J->L = L;	6,496,490✔
423	lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */	6,496,490✔
424	lj_assertG(L->top - L->base == delta,	24,003,288✔
425	"unbalanced stack after tracing of instruction");
426	}
427	}
428	#endif
429	if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {	24,003,288✔
430	g->hookcount = g->hookcstart;	3,462✔
431	callhook(L, LUA_HOOKCOUNT, -1);	3,462✔
432	L->top = L->base + slots; /* Fix top again. */	3,460✔
433	}
434	if ((g->hookmask & LUA_MASKLINE)) {	24,003,286✔
435	BCPos npc = proto_bcpos(pt, pc) - 1;	17,507,668✔
436	BCPos opc = proto_bcpos(pt, oldpc) - 1;	17,507,668✔
437	BCLine line = lj_debug_line(pt, npc);	17,507,668✔
438	if (pc <= oldpc \|\| opc >= pt->sizebc \|\| line != lj_debug_line(pt, opc)) {	17,507,668✔
439	callhook(L, LUA_HOOKLINE, line);	4,285,053✔
440	L->top = L->base + slots; /* Fix top again. */	4,285,047✔
441	}
442	}
443	if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))	24,003,280✔
444	callhook(L, LUA_HOOKRET, -1);	11✔
445	ERRNO_RESTORE	24,003,280✔
446	}	24,003,280✔
447
448	/* Initialize call. Ensure stack space and return # of missing parameters. */
449	static int call_init(lua_State L, GCfunc fn)
450	{
451	if (isluafunc(fn)) {
452	GCproto *pt = funcproto(fn);
453	int numparams = pt->numparams;
454	int gotparams = (int)(L->top - L->base);
455	int need = pt->framesize;
456	if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
457	lj_state_checkstack(L, (MSize)need);
458	numparams -= gotparams;
459	return numparams >= 0 ? numparams : 0;
460	} else {
461	lj_state_checkstack(L, LUA_MINSTACK);
462	return 0;
463	}
464	}
465
466	/* Call dispatch. Used by call hooks, hot calls or when recording. */
467	ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State L, const BCIns pc)	1,120,208✔
468	{
469	ERRNO_SAVE	1,120,208✔
470	GCfunc *fn = curr_func(L);	1,120,208✔
471	BCOp op;	1,120,208✔
472	global_State *g = G(L);	1,120,208✔
473	#if LJ_HASJIT
474	jit_State *J = G2J(g);	1,120,208✔
475	#endif
476	int missing = call_init(L, fn);	1,120,208✔
477	#if LJ_HASJIT
478	J->L = L;	1,120,208✔
479	if ((uintptr_t)pc & 1) { /* Marker for hot call. */	1,120,208✔
480	#ifdef LUA_USE_ASSERT
481	ptrdiff_t delta = L->top - L->base;
482	#endif
483	pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);	16,145✔
484	lj_trace_hot(J, pc);	16,145✔
485	lj_assertG(L->top - L->base == delta,	16,145✔
486	"unbalanced stack after hot call");
487	goto out;	16,145✔
488	} else if (J->state != LJ_TRACE_IDLE &&	1,104,063✔
489	!(g->hookmask & (HOOK_GC\|HOOK_VMEVENT))) {	1,102,444✔
490	#ifdef LUA_USE_ASSERT
491	ptrdiff_t delta = L->top - L->base;
492	#endif
493	/* Record the FUNC* bytecodes, too. */
494	lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */	1,096,108✔
495	lj_assertG(L->top - L->base == delta,	1,104,063✔
496	"unbalanced stack after hot instruction");
497	}
498	#endif
499	if ((g->hookmask & LUA_MASKCALL)) {	1,104,063✔
500	int i;
501	for (i = 0; i < missing; i++) /* Add missing parameters. */	1,636✔
502	setnilV(L->top++);	14✔
503	callhook(L, LUA_HOOKCALL, -1);	1,622✔
504	/* Preserve modifications of missing parameters by lua_setlocal(). */
505	while (missing-- > 0 && tvisnil(L->top - 1))	1,635✔
506	L->top--;	13✔
507	}
508	#if LJ_HASJIT
509	out:	1,104,063✔
510	#endif
511	op = bc_op(pc[-1]); /* Get FUNC* op. */	1,120,208✔
512	#if LJ_HASJIT
513	/* Use the non-hotcounting variants if JIT is off or while recording. */
514	if ((!(J->flags & JIT_F_ON) \|\| J->state != LJ_TRACE_IDLE) &&	1,120,208✔
515	(op == BC_FUNCF \|\| op == BC_FUNCV))	1,099,969✔
516	op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);	774,354✔
517	#endif
518	ERRNO_RESTORE	1,120,208✔
519	return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */	1,120,208✔
520	}
521
522	#if LJ_HASJIT
523	/* Stitch a new trace. */
524	void LJ_FASTCALL lj_dispatch_stitch(jit_State J, const BCIns pc)	137✔
525	{
526	ERRNO_SAVE	137✔
527	lua_State *L = J->L;	137✔
528	void *cf = cframe_raw(L->cframe);	137✔
529	const BCIns *oldpc = cframe_pc(cf);	137✔
530	setcframe_pc(cf, pc);	137✔
531	/* Before dispatch, have to bias PC by 1. */
532	L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));	137✔
533	lj_trace_stitch(J, pc-1); /* Point to the CALL instruction. */	137✔
534	setcframe_pc(cf, oldpc);	137✔
535	ERRNO_RESTORE	137✔
536	}	137✔
537	#endif
538
539	#if LJ_HASPROFILE
540	/* Profile dispatch. */
541	void LJ_FASTCALL lj_dispatch_profile(lua_State L, const BCIns pc)	96✔
542	{
543	ERRNO_SAVE	96✔
544	GCfunc *fn = curr_func(L);	96✔
545	GCproto *pt = funcproto(fn);	96✔
546	void *cf = cframe_raw(L->cframe);	96✔
547	const BCIns *oldpc = cframe_pc(cf);	96✔
548	global_State *g;	96✔
549	setcframe_pc(cf, pc);	96✔
550	L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));	96✔
551	lj_profile_interpreter(L);	96✔
552	setcframe_pc(cf, oldpc);	96✔
553	g = G(L);	96✔
554	setgcref(g->cur_L, obj2gco(L));	96✔
555	setvmstate(g, INTERP);	96✔
556	ERRNO_RESTORE	96✔
557	}	96✔
558	#endif
559

tarantool / luajit / 12862147048

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous