8752988570

Committed 19 Apr 2024 11:39AM UTC coverage: 92.62% (-0.03%) from 92.647%

Build # 8752988570

Build Type

push

github

Committed by

Buristan

Commit Message

Use generic trace error for OOM during trace stitching.

Thanks to Sergey Kaplun.

(cherry picked from commit b8b49bf39)

The previous commit doesn't handle the case when the error code is
`LUA_ERRMEM`. This patch adds a workaround by using the generic error
message.

Sergey Kaplun:
* added the description and the test for the problem

Part of tarantool/tarantool#9924

Run Details

5662 of 6019 branches covered (94.07%)

Branch coverage included in aggregate %.

1 of 1 new or added line in 1 file covered. (100.0%)

42 existing lines in 3 files now uncovered.

21610 of 23426 relevant lines covered (92.25%)

2925817.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.09

/src/lj_ir.c

/*
** SSA IR (Intermediate Representation) emitter.
** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
*/

#define lj_ir_c
#define LUA_CORE

/* For pointers to libc/libm functions. */
#include <stdio.h>
#include <math.h>

#include "lj_obj.h"

#if LJ_HASJIT

#include "lj_gc.h"
#include "lj_buf.h"
#include "lj_str.h"
#include "lj_tab.h"
#include "lj_ir.h"
#include "lj_jit.h"
#include "lj_ircall.h"
#include "lj_iropt.h"
#include "lj_trace.h"
#if LJ_HASFFI
#include "lj_ctype.h"
#include "lj_cdata.h"
#include "lj_carith.h"
#endif
#include "lj_vm.h"
#include "lj_strscan.h"
#include "lj_strfmt.h"
#include "lj_lib.h"

/* Some local macros to save typing. Undef'd at the end. */
#define IR(ref)                        (&J->cur.ir[(ref)])
#define fins                        (&J->fold.ins)

/* Pass IR on to next optimization in chain (FOLD). */
#define emitir(ot, a, b)        (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))

/* -- IR tables ----------------------------------------------------------- */

/* IR instruction modes. */
LJ_DATADEF const uint8_t lj_ir_mode[IR__MAX+1] = {
IRDEF(IRMODE)
  0
};

/* IR type sizes. */
LJ_DATADEF const uint8_t lj_ir_type_size[IRT__MAX+1] = {
#define IRTSIZE(name, size)        size,
IRTDEF(IRTSIZE)
#undef IRTSIZE
  0
};

/* C call info for CALL* instructions. */
LJ_DATADEF const CCallInfo lj_ir_callinfo[] = {
#define IRCALLCI(cond, name, nargs, kind, type, flags) \
  { (ASMFunction)IRCALLCOND_##cond(name), \
    (nargs)|(CCI_CALL_##kind)|(IRT_##type<<CCI_OTSHIFT)|(flags) },
IRCALLDEF(IRCALLCI)
#undef IRCALLCI
  { NULL, 0 }
};

/* -- IR emitter ---------------------------------------------------------- */

/* Grow IR buffer at the top. */
void LJ_FASTCALL lj_ir_growtop(jit_State *J)
{
  IRIns *baseir = J->irbuf + J->irbotlim;
  MSize szins = J->irtoplim - J->irbotlim;
  if (szins) {
    baseir = (IRIns *)lj_mem_realloc(J->L, baseir, szins*sizeof(IRIns),
                                     2*szins*sizeof(IRIns));
    J->irtoplim = J->irbotlim + 2*szins;
  } else {
    baseir = (IRIns *)lj_mem_realloc(J->L, NULL, 0, LJ_MIN_IRSZ*sizeof(IRIns));
    J->irbotlim = REF_BASE - LJ_MIN_IRSZ/4;
    J->irtoplim = J->irbotlim + LJ_MIN_IRSZ;
  }
  J->cur.ir = J->irbuf = baseir - J->irbotlim;
}

/* Grow IR buffer at the bottom or shift it up. */
static void lj_ir_growbot(jit_State *J)
{
  IRIns *baseir = J->irbuf + J->irbotlim;
  MSize szins = J->irtoplim - J->irbotlim;
  lj_assertJ(szins != 0, "zero IR size");
  lj_assertJ(J->cur.nk == J->irbotlim || J->cur.nk-1 == J->irbotlim,
             "unexpected IR growth");
  if (J->cur.nins + (szins >> 1) < J->irtoplim) {
    /* More than half of the buffer is free on top: shift up by a quarter. */
    MSize ofs = szins >> 2;
    memmove(baseir + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns));
    J->irbotlim -= ofs;
    J->irtoplim -= ofs;
    J->cur.ir = J->irbuf = baseir - J->irbotlim;
  } else {
    /* Double the buffer size, but split the growth amongst top/bottom. */
    IRIns *newbase = lj_mem_newt(J->L, 2*szins*sizeof(IRIns), IRIns);
    MSize ofs = szins >= 256 ? 128 : (szins >> 1);  /* Limit bottom growth. */
    memcpy(newbase + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns));
    lj_mem_free(G(J->L), baseir, szins*sizeof(IRIns));
    J->irbotlim -= ofs;
    J->irtoplim = J->irbotlim + 2*szins;
    J->cur.ir = J->irbuf = newbase - J->irbotlim;
  }
}

/* Emit IR without any optimizations. */
TRef LJ_FASTCALL lj_ir_emit(jit_State *J)
{
  IRRef ref = lj_ir_nextins(J);
  IRIns *ir = IR(ref);
  IROp op = fins->o;
  ir->prev = J->chain[op];
  J->chain[op] = (IRRef1)ref;
  ir->o = op;
  ir->op1 = fins->op1;
  ir->op2 = fins->op2;
  J->guardemit.irt |= fins->t.irt;
  return TREF(ref, irt_t((ir->t = fins->t)));
}

/* Emit call to a C function. */
TRef lj_ir_call(jit_State *J, IRCallID id, ...)
{
  const CCallInfo *ci = &lj_ir_callinfo[id];
  uint32_t n = CCI_NARGS(ci);
  TRef tr = TREF_NIL;
  va_list argp;
  va_start(argp, id);
  if ((ci->flags & CCI_L)) n--;
  if (n > 0)
    tr = va_arg(argp, IRRef);
  while (n-- > 1)
    tr = emitir(IRT(IR_CARG, IRT_NIL), tr, va_arg(argp, IRRef));
  va_end(argp);
  if (CCI_OP(ci) == IR_CALLS)
    J->needsnap = 1;  /* Need snapshot after call with side effect. */
  return emitir(CCI_OPTYPE(ci), tr, id);
}

/* Load field of type t from GG_State + offset. Must be 32 bit aligned. */
LJ_FUNC TRef lj_ir_ggfload(jit_State *J, IRType t, uintptr_t ofs)
{
  lj_assertJ((ofs & 3) == 0, "unaligned GG_State field offset");
  ofs >>= 2;
  lj_assertJ(ofs >= IRFL__MAX && ofs <= 0x3ff,
             "GG_State field offset breaks 10 bit FOLD key limit");
  lj_ir_set(J, IRT(IR_FLOAD, t), REF_NIL, ofs);
  return lj_opt_fold(J);
}

/* -- Interning of constants ---------------------------------------------- */

/*
** IR instructions for constants are kept between J->cur.nk >= ref < REF_BIAS.
** They are chained like all other instructions, but grow downwards.
** The are interned (like strings in the VM) to facilitate reference
** comparisons. The same constant must get the same reference.
*/

/* Get ref of next IR constant and optionally grow IR.
** Note: this may invalidate all IRIns *!
*/
static LJ_AINLINE IRRef ir_nextk(jit_State *J)
{
  IRRef ref = J->cur.nk;
  if (LJ_UNLIKELY(ref <= J->irbotlim)) lj_ir_growbot(J);
  J->cur.nk = --ref;
  return ref;
}

/* Get ref of next 64 bit IR constant and optionally grow IR.
** Note: this may invalidate all IRIns *!
*/
static LJ_AINLINE IRRef ir_nextk64(jit_State *J)
{
  IRRef ref = J->cur.nk - 2;
  lj_assertJ(J->state != LJ_TRACE_ASM, "bad JIT state");
  if (LJ_UNLIKELY(ref < J->irbotlim)) lj_ir_growbot(J);
  J->cur.nk = ref;
  return ref;
}

#if LJ_GC64
#define ir_nextkgc ir_nextk64
#else
#define ir_nextkgc ir_nextk
#endif

/* Intern int32_t constant. */
TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k)
{
  IRIns *ir, *cir = J->cur.ir;
  IRRef ref;
  for (ref = J->chain[IR_KINT]; ref; ref = cir[ref].prev)
    if (cir[ref].i == k)
      goto found;
  ref = ir_nextk(J);
  ir = IR(ref);
  ir->i = k;
  ir->t.irt = IRT_INT;
  ir->o = IR_KINT;
  ir->prev = J->chain[IR_KINT];
  J->chain[IR_KINT] = (IRRef1)ref;
found:
  return TREF(ref, IRT_INT);
}

/* Intern 64 bit constant, given by its 64 bit pattern. */
TRef lj_ir_k64(jit_State *J, IROp op, uint64_t u64)
{
  IRIns *ir, *cir = J->cur.ir;
  IRRef ref;
  IRType t = op == IR_KNUM ? IRT_NUM : IRT_I64;
  for (ref = J->chain[op]; ref; ref = cir[ref].prev)
    if (ir_k64(&cir[ref])->u64 == u64)
      goto found;
  ref = ir_nextk64(J);
  ir = IR(ref);
  ir[1].tv.u64 = u64;
  ir->t.irt = t;
  ir->o = op;
  ir->op12 = 0;
  ir->prev = J->chain[op];
  J->chain[op] = (IRRef1)ref;
found:
  return TREF(ref, t);
}

/* Intern FP constant, given by its 64 bit pattern. */
TRef lj_ir_knum_u64(jit_State *J, uint64_t u64)
{
  return lj_ir_k64(J, IR_KNUM, u64);
}

/* Intern 64 bit integer constant. */
TRef lj_ir_kint64(jit_State *J, uint64_t u64)
{
  return lj_ir_k64(J, IR_KINT64, u64);
}

/* Check whether a number is int and return it. -0 is NOT considered an int. */
static int numistrueint(lua_Number n, int32_t *kp)
{
  int32_t k = lj_num2int(n);
  if (n == (lua_Number)k) {
    if (kp) *kp = k;
    if (k == 0) {  /* Special check for -0. */
      TValue tv;
      setnumV(&tv, n);
      if (tv.u32.hi != 0)
        return 0;
    }
    return 1;
  }
  return 0;
}

/* Intern number as int32_t constant if possible, otherwise as FP constant. */
TRef lj_ir_knumint(jit_State *J, lua_Number n)
{
  int32_t k;
  if (numistrueint(n, &k))
    return lj_ir_kint(J, k);
  else
    return lj_ir_knum(J, n);
}

/* Intern GC object "constant". */
TRef lj_ir_kgc(jit_State *J, GCobj *o, IRType t)
{
  IRIns *ir, *cir = J->cur.ir;
  IRRef ref;
  lj_assertJ(!isdead(J2G(J), o), "interning of dead GC object");
  for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev)
    if (ir_kgc(&cir[ref]) == o)
      goto found;
  ref = ir_nextkgc(J);
  ir = IR(ref);
  /* NOBARRIER: Current trace is a GC root. */
  ir->op12 = 0;
  setgcref(ir[LJ_GC64].gcr, o);
  ir->t.irt = (uint8_t)t;
  ir->o = IR_KGC;
  ir->prev = J->chain[IR_KGC];
  J->chain[IR_KGC] = (IRRef1)ref;
found:
  return TREF(ref, t);
}

/* Allocate GCtrace constant placeholder (no interning). */
TRef lj_ir_ktrace(jit_State *J)
{
  IRRef ref = ir_nextkgc(J);
  IRIns *ir = IR(ref);
  lj_assertJ(irt_toitype_(IRT_P64) == LJ_TTRACE, "mismatched type mapping");
  ir->t.irt = IRT_P64;
  ir->o = LJ_GC64 ? IR_KNUM : IR_KNULL;  /* Not IR_KGC yet, but same size. */
  ir->op12 = 0;
  ir->prev = 0;
  return TREF(ref, IRT_P64);
}

/* Intern pointer constant. */
TRef lj_ir_kptr_(jit_State *J, IROp op, void *ptr)
{
  IRIns *ir, *cir = J->cur.ir;
  IRRef ref;
#if LJ_64 && !LJ_GC64
  lj_assertJ((void *)(uintptr_t)u32ptr(ptr) == ptr, "out-of-range GC pointer");
#endif
  for (ref = J->chain[op]; ref; ref = cir[ref].prev)
    if (ir_kptr(&cir[ref]) == ptr)
      goto found;
#if LJ_GC64
  ref = ir_nextk64(J);
#else
  ref = ir_nextk(J);
#endif
  ir = IR(ref);
  ir->op12 = 0;
  setmref(ir[LJ_GC64].ptr, ptr);
  ir->t.irt = IRT_PGC;
  ir->o = op;
  ir->prev = J->chain[op];
  J->chain[op] = (IRRef1)ref;
found:
  return TREF(ref, IRT_PGC);
}

/* Intern typed NULL constant. */
TRef lj_ir_knull(jit_State *J, IRType t)
{
  IRIns *ir, *cir = J->cur.ir;
  IRRef ref;
  for (ref = J->chain[IR_KNULL]; ref; ref = cir[ref].prev)
    if (irt_t(cir[ref].t) == t)
      goto found;
  ref = ir_nextk(J);
  ir = IR(ref);
  ir->i = 0;
  ir->t.irt = (uint8_t)t;
  ir->o = IR_KNULL;
  ir->prev = J->chain[IR_KNULL];
  J->chain[IR_KNULL] = (IRRef1)ref;
found:
  return TREF(ref, t);
}

/* Intern key slot. */
TRef lj_ir_kslot(jit_State *J, TRef key, IRRef slot)
{
  IRIns *ir, *cir = J->cur.ir;
  IRRef2 op12 = IRREF2((IRRef1)key, (IRRef1)slot);
  IRRef ref;
  /* Const part is not touched by CSE/DCE, so 0-65535 is ok for IRMlit here. */
  lj_assertJ(tref_isk(key) && slot == (IRRef)(IRRef1)slot,
             "out-of-range key/slot");
  for (ref = J->chain[IR_KSLOT]; ref; ref = cir[ref].prev)
    if (cir[ref].op12 == op12)
      goto found;
  ref = ir_nextk(J);
  ir = IR(ref);
  ir->op12 = op12;
  ir->t.irt = IRT_P32;
  ir->o = IR_KSLOT;
  ir->prev = J->chain[IR_KSLOT];
  J->chain[IR_KSLOT] = (IRRef1)ref;
found:
  return TREF(ref, IRT_P32);
}

/* -- Access to IR constants ---------------------------------------------- */

/* Copy value of IR constant. */
void lj_ir_kvalue(lua_State *L, TValue *tv, const IRIns *ir)
{
  UNUSED(L);
  lj_assertL(ir->o != IR_KSLOT, "unexpected KSLOT");  /* Common mistake. */
  switch (ir->o) {
  case IR_KPRI: setpriV(tv, irt_toitype(ir->t)); break;
  case IR_KINT: setintV(tv, ir->i); break;
  case IR_KGC: setgcV(L, tv, ir_kgc(ir), irt_toitype(ir->t)); break;
  case IR_KPTR: case IR_KKPTR:
    setnumV(tv, (lua_Number)(uintptr_t)ir_kptr(ir));
    break;
  case IR_KNULL: setintV(tv, 0); break;
  case IR_KNUM: setnumV(tv, ir_knum(ir)->n); break;
#if LJ_HASFFI
  case IR_KINT64: {
    GCcdata *cd = lj_cdata_new_(L, CTID_INT64, 8);
    *(uint64_t *)cdataptr(cd) = ir_kint64(ir)->u64;
    setcdataV(L, tv, cd);
    break;
    }
#endif
  default: lj_assertL(0, "bad IR constant op %d", ir->o); break;
  }
}

/* -- Convert IR operand types -------------------------------------------- */

/* Convert from string to number. */
TRef LJ_FASTCALL lj_ir_tonumber(jit_State *J, TRef tr)
{
  if (!tref_isnumber(tr)) {
    if (tref_isstr(tr))
      tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
    else
      lj_trace_err(J, LJ_TRERR_BADTYPE);
  }
  return tr;
}

/* Convert from integer or string to number. */
TRef LJ_FASTCALL lj_ir_tonum(jit_State *J, TRef tr)
{
  if (!tref_isnum(tr)) {
    if (tref_isinteger(tr))
      tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);
    else if (tref_isstr(tr))
      tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);
    else
      lj_trace_err(J, LJ_TRERR_BADTYPE);
  }
  return tr;
}

/* Convert from integer or number to string. */
TRef LJ_FASTCALL lj_ir_tostr(jit_State *J, TRef tr)
{
  if (!tref_isstr(tr)) {
    if (!tref_isnumber(tr))
      lj_trace_err(J, LJ_TRERR_BADTYPE);
    tr = emitir(IRT(IR_TOSTR, IRT_STR), tr,
                tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
  }
  return tr;
}

/* -- Miscellaneous IR ops ------------------------------------------------ */

/* Evaluate numeric comparison. */
int lj_ir_numcmp(lua_Number a, lua_Number b, IROp op)
{
  switch (op) {
  case IR_EQ: return (a == b);
  case IR_NE: return (a != b);
  case IR_LT: return (a < b);
  case IR_GE: return (a >= b);
  case IR_LE: return (a <= b);
  case IR_GT: return (a > b);
  case IR_ULT: return !(a >= b);
  case IR_UGE: return !(a < b);
  case IR_ULE: return !(a > b);
  case IR_UGT: return !(a <= b);
  default: lj_assertX(0, "bad IR op %d", op); return 0;
  }
}

/* Evaluate string comparison. */
int lj_ir_strcmp(GCstr *a, GCstr *b, IROp op)
{
  int res = lj_str_cmp(a, b);
  switch (op) {
  case IR_LT: return (res < 0);
  case IR_GE: return (res >= 0);
  case IR_LE: return (res <= 0);
  case IR_GT: return (res > 0);
  default: lj_assertX(0, "bad IR op %d", op); return 0;
  }
}

/* Rollback IR to previous state. */
void lj_ir_rollback(jit_State *J, IRRef ref)
{
  IRRef nins = J->cur.nins;
  while (nins > ref) {
    IRIns *ir;
    nins--;
    ir = IR(nins);
    J->chain[ir->o] = ir->prev;
  }
  J->cur.nins = nins;
}

#undef IR
#undef fins
#undef emitir

#endif

1	/*
2	** SSA IR (Intermediate Representation) emitter.
3	** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
4	*/
5
6	#define lj_ir_c
7	#define LUA_CORE
8
9	/* For pointers to libc/libm functions. */
10	#include <stdio.h>
11	#include <math.h>
12
13	#include "lj_obj.h"
14
15	#if LJ_HASJIT
16
17	#include "lj_gc.h"
18	#include "lj_buf.h"
19	#include "lj_str.h"
20	#include "lj_tab.h"
21	#include "lj_ir.h"
22	#include "lj_jit.h"
23	#include "lj_ircall.h"
24	#include "lj_iropt.h"
25	#include "lj_trace.h"
26	#if LJ_HASFFI
27	#include "lj_ctype.h"
28	#include "lj_cdata.h"
29	#include "lj_carith.h"
30	#endif
31	#include "lj_vm.h"
32	#include "lj_strscan.h"
33	#include "lj_strfmt.h"
34	#include "lj_lib.h"
35
36	/* Some local macros to save typing. Undef'd at the end. */
37	#define IR(ref) (&J->cur.ir[(ref)])
38	#define fins (&J->fold.ins)
39
40	/* Pass IR on to next optimization in chain (FOLD). */
41	#define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
42
43	/* -- IR tables ----------------------------------------------------------- */
44
45	/* IR instruction modes. */
46	LJ_DATADEF const uint8_t lj_ir_mode[IR__MAX+1] = {
47	IRDEF(IRMODE)
48	0
49	};
50
51	/* IR type sizes. */
52	LJ_DATADEF const uint8_t lj_ir_type_size[IRT__MAX+1] = {
53	#define IRTSIZE(name, size) size,
54	IRTDEF(IRTSIZE)
55	#undef IRTSIZE
56	0
57	};
58
59	/* C call info for CALL* instructions. */
60	LJ_DATADEF const CCallInfo lj_ir_callinfo[] = {
61	#define IRCALLCI(cond, name, nargs, kind, type, flags) \
62	{ (ASMFunction)IRCALLCOND_##cond(name), \
63	(nargs)\|(CCI_CALL_##kind)\|(IRT_##type<<CCI_OTSHIFT)\|(flags) },
64	IRCALLDEF(IRCALLCI)
65	#undef IRCALLCI
66	{ NULL, 0 }
67	};
68
69	/* -- IR emitter ---------------------------------------------------------- */
70
71	/* Grow IR buffer at the top. */
72	void LJ_FASTCALL lj_ir_growtop(jit_State *J)	206✔
73	{
74	IRIns *baseir = J->irbuf + J->irbotlim;	206✔
75	MSize szins = J->irtoplim - J->irbotlim;	206✔
76	if (szins) {	206✔
77	baseir = (IRIns )lj_mem_realloc(J->L, baseir, szinssizeof(IRIns),	161✔
78	2szinssizeof(IRIns));	81✔
79	J->irtoplim = J->irbotlim + 2*szins;	80✔
80	} else {
81	baseir = (IRIns )lj_mem_realloc(J->L, NULL, 0, LJ_MIN_IRSZsizeof(IRIns));	125✔
82	J->irbotlim = REF_BASE - LJ_MIN_IRSZ/4;	125✔
83	J->irtoplim = J->irbotlim + LJ_MIN_IRSZ;	125✔
84	}
85	J->cur.ir = J->irbuf = baseir - J->irbotlim;	205✔
86	}	205✔
87
88	/* Grow IR buffer at the bottom or shift it up. */
89	static void lj_ir_growbot(jit_State *J)	307✔
90	{
91	IRIns *baseir = J->irbuf + J->irbotlim;	307✔
92	MSize szins = J->irtoplim - J->irbotlim;	307✔
93	lj_assertJ(szins != 0, "zero IR size");	307✔
94	lj_assertJ(J->cur.nk == J->irbotlim \|\| J->cur.nk-1 == J->irbotlim,	307✔
95	"unexpected IR growth");
96	if (J->cur.nins + (szins >> 1) < J->irtoplim) {	307✔
97	/* More than half of the buffer is free on top: shift up by a quarter. */
98	MSize ofs = szins >> 2;	84✔
99	memmove(baseir + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns));	84✔
100	J->irbotlim -= ofs;	84✔
101	J->irtoplim -= ofs;	84✔
102	J->cur.ir = J->irbuf = baseir - J->irbotlim;	84✔
103	} else {
104	/* Double the buffer size, but split the growth amongst top/bottom. */
105	IRIns newbase = lj_mem_newt(J->L, 2szins*sizeof(IRIns), IRIns);	223✔
106	MSize ofs = szins >= 256 ? 128 : (szins >> 1); /* Limit bottom growth. */	223✔
107	memcpy(newbase + ofs, baseir, (J->cur.nins - J->irbotlim)*sizeof(IRIns));	223✔
108	lj_mem_free(G(J->L), baseir, szins*sizeof(IRIns));	223✔
109	J->irbotlim -= ofs;	223✔
110	J->irtoplim = J->irbotlim + 2*szins;	223✔
111	J->cur.ir = J->irbuf = newbase - J->irbotlim;	223✔
112	}
113	}	307✔
114
115	/* Emit IR without any optimizations. */
116	TRef LJ_FASTCALL lj_ir_emit(jit_State *J)	1,802,542✔
117	{
118	IRRef ref = lj_ir_nextins(J);	1,802,542✔
119	IRIns *ir = IR(ref);	1,802,542✔
120	IROp op = fins->o;	1,802,542✔
121	ir->prev = J->chain[op];	1,802,542✔
122	J->chain[op] = (IRRef1)ref;	1,802,542✔
123	ir->o = op;	1,802,542✔
124	ir->op1 = fins->op1;	1,802,542✔
125	ir->op2 = fins->op2;	1,802,542✔
126	J->guardemit.irt \|= fins->t.irt;	1,802,542✔
127	return TREF(ref, irt_t((ir->t = fins->t)));	1,802,542✔
128	}
129
130	/* Emit call to a C function. */
131	TRef lj_ir_call(jit_State *J, IRCallID id, ...)	520✔
132	{
133	const CCallInfo *ci = &lj_ir_callinfo[id];	520✔
134	uint32_t n = CCI_NARGS(ci);	520✔
135	TRef tr = TREF_NIL;	520✔
136	va_list argp;	520✔
137	va_start(argp, id);	520✔
138	if ((ci->flags & CCI_L)) n--;	520✔
139	if (n > 0)	520✔
140	tr = va_arg(argp, IRRef);	519✔
141	while (n-- > 1)	1,498✔
142	tr = emitir(IRT(IR_CARG, IRT_NIL), tr, va_arg(argp, IRRef));	978✔
143	va_end(argp);	520✔
144	if (CCI_OP(ci) == IR_CALLS)	520✔
145	J->needsnap = 1; /* Need snapshot after call with side effect. */	130✔
146	return emitir(CCI_OPTYPE(ci), tr, id);	520✔
147	}
148
149	/* Load field of type t from GG_State + offset. Must be 32 bit aligned. */
150	LJ_FUNC TRef lj_ir_ggfload(jit_State *J, IRType t, uintptr_t ofs)	448✔
151	{
152	lj_assertJ((ofs & 3) == 0, "unaligned GG_State field offset");	448✔
153	ofs >>= 2;	448✔
154	lj_assertJ(ofs >= IRFL__MAX && ofs <= 0x3ff,	448✔
155	"GG_State field offset breaks 10 bit FOLD key limit");
156	lj_ir_set(J, IRT(IR_FLOAD, t), REF_NIL, ofs);	448✔
157	return lj_opt_fold(J);	448✔
158	}
159
160	/* -- Interning of constants ---------------------------------------------- */
161
162	/*
163	** IR instructions for constants are kept between J->cur.nk >= ref < REF_BIAS.
164	** They are chained like all other instructions, but grow downwards.
165	** The are interned (like strings in the VM) to facilitate reference
166	** comparisons. The same constant must get the same reference.
167	*/
168
169	/* Get ref of next IR constant and optionally grow IR.
170	** Note: this may invalidate all IRIns *!
171	*/
172	static LJ_AINLINE IRRef ir_nextk(jit_State *J)	353,834✔
173	{
174	IRRef ref = J->cur.nk;	353,834✔
175	if (LJ_UNLIKELY(ref <= J->irbotlim)) lj_ir_growbot(J);	87✔
176	J->cur.nk = --ref;	353,834✔
177	return ref;	353,834✔
178	}
179
180	/* Get ref of next 64 bit IR constant and optionally grow IR.
181	** Note: this may invalidate all IRIns *!
182	*/
183	static LJ_AINLINE IRRef ir_nextk64(jit_State *J)	805,430✔
184	{
185	IRRef ref = J->cur.nk - 2;	805,430✔
186	lj_assertJ(J->state != LJ_TRACE_ASM, "bad JIT state");	805,430✔
187	if (LJ_UNLIKELY(ref < J->irbotlim)) lj_ir_growbot(J);	220✔
188	J->cur.nk = ref;	805,430✔
189	return ref;	805,430✔
190	}
191
192	#if LJ_GC64
193	#define ir_nextkgc ir_nextk64
194	#else
195	#define ir_nextkgc ir_nextk
196	#endif
197
198	/* Intern int32_t constant. */
199	TRef LJ_FASTCALL lj_ir_kint(jit_State *J, int32_t k)	1,906,089✔
200	{
201	IRIns ir, cir = J->cur.ir;	1,906,089✔
202	IRRef ref;	1,906,089✔
203	for (ref = J->chain[IR_KINT]; ref; ref = cir[ref].prev)	8,125,426✔
204	if (cir[ref].i == k)	7,972,292✔
205	goto found;	1,752,955✔
206	ref = ir_nextk(J);	153,134✔
207	ir = IR(ref);	153,134✔
208	ir->i = k;	153,134✔
209	ir->t.irt = IRT_INT;	153,134✔
210	ir->o = IR_KINT;	153,134✔
211	ir->prev = J->chain[IR_KINT];	153,134✔
212	J->chain[IR_KINT] = (IRRef1)ref;	153,134✔
213	found:	1,906,089✔
214	return TREF(ref, IRT_INT);	1,906,089✔
215	}
216
217	/* Intern 64 bit constant, given by its 64 bit pattern. */
218	TRef lj_ir_k64(jit_State *J, IROp op, uint64_t u64)	1,143,405✔
219	{
220	IRIns ir, cir = J->cur.ir;	1,143,405✔
221	IRRef ref;	1,143,405✔
222	IRType t = op == IR_KNUM ? IRT_NUM : IRT_I64;	1,143,405✔
223	for (ref = J->chain[op]; ref; ref = cir[ref].prev)	8,526,916✔
224	if (ir_k64(&cir[ref])->u64 == u64)	8,251,463✔
225	goto found;	867,952✔
226	ref = ir_nextk64(J);	275,453✔
227	ir = IR(ref);	275,453✔
228	ir[1].tv.u64 = u64;	275,453✔
229	ir->t.irt = t;	275,453✔
230	ir->o = op;	275,453✔
231	ir->op12 = 0;	275,453✔
232	ir->prev = J->chain[op];	275,453✔
233	J->chain[op] = (IRRef1)ref;	275,453✔
234	found:	1,143,405✔
235	return TREF(ref, t);	1,143,405✔
236	}
237
238	/* Intern FP constant, given by its 64 bit pattern. */
239	TRef lj_ir_knum_u64(jit_State *J, uint64_t u64)	339,850✔
240	{
241	return lj_ir_k64(J, IR_KNUM, u64);	339,850✔
242	}
243
244	/* Intern 64 bit integer constant. */
245	TRef lj_ir_kint64(jit_State *J, uint64_t u64)	109,541✔
246	{
247	return lj_ir_k64(J, IR_KINT64, u64);	109,541✔
248	}
249
250	/* Check whether a number is int and return it. -0 is NOT considered an int. */
251	static int numistrueint(lua_Number n, int32_t *kp)	215,705✔
252	{
253	int32_t k = lj_num2int(n);	215,705✔
254	if (n == (lua_Number)k) {	215,705✔
255	if (kp) *kp = k;	213,691✔
256	if (k == 0) { /* Special check for -0. */	213,691✔
257	TValue tv;	29,563✔
258	setnumV(&tv, n);	29,563✔
259	if (tv.u32.hi != 0)	29,563✔
260	return 0;
261	}
262	return 1;	213,691✔
263	}
264	return 0;
265	}
266
267	/* Intern number as int32_t constant if possible, otherwise as FP constant. */
268	TRef lj_ir_knumint(jit_State *J, lua_Number n)	215,705✔
269	{
270	int32_t k;	215,705✔
271	if (numistrueint(n, &k))	215,705✔
272	return lj_ir_kint(J, k);	213,691✔
273	else
274	return lj_ir_knum(J, n);	2,014✔
275	}
276
277	/* Intern GC object "constant". */
278	TRef lj_ir_kgc(jit_State J, GCobj o, IRType t)	3,417,013✔
279	{
280	IRIns ir, cir = J->cur.ir;	3,417,013✔
281	IRRef ref;	3,417,013✔
282	lj_assertJ(!isdead(J2G(J), o), "interning of dead GC object");	3,417,013✔
283	for (ref = J->chain[IR_KGC]; ref; ref = cir[ref].prev)	30,004,311✔
284	if (ir_kgc(&cir[ref]) == o)	29,538,946✔
285	goto found;	2,951,648✔
286	ref = ir_nextkgc(J);	465,365✔
287	ir = IR(ref);	465,365✔
288	/* NOBARRIER: Current trace is a GC root. */
289	ir->op12 = 0;	465,365✔
290	setgcref(ir[LJ_GC64].gcr, o);	465,365✔
291	ir->t.irt = (uint8_t)t;	465,365✔
292	ir->o = IR_KGC;	465,365✔
293	ir->prev = J->chain[IR_KGC];	465,365✔
294	J->chain[IR_KGC] = (IRRef1)ref;	465,365✔
295	found:	3,417,013✔
296	return TREF(ref, t);	3,417,013✔
297	}
298
299	/* Allocate GCtrace constant placeholder (no interning). */
300	TRef lj_ir_ktrace(jit_State *J)	162✔
301	{
302	IRRef ref = ir_nextkgc(J);	162✔
303	IRIns *ir = IR(ref);	162✔
304	lj_assertJ(irt_toitype_(IRT_P64) == LJ_TTRACE, "mismatched type mapping");	162✔
305	ir->t.irt = IRT_P64;	162✔
306	ir->o = LJ_GC64 ? IR_KNUM : IR_KNULL; /* Not IR_KGC yet, but same size. */	162✔
307	ir->op12 = 0;	162✔
308	ir->prev = 0;	162✔
309	return TREF(ref, IRT_P64);	162✔
310	}
311
312	/* Intern pointer constant. */
313	TRef lj_ir_kptr_(jit_State J, IROp op, void ptr)	163,746✔
314	{
315	IRIns ir, cir = J->cur.ir;	163,746✔
316	IRRef ref;	163,746✔
317	#if LJ_64 && !LJ_GC64
318	lj_assertJ((void *)(uintptr_t)u32ptr(ptr) == ptr, "out-of-range GC pointer");
319	#endif
320	for (ref = J->chain[op]; ref; ref = cir[ref].prev)	302,906✔
321	if (ir_kptr(&cir[ref]) == ptr)	238,456✔
322	goto found;	99,296✔
323	#if LJ_GC64
324	ref = ir_nextk64(J);	64,450✔
325	#else
326	ref = ir_nextk(J);
327	#endif
328	ir = IR(ref);	64,450✔
329	ir->op12 = 0;	64,450✔
330	setmref(ir[LJ_GC64].ptr, ptr);	64,450✔
331	ir->t.irt = IRT_PGC;	64,450✔
332	ir->o = op;	64,450✔
333	ir->prev = J->chain[op];	64,450✔
334	J->chain[op] = (IRRef1)ref;	64,450✔
335	found:	163,746✔
336	return TREF(ref, IRT_PGC);	163,746✔
337	}
338
339	/* Intern typed NULL constant. */
340	TRef lj_ir_knull(jit_State *J, IRType t)	276,188✔
341	{
342	IRIns ir, cir = J->cur.ir;	276,188✔
343	IRRef ref;	276,188✔
344	for (ref = J->chain[IR_KNULL]; ref; ref = cir[ref].prev)	276,267✔
345	if (irt_t(cir[ref].t) == t)	262,273✔
346	goto found;	262,194✔
347	ref = ir_nextk(J);	13,994✔
348	ir = IR(ref);	13,994✔
349	ir->i = 0;	13,994✔
350	ir->t.irt = (uint8_t)t;	13,994✔
351	ir->o = IR_KNULL;	13,994✔
352	ir->prev = J->chain[IR_KNULL];	13,994✔
353	J->chain[IR_KNULL] = (IRRef1)ref;	13,994✔
354	found:	276,188✔
355	return TREF(ref, t);	276,188✔
356	}
357
358	/* Intern key slot. */
359	TRef lj_ir_kslot(jit_State *J, TRef key, IRRef slot)	1,224,463✔
360	{
361	IRIns ir, cir = J->cur.ir;	1,224,463✔
362	IRRef2 op12 = IRREF2((IRRef1)key, (IRRef1)slot);	1,224,463✔
363	IRRef ref;	1,224,463✔
364	/* Const part is not touched by CSE/DCE, so 0-65535 is ok for IRMlit here. */
365	lj_assertJ(tref_isk(key) && slot == (IRRef)(IRRef1)slot,	1,224,463✔
366	"out-of-range key/slot");
367	for (ref = J->chain[IR_KSLOT]; ref; ref = cir[ref].prev)	5,452,931✔
368	if (cir[ref].op12 == op12)	5,266,225✔
369	goto found;	1,037,757✔
370	ref = ir_nextk(J);	186,706✔
371	ir = IR(ref);	186,706✔
372	ir->op12 = op12;	186,706✔
373	ir->t.irt = IRT_P32;	186,706✔
374	ir->o = IR_KSLOT;	186,706✔
375	ir->prev = J->chain[IR_KSLOT];	186,706✔
376	J->chain[IR_KSLOT] = (IRRef1)ref;	186,706✔
377	found:	1,224,463✔
378	return TREF(ref, IRT_P32);	1,224,463✔
379	}
380
381	/* -- Access to IR constants ---------------------------------------------- */
382
383	/* Copy value of IR constant. */
384	void lj_ir_kvalue(lua_State L, TValue tv, const IRIns *ir)	487,985✔
385	{
386	UNUSED(L);	487,985✔
387	lj_assertL(ir->o != IR_KSLOT, "unexpected KSLOT"); /* Common mistake. */	487,985✔
388	switch (ir->o) {	487,985✔
389	case IR_KPRI: setpriV(tv, irt_toitype(ir->t)); break;	13,605✔
390	case IR_KINT: setintV(tv, ir->i); break;	8,293✔
391	case IR_KGC: setgcV(L, tv, ir_kgc(ir), irt_toitype(ir->t)); break;	335,634✔
392	case IR_KPTR: case IR_KKPTR:	×
393	setnumV(tv, (lua_Number)(uintptr_t)ir_kptr(ir));	×
394	break;	×
395	case IR_KNULL: setintV(tv, 0); break;	1✔
396	case IR_KNUM: setnumV(tv, ir_knum(ir)->n); break;	130,452✔
397	#if LJ_HASFFI
398	case IR_KINT64: {
399	GCcdata *cd = lj_cdata_new_(L, CTID_INT64, 8);	×
400	(uint64_t )cdataptr(cd) = ir_kint64(ir)->u64;	×
401	setcdataV(L, tv, cd);	×
402	break;
403	}
404	#endif
405	default: lj_assertL(0, "bad IR constant op %d", ir->o); break;
406	}
407	}	487,985✔
408
409	/* -- Convert IR operand types -------------------------------------------- */
410
411	/* Convert from string to number. */
412	TRef LJ_FASTCALL lj_ir_tonumber(jit_State *J, TRef tr)	186,476✔
413	{
414	if (!tref_isnumber(tr)) {	186,476✔
415	if (tref_isstr(tr))	×
416	tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);	×
417	else
418	lj_trace_err(J, LJ_TRERR_BADTYPE);	×
419	}
420	return tr;	186,476✔
421	}
422
423	/* Convert from integer or string to number. */
424	TRef LJ_FASTCALL lj_ir_tonum(jit_State *J, TRef tr)	807✔
425	{
426	if (!tref_isnum(tr)) {	807✔
427	if (tref_isinteger(tr))	175✔
428	tr = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT);	175✔
429	else if (tref_isstr(tr))	×
430	tr = emitir(IRTG(IR_STRTO, IRT_NUM), tr, 0);	×
431	else
432	lj_trace_err(J, LJ_TRERR_BADTYPE);	×
433	}
434	return tr;	807✔
435	}
436
437	/* Convert from integer or number to string. */
438	TRef LJ_FASTCALL lj_ir_tostr(jit_State *J, TRef tr)	623✔
439	{
440	if (!tref_isstr(tr)) {	623✔
441	if (!tref_isnumber(tr))	4✔
442	lj_trace_err(J, LJ_TRERR_BADTYPE);	×
443	tr = emitir(IRT(IR_TOSTR, IRT_STR), tr,	4✔
444	tref_isnum(tr) ? IRTOSTR_NUM : IRTOSTR_INT);
445	}
446	return tr;	623✔
447	}
448
449	/* -- Miscellaneous IR ops ------------------------------------------------ */
450
451	/* Evaluate numeric comparison. */
452	int lj_ir_numcmp(lua_Number a, lua_Number b, IROp op)	180,746✔
453	{
454	switch (op) {	180,746✔
455	case IR_EQ: return (a == b);	4✔
456	case IR_NE: return (a != b);	×
457	case IR_LT: return (a < b);	34✔
458	case IR_GE: return (a >= b);	39✔
459	case IR_LE: return (a <= b);	256✔
460	case IR_GT: return (a > b);	12✔
461	case IR_ULT: return !(a >= b);	×
462	case IR_UGE: return !(a < b);	12,647✔
463	case IR_ULE: return !(a > b);	×
464	case IR_UGT: return !(a <= b);	167,754✔
465	default: lj_assertX(0, "bad IR op %d", op); return 0;
466	}
467	}
468
469	/* Evaluate string comparison. */
470	int lj_ir_strcmp(GCstr a, GCstr b, IROp op)	1✔
471	{
472	int res = lj_str_cmp(a, b);	1✔
473	switch (op) {	1✔
UNCOV 474	case IR_LT: return (res < 0);	×
UNCOV 475	case IR_GE: return (res >= 0);	×
476	case IR_LE: return (res <= 0);	1✔
UNCOV 477	case IR_GT: return (res > 0);	×
478	default: lj_assertX(0, "bad IR op %d", op); return 0;
479	}
480	}
481
482	/* Rollback IR to previous state. */
483	void lj_ir_rollback(jit_State *J, IRRef ref)	491,490✔
484	{
485	IRRef nins = J->cur.nins;	491,490✔
486	while (nins > ref) {	496,670✔
487	IRIns *ir;	5,180✔
488	nins--;	5,180✔
489	ir = IR(nins);	5,180✔
490	J->chain[ir->o] = ir->prev;	5,180✔
491	}
492	J->cur.nins = nins;	491,490✔
493	}	491,490✔
494
495	#undef IR
496	#undef fins
497	#undef emitir
498
499	#endif

tarantool / luajit / 8752988570

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous