19798590866

Committed 30 Nov 2025 11:44AM UTC coverage: 93.048% (-0.02%) from 93.065%

Build # 19798590866

Build Type

push

github

Committed by

Buristan

Commit Message

ARM64: Fix assembly of HREFK (again).

Thanks to Peter Cawley.

(cherry picked from commit 93ce12ee1)

When assembling the HREFK IR with the huge offset of the target node
from the table, this offset calculation and the key loading from the
node are emitted like the following:
|  ldr   x16, [x2, 40]
|  add   x16, x16, x21
|  ldr   x27, [x16, 8]
|  cmp   x27, x17

Here, `x16` is the node register, `x27` is the key register, and `x21`
is the register containing the offset.

It is possible that the register for holding the constant operand in the
addition may be chosen as the same register containing the node address,
since the full `RSET_GPR` is given to the `emit_opk()`. It will result
in the following invalid mcode:
|  ldr   x27, [x2, 40]
|  str   x27, [sp, 8]
|  add   x16, x16, x16
|  ldr   x16, [sp, 8]
|  ldr   x27, [x16, 8]
|  cmp   x27, x17

It seems that in the current implementation the LuaJIT's register
allocator always prefers the register holding the key instead, so this
does not lead to the invalid emitting. Hence, it is impossible to come
up with any valid reproducer. However, to avoid possible regressions in
the future, this patch fixes the invalid register set by excluding the
node register from it.

Sergey Kaplun:
* added the description for the problem

Part of tarantool/tarantool#11691

Reviewed-by: Sergey Bronnikov <sergeyb@tarantool.org>
Signed-off-by: Sergey Kaplun <skaplun@tarantool.org>
(cherry picked from commit fd5e7cb52)

Run Details

5712 of 6046 branches covered (94.48%)

Branch coverage included in aggregate %.

21792 of 23513 relevant lines covered (92.68%)

3873117.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.59

/src/lj_sysprof.c

/*
** Implementation of sysprof - platform and Lua profiler.
*/

#define lj_sysprof_c
#define LUA_CORE

#include "lj_arch.h"
#include "lj_sysprof.h"

#if LJ_HASSYSPROF

#include "lj_obj.h"
#include "lj_debug.h"
#include "lj_dispatch.h"
#include "lj_frame.h"

#if LJ_HASJIT
#include "lj_jit.h"
#include "lj_trace.h"
#endif

#include "lj_wbuf.h"
#include "lj_profile_timer.h"
#include "lj_symtab.h"

#include <pthread.h>
#include <errno.h>
#include <execinfo.h>

/*
** Number of profiler frames we need to omit during stack
** unwinding.
**   +------------------------+
** 0 | default_backtrace_host |
**   +------------------------+
** 1 | stream_backtrace_host  |
**   +------------------------+
** 2 |  stream_{guest/host}   |
**   +------------------------+
** 3 |      stream_event      |
**   +------------------------+
** 4 | sysprof_record_sample  |
**   +------------------------+
** 5 | sysprof_signal_handler |
**   +------------------------+
*/
#define SYSPROF_HANDLER_STACK_DEPTH 6
#define SYSPROF_BACKTRACE_FRAME_MAX 512

/* Check that vmstate fits in 4 bits (see streaming format) */
#define vmstfit4(st) ((st & ~(uint32_t)((1 << 4) - 1)) == 0)

enum sysprof_state {
  /* Profiler is not running. */
  SPS_IDLE,
  /* Profiler is running. */
  SPS_PROFILE,
  /*
  ** Stopped in case of stopped or failed stream.
  ** Saved errno is set at luaM_sysprof_stop.
  */
  SPS_HALT
};

struct sysprof {
  global_State *g; /* Profiled VM. */
  pthread_t thread; /* Profiled thread. */
  volatile sig_atomic_t state; /* Internal state. */
  struct lj_wbuf out; /* Output accumulator. */
  struct luam_Sysprof_Counters counters; /* Profiling counters. */
  struct luam_Sysprof_Options opt; /* Profiling options. */
  luam_Sysprof_writer writer; /* Writer function for profile events. */
  luam_Sysprof_on_stop on_stop; /* Callback on profiling stopping. */
  luam_Sysprof_backtracer backtracer; /* Backtracing function for the host stack. */
  lj_profile_timer timer; /* Profiling timer. */
  int saved_errno; /* Saved errno when profiler failed. */
};
/*
** XXX: Only one VM can be profiled at a time.
*/

static struct sysprof sysprof = {0};

/* --- Stream ------------------------------------------------------------- */

static const uint8_t ljp_header[] = {'l', 'j', 'p', LJP_FORMAT_VERSION,
                                      0x0, 0x0, 0x0};

static int stream_is_needed(struct sysprof *sp)
{
  return sp->opt.mode != LUAM_SYSPROF_DEFAULT;
}

static int is_unconfigured(struct sysprof *sp)
{
  return sp->backtracer == NULL || sp->on_stop == NULL || sp->writer == NULL;
}

static void stream_prologue(struct sysprof *sp)
{
  /*
  ** XXX: Must be zero for the symtab module to dump all loaded libraries.
  */
  uint32_t unused_lib_adds = 0;
  lj_symtab_dump(&sp->out, sp->g, &unused_lib_adds);
  lj_wbuf_addn(&sp->out, ljp_header, sizeof(ljp_header));
}

static void stream_epilogue(struct sysprof *sp)
{
  lj_wbuf_addbyte(&sp->out, LJP_EPILOGUE_BYTE);
}

static void stream_ffunc_impl(struct lj_wbuf *buf, uint8_t ffid)
{
  lj_wbuf_addbyte(buf, LJP_FRAME_FFUNC);
  lj_wbuf_addu64(buf, ffid);
}

static void stream_lfunc(struct lj_wbuf *buf, const GCfunc *func)
{
  lj_assertX(isluafunc(func), "bad lua function in sysprof stream");
  const GCproto *pt = funcproto(func);
  lj_assertX(pt != NULL, "bad lua function prototype in sysprof stream");
  lj_wbuf_addbyte(buf, LJP_FRAME_LFUNC);
  lj_wbuf_addu64(buf, (uintptr_t)pt);
  lj_wbuf_addu64(buf, (uint64_t)pt->firstline);
}

static void stream_cfunc(struct lj_wbuf *buf, const GCfunc *func)
{
  lj_assertX(iscfunc(func), "bad C function in sysprof stream");
  lj_wbuf_addbyte(buf, LJP_FRAME_CFUNC);
  lj_wbuf_addu64(buf, (uintptr_t)func->c.f);
}

static void stream_ffunc(struct lj_wbuf *buf, const GCfunc *func)
{
  lj_assertX(isffunc(func), "bad fast function in sysprof stream");
  stream_ffunc_impl(buf, func->c.ffid);
}

static void stream_frame_lua(struct lj_wbuf *buf, const cTValue *frame)
{
  const GCfunc *func = frame_func(frame);
  lj_assertX(func != NULL, "bad function in sysprof stream");
  if (isluafunc(func))
    stream_lfunc(buf, func);
  else if (isffunc(func))
    stream_ffunc(buf, func);
  else if (iscfunc(func))
    stream_cfunc(buf, func);
  else
    /* Unreachable. */
    lj_assertX(0, "bad function type in sysprof stream");
}

static void stream_backtrace_lua(struct sysprof *sp, uint32_t vmstate)
{
  global_State *g = sp->g;
  struct lj_wbuf *buf = &sp->out;
  cTValue *top_frame = NULL, *frame = NULL, *bot = NULL;
  lua_State *L = NULL;

  lj_assertX(g != NULL, "uninitialized global state in sysprof state");
  L = gco2th(gcref(g->cur_L));
  lj_assertG(L != NULL, "uninitialized Lua state in sysprof state");
  /*
  ** Lua stack may be inconsistent during the execution of a
  ** fast-function, so instead of updating the `top_frame` for
  ** it, its `ffid` is set instead. The first frame on the
  ** result stack is streamed manually, and the rest of the
  ** stack is streamed based on the previous `top_frame` value.
  */
  if (vmstate == LJ_VMST_FFUNC) {
    uint8_t ffid = g->top_frame_info.ffid;
    stream_ffunc_impl(buf, ffid);
  }

  top_frame = g->top_frame_info.top_frame - 1;

  bot = tvref(L->stack) + LJ_FR2;
  /* Traverse frames backwards */
  for (frame = top_frame; frame > bot; frame = frame_prev(frame)) {
    if (frame_gc(frame) == obj2gco(L) || frame_isvarg(frame))
      continue;  /* Skip dummy frames. See lj_err_optype_call(). */
    stream_frame_lua(buf, frame);
  }

  lj_wbuf_addbyte(buf, LJP_FRAME_LUA_LAST);
}

static void *stream_frame_host(int frame_no, void *addr)
{
  struct sysprof *sp = &sysprof;
  /*
  ** We don't want the profiler stack to be streamed, as it will
  ** burden the profile with unnecessary information.
  */
  if (LJ_UNLIKELY(frame_no <= SYSPROF_HANDLER_STACK_DEPTH))
    return addr;
  else if (LJ_UNLIKELY(sp->opt.mode == LUAM_SYSPROF_LEAF &&
                         frame_no > SYSPROF_HANDLER_STACK_DEPTH))
    return NULL;

  lj_wbuf_addu64(&sp->out, (uintptr_t)addr);
  return addr;
}

static void default_backtrace_host(void *(writer)(int frame_no, void *addr))
{
  static void *backtrace_buf[SYSPROF_BACKTRACE_FRAME_MAX] = {};

  struct sysprof *sp = &sysprof;
  int max_depth = sp->opt.mode == LUAM_SYSPROF_LEAF
                  ? SYSPROF_HANDLER_STACK_DEPTH + 1
                  : SYSPROF_BACKTRACE_FRAME_MAX;
  const int depth = backtrace(backtrace_buf, max_depth);
  int level;

  lj_assertX(depth <= max_depth, "depth of C stack is too big");
  for (level = SYSPROF_HANDLER_STACK_DEPTH; level < depth; ++level) {
    if (!writer(level - SYSPROF_HANDLER_STACK_DEPTH + 1, backtrace_buf[level]))
      return;
  }
}

static void stream_backtrace_host(struct sysprof *sp)
{
  lj_assertX(sp->backtracer != NULL, "uninitialized sysprof backtracer");
  sp->backtracer(stream_frame_host);
  lj_wbuf_addu64(&sp->out, (uintptr_t)LJP_FRAME_HOST_LAST);
}

#if LJ_HASJIT
static void stream_trace(struct sysprof *sp, uint32_t vmstate)
{
  lj_wbuf_addbyte(&sp->out, (uint8_t)vmstate);
  struct lj_wbuf *out = &sp->out;
  uint32_t traceno = sp->g->vmstate;
  jit_State *J = G2J(sp->g);
  GCtrace *trace = traceref(J, traceno);

  GCproto *startpt = gco2pt(gcref(trace->startpt));

  lj_wbuf_addu64(out, traceno);
  lj_wbuf_addu64(out, (uintptr_t)startpt);
  lj_wbuf_addu64(out, startpt->firstline);
}
#endif

static void stream_guest(struct sysprof *sp, uint32_t vmstate)
{
  lj_wbuf_addbyte(&sp->out, (uint8_t)vmstate);
  stream_backtrace_lua(sp, vmstate);
  stream_backtrace_host(sp);
}

static void stream_host(struct sysprof *sp, uint32_t vmstate)
{
  lj_wbuf_addbyte(&sp->out, (uint8_t)vmstate);
  stream_backtrace_host(sp);
}

typedef void (*event_streamer)(struct sysprof *sp, uint32_t vmstate);

static event_streamer event_streamers[] = {
  /* XXX: order is important */
  stream_host,  /* LJ_VMST_INTERP */
  stream_guest, /* LJ_VMST_LFUNC */
  stream_guest, /* LJ_VMST_FFUNC */
  stream_guest, /* LJ_VMST_CFUNC */
  stream_host,  /* LJ_VMST_GC */
  stream_host,  /* LJ_VMST_EXIT */
  stream_host,  /* LJ_VMST_RECORD */
  stream_host,  /* LJ_VMST_OPT */
  stream_host,  /* LJ_VMST_ASM */
#if LJ_HASJIT
  stream_trace  /* LJ_VMST_TRACE */
#endif
};

static void stream_event(struct sysprof *sp, uint32_t vmstate)
{
  event_streamer stream = NULL;

  lj_assertX(vmstfit4(vmstate), "vmstate don't fit in 4 bits");
  stream = event_streamers[vmstate];
  lj_assertX(stream != NULL, "uninitialized sysprof stream");
  stream(sp, vmstate);
}

/* -- Signal handler ------------------------------------------------------ */

static void sysprof_record_sample(struct sysprof *sp, siginfo_t *info)
{
  global_State *g = sp->g;
  uint32_t _vmstate = ~(uint32_t)(g->vmstate);
  uint32_t vmstate;

  /* `g->vmstate` is 0 outside the VM. Hence, dump only the host stack. */
  if (g->vmstate == 0)
    vmstate = LJ_VMST_INTERP;
  else if (_vmstate < LJ_VMST_TRACE)
    vmstate = _vmstate;
  else
    vmstate = LJ_VMST_TRACE;

  lj_assertX(pthread_self() == sp->thread,
             "bad thread during sysprof record sample");

  /* Caveat: order of counters must match vmstate order in <lj_obj.h>. */
  ((uint64_t *)&sp->counters)[vmstate]++;

  sp->counters.samples++;

  if (!stream_is_needed(sp))
    return;

  stream_event(sp, vmstate);
  if (LJ_UNLIKELY(lj_wbuf_test_flag(&sp->out, STREAM_ERRIO|STREAM_STOP))) {
    sp->saved_errno = lj_wbuf_errno(&sp->out);
    lj_wbuf_terminate(&sp->out);
    sp->state = SPS_HALT;
  }
}

static void sysprof_signal_handler(int sig, siginfo_t *info, void *ctx)
{
  struct sysprof *sp = &sysprof;
  UNUSED(sig);
  UNUSED(ctx);

  switch (sp->state) {
    case SPS_PROFILE:
      sysprof_record_sample(sp, info);
      break;

    case SPS_IDLE:
    case SPS_HALT:
      /* noop */
      break;

    default:
      lj_assertX(0, "bad sysprof profiler state");
      break;
  }
}

/* -- Internal ------------------------------------------------------------ */

static int sysprof_validate(struct sysprof *sp,
                            const struct luam_Sysprof_Options *opt)
{
  switch (sp->state) {
    case SPS_IDLE:
      if (opt->mode > LUAM_SYSPROF_CALLGRAPH) {
        return PROFILE_ERRUSE;
      } else if (opt->mode != LUAM_SYSPROF_DEFAULT &&
                 (opt->buf == NULL || opt->len == 0 || is_unconfigured(sp))) {
        return PROFILE_ERRUSE;
      } else if (opt->interval == 0) {
        return PROFILE_ERRUSE;
      }
      break;

    case SPS_PROFILE:
    case SPS_HALT:
      return PROFILE_ERRRUN;

    default:
      lj_assertX(0, "bad sysprof profiler state");
      break;
  }

  return PROFILE_SUCCESS;
}

static int sysprof_init(struct sysprof *sp, lua_State *L,
                        const struct luam_Sysprof_Options *opt)
{
  const int status = sysprof_validate(sp, opt);
  if (PROFILE_SUCCESS != status)
    return status;

  /* Copy validated options to sysprof state. */
  memcpy(&sp->opt, opt, sizeof(sp->opt));

  /* Init general fields. */
  sp->g = G(L);
  sp->thread = pthread_self();

  /* Reset counters. */
  memset(&sp->counters, 0, sizeof(sp->counters));

  /* Reset saved errno. */
  sp->saved_errno = 0;

  if (stream_is_needed(sp))
    lj_wbuf_init(&sp->out, sp->writer, opt->ctx, opt->buf, opt->len);

  return PROFILE_SUCCESS;
}

/* -- Public profiling API ------------------------------------------------ */

int lj_sysprof_set_writer(luam_Sysprof_writer writer) {
  struct sysprof *sp = &sysprof;

  if (sp->state != SPS_IDLE || writer == NULL)
    return PROFILE_ERRUSE;

  sp->writer = writer;
  if (!is_unconfigured(sp)) {
    sp->state = SPS_IDLE;
  }
  return PROFILE_SUCCESS;
}

int lj_sysprof_set_on_stop(luam_Sysprof_on_stop on_stop) {
  struct sysprof *sp = &sysprof;

  if (sp->state != SPS_IDLE || on_stop == NULL)
    return PROFILE_ERRUSE;

  sp->on_stop = on_stop;
  if (!is_unconfigured(sp)) {
    sp->state = SPS_IDLE;
  }
  return PROFILE_SUCCESS;
}

int lj_sysprof_set_backtracer(luam_Sysprof_backtracer backtracer) {
  struct sysprof *sp = &sysprof;

  if (sp->state != SPS_IDLE)
    return PROFILE_ERRUSE;
  if (backtracer == NULL) {
    sp->backtracer = default_backtrace_host;
    /*
    ** XXX: `backtrace` is not signal-safe, according to man,
    ** because it is lazy loaded on the first call, which triggers
    ** allocations. We need to call `backtrace` before starting profiling
    ** to avoid lazy loading.
    */
    void *dummy = NULL;
    backtrace(&dummy, 1);
  }
  else {
    sp->backtracer = backtracer;
  }
  if (!is_unconfigured(sp)) {
    sp->state = SPS_IDLE;
  }
  return PROFILE_SUCCESS;
}

int lj_sysprof_start(lua_State *L, const struct luam_Sysprof_Options *opt)
{
  struct sysprof *sp = &sysprof;

  int status = sysprof_init(sp, L, opt);
  if (PROFILE_SUCCESS != status) {
    if (NULL != sp->on_stop) {
      /*
      ** Initialization may fail in case of unconfigured sysprof,
      ** so we cannot guarantee cleaning up resources in this case.
      */
      sp->on_stop(opt->ctx, opt->buf);
    }
    return status;
  }

  sp->state = SPS_PROFILE;

  if (stream_is_needed(sp)) {
    stream_prologue(sp);
    if (LJ_UNLIKELY(lj_wbuf_test_flag(&sp->out, STREAM_ERRIO|STREAM_STOP))) {
      /* on_stop call may change errno value. */
      const int saved_errno = lj_wbuf_errno(&sp->out);
      /* Ignore possible errors. mp->out.buf may be NULL here. */
      sp->on_stop(opt->ctx, sp->out.buf);
      lj_wbuf_terminate(&sp->out);
      sp->state = SPS_IDLE;
      errno = saved_errno;
      return PROFILE_ERRIO;
    }
  }

  sp->timer.opt.interval_msec = opt->interval;
  sp->timer.opt.handler = sysprof_signal_handler;
  lj_profile_timer_start(&sp->timer);

  return PROFILE_SUCCESS;
}

int lj_sysprof_stop(lua_State *L)
{
  struct sysprof *sp = &sysprof;
  global_State *g = sp->g;
  struct lj_wbuf *out = &sp->out;

  if (SPS_IDLE == sp->state)
    return PROFILE_ERRRUN;
  else if (G(L) != g)
    return PROFILE_ERRUSE;

  lj_profile_timer_stop(&sp->timer);

  if (SPS_HALT == sp->state) {
    errno = sp->saved_errno;
    sp->state = SPS_IDLE;
    /* wbuf was terminated when error occurred. */
    return PROFILE_ERRIO;
  }

  sp->state = SPS_IDLE;

  if (stream_is_needed(sp)) {
    int cb_status = 0;

    stream_epilogue(sp);
    lj_wbuf_flush(out);

    cb_status = sp->on_stop(sp->opt.ctx, out->buf);
    if (LJ_UNLIKELY(lj_wbuf_test_flag(out, STREAM_ERRIO | STREAM_STOP)) ||
        cb_status != 0) {
      errno = lj_wbuf_errno(out);
      lj_wbuf_terminate(out);
      return PROFILE_ERRIO;
    }

    lj_wbuf_terminate(out);
  }

  return PROFILE_SUCCESS;
}

int lj_sysprof_report(struct luam_Sysprof_Counters *counters)
{
  const struct sysprof *sp = &sysprof;
  memcpy(counters, &sp->counters, sizeof(sp->counters));
  return PROFILE_SUCCESS;
}

void lj_sysprof_add_proto(const struct GCproto *pt)
{
  struct sysprof *sp = &sysprof;

  if (sp->state != SPS_PROFILE || sp->opt.mode == LUAM_SYSPROF_DEFAULT)
    return;

  /*
  ** XXX: Avoid sampling during the symtab extension. That shouldn't have any
  ** significant effect on profile precision, but if it does, it's better to
  ** implement an async-safe queue for the symtab events.
  */
  sp->state = SPS_IDLE;
  lj_wbuf_addbyte(&sp->out, LJP_SYMTAB_LFUNC_EVENT);
  lj_symtab_dump_proto(&sp->out, pt);
  sp->state = SPS_PROFILE;
}

#if LJ_HASJIT
void lj_sysprof_add_trace(const struct GCtrace *tr)
{
  struct sysprof *sp = &sysprof;

  if (sp->state != SPS_PROFILE || sp->opt.mode == LUAM_SYSPROF_DEFAULT)
    return;

  /* See the comment about the sysprof state above. */
  sp->state = SPS_IDLE;
  lj_wbuf_addbyte(&sp->out, LJP_SYMTAB_TRACE_EVENT);
  lj_symtab_dump_trace(&sp->out, tr);
  sp->state = SPS_PROFILE;
}
#endif /* LJ_HASJIT */

#else /* LJ_HASSYSPROF */

int lj_sysprof_set_writer(luam_Sysprof_writer writer) {
  UNUSED(writer);
  return PROFILE_ERRUSE;
}

int lj_sysprof_set_on_stop(luam_Sysprof_on_stop on_stop) {
  UNUSED(on_stop);
  return PROFILE_ERRUSE;
}

int lj_sysprof_set_backtracer(luam_Sysprof_backtracer backtracer) {
  UNUSED(backtracer);
  return PROFILE_ERRUSE;
}

int lj_sysprof_start(lua_State *L, const struct luam_Sysprof_Options *opt)
{
  UNUSED(L);
  return PROFILE_ERRUSE;
}

int lj_sysprof_stop(lua_State *L)
{
  UNUSED(L);
  return PROFILE_ERRUSE;
}

int lj_sysprof_report(struct luam_Sysprof_Counters *counters)
{
  UNUSED(counters);
  return PROFILE_ERRUSE;
}

void lj_sysprof_add_proto(const struct GCproto *pt)
{
  UNUSED(pt);
}

#if LJ_HASJIT
void lj_sysprof_add_trace(const struct GCtrace *tr)
{
  UNUSED(tr);
}
#endif /* LJ_HASJIT */

#endif /* LJ_HASSYSPROF */

1	/*
2	** Implementation of sysprof - platform and Lua profiler.
3	*/
4
5	#define lj_sysprof_c
6	#define LUA_CORE
7
8	#include "lj_arch.h"
9	#include "lj_sysprof.h"
10
11	#if LJ_HASSYSPROF
12
13	#include "lj_obj.h"
14	#include "lj_debug.h"
15	#include "lj_dispatch.h"
16	#include "lj_frame.h"
17
18	#if LJ_HASJIT
19	#include "lj_jit.h"
20	#include "lj_trace.h"
21	#endif
22
23	#include "lj_wbuf.h"
24	#include "lj_profile_timer.h"
25	#include "lj_symtab.h"
26
27	#include <pthread.h>
28	#include <errno.h>
29	#include <execinfo.h>
30
31	/*
32	** Number of profiler frames we need to omit during stack
33	** unwinding.
34	** +------------------------+
35	** 0 \| default_backtrace_host \|
36	** +------------------------+
37	** 1 \| stream_backtrace_host \|
38	** +------------------------+
39	** 2 \| stream_{guest/host} \|
40	** +------------------------+
41	** 3 \| stream_event \|
42	** +------------------------+
43	** 4 \| sysprof_record_sample \|
44	** +------------------------+
45	** 5 \| sysprof_signal_handler \|
46	** +------------------------+
47	*/
48	#define SYSPROF_HANDLER_STACK_DEPTH 6
49	#define SYSPROF_BACKTRACE_FRAME_MAX 512
50
51	/* Check that vmstate fits in 4 bits (see streaming format) */
52	#define vmstfit4(st) ((st & ~(uint32_t)((1 << 4) - 1)) == 0)
53
54	enum sysprof_state {
55	/* Profiler is not running. */
56	SPS_IDLE,
57	/* Profiler is running. */
58	SPS_PROFILE,
59	/*
60	** Stopped in case of stopped or failed stream.
61	** Saved errno is set at luaM_sysprof_stop.
62	*/
63	SPS_HALT
64	};
65
66	struct sysprof {
67	global_State g; / Profiled VM. */
68	pthread_t thread; /* Profiled thread. */
69	volatile sig_atomic_t state; /* Internal state. */
70	struct lj_wbuf out; /* Output accumulator. */
71	struct luam_Sysprof_Counters counters; /* Profiling counters. */
72	struct luam_Sysprof_Options opt; /* Profiling options. */
73	luam_Sysprof_writer writer; /* Writer function for profile events. */
74	luam_Sysprof_on_stop on_stop; /* Callback on profiling stopping. */
75	luam_Sysprof_backtracer backtracer; /* Backtracing function for the host stack. */
76	lj_profile_timer timer; /* Profiling timer. */
77	int saved_errno; /* Saved errno when profiler failed. */
78	};
79	/*
80	** XXX: Only one VM can be profiled at a time.
81	*/
82
83	static struct sysprof sysprof = {0};
84
85	/* --- Stream ------------------------------------------------------------- */
86
87	static const uint8_t ljp_header[] = {'l', 'j', 'p', LJP_FORMAT_VERSION,
88	0x0, 0x0, 0x0};
89
90	static int stream_is_needed(struct sysprof *sp)	28✔
91	{
92	return sp->opt.mode != LUAM_SYSPROF_DEFAULT;	28✔
93	}
94
95	static int is_unconfigured(struct sysprof *sp)	1,184✔
96	{
97	return sp->backtracer == NULL \|\| sp->on_stop == NULL \|\| sp->writer == NULL;	400✔
98	}
99
100	static void stream_prologue(struct sysprof *sp)	6✔
101	{
102	/*
103	** XXX: Must be zero for the symtab module to dump all loaded libraries.
104	*/
105	uint32_t unused_lib_adds = 0;	6✔
106	lj_symtab_dump(&sp->out, sp->g, &unused_lib_adds);	6✔
107	lj_wbuf_addn(&sp->out, ljp_header, sizeof(ljp_header));	6✔
108	}	6✔
109
110	static void stream_epilogue(struct sysprof *sp)	6✔
111	{
112	lj_wbuf_addbyte(&sp->out, LJP_EPILOGUE_BYTE);	6✔
113	}
114
115	static void stream_ffunc_impl(struct lj_wbuf *buf, uint8_t ffid)	1✔
116	{
117	lj_wbuf_addbyte(buf, LJP_FRAME_FFUNC);	1✔
118	lj_wbuf_addu64(buf, ffid);	1✔
119	}	1✔
120
121	static void stream_lfunc(struct lj_wbuf buf, const GCfunc func)
122	{
123	lj_assertX(isluafunc(func), "bad lua function in sysprof stream");
124	const GCproto *pt = funcproto(func);
125	lj_assertX(pt != NULL, "bad lua function prototype in sysprof stream");
126	lj_wbuf_addbyte(buf, LJP_FRAME_LFUNC);
127	lj_wbuf_addu64(buf, (uintptr_t)pt);
128	lj_wbuf_addu64(buf, (uint64_t)pt->firstline);
129	}
130
131	static void stream_cfunc(struct lj_wbuf buf, const GCfunc func)	9✔
132	{
133	lj_assertX(iscfunc(func), "bad C function in sysprof stream");	9✔
134	lj_wbuf_addbyte(buf, LJP_FRAME_CFUNC);	9✔
135	lj_wbuf_addu64(buf, (uintptr_t)func->c.f);	9✔
136	}	9✔
137
138	static void stream_ffunc(struct lj_wbuf buf, const GCfunc func)	×
139	{
140	lj_assertX(isffunc(func), "bad fast function in sysprof stream");	×
141	stream_ffunc_impl(buf, func->c.ffid);	×
142	}	×
143
144	static void stream_frame_lua(struct lj_wbuf buf, const cTValue frame)	230✔
145	{
146	const GCfunc *func = frame_func(frame);	230✔
147	lj_assertX(func != NULL, "bad function in sysprof stream");	230✔
148	if (isluafunc(func))	230✔
149	stream_lfunc(buf, func);	221✔
150	else if (isffunc(func))	9✔
151	stream_ffunc(buf, func);	×
152	else if (iscfunc(func))	9✔
153	stream_cfunc(buf, func);	9✔
154	else
155	/* Unreachable. */
156	lj_assertX(0, "bad function type in sysprof stream");	230✔
157	}	230✔
158
159	static void stream_backtrace_lua(struct sysprof *sp, uint32_t vmstate)	10✔
160	{
161	global_State *g = sp->g;	10✔
162	struct lj_wbuf *buf = &sp->out;	10✔
163	cTValue top_frame = NULL, frame = NULL, *bot = NULL;	10✔
164	lua_State *L = NULL;	10✔
165
166	lj_assertX(g != NULL, "uninitialized global state in sysprof state");	10✔
167	L = gco2th(gcref(g->cur_L));	10✔
168	lj_assertG(L != NULL, "uninitialized Lua state in sysprof state");	10✔
169	/*
170	** Lua stack may be inconsistent during the execution of a
171	** fast-function, so instead of updating the `top_frame` for
172	** it, its `ffid` is set instead. The first frame on the
173	** result stack is streamed manually, and the rest of the
174	** stack is streamed based on the previous `top_frame` value.
175	*/
176	if (vmstate == LJ_VMST_FFUNC) {	10✔
177	uint8_t ffid = g->top_frame_info.ffid;	1✔
178	stream_ffunc_impl(buf, ffid);	1✔
179	}
180
181	top_frame = g->top_frame_info.top_frame - 1;	10✔
182
183	bot = tvref(L->stack) + LJ_FR2;	10✔
184	/* Traverse frames backwards */
185	for (frame = top_frame; frame > bot; frame = frame_prev(frame)) {	488✔
186	if (frame_gc(frame) == obj2gco(L) \|\| frame_isvarg(frame))	239✔
187	continue; /* Skip dummy frames. See lj_err_optype_call(). */	9✔
188	stream_frame_lua(buf, frame);	230✔
189	}
190
191	lj_wbuf_addbyte(buf, LJP_FRAME_LUA_LAST);	10✔
192	}	10✔
193
194	static void stream_frame_host(int frame_no, void addr)	110✔
195	{
196	struct sysprof *sp = &sysprof;	110✔
197	/*
198	** We don't want the profiler stack to be streamed, as it will
199	** burden the profile with unnecessary information.
200	*/
201	if (LJ_UNLIKELY(frame_no <= SYSPROF_HANDLER_STACK_DEPTH))	110✔
202	return addr;
203	else if (LJ_UNLIKELY(sp->opt.mode == LUAM_SYSPROF_LEAF &&	14✔
204	frame_no > SYSPROF_HANDLER_STACK_DEPTH))
205	return NULL;
206
207	lj_wbuf_addu64(&sp->out, (uintptr_t)addr);	14✔
208	return addr;	14✔
209	}
210
211	static void default_backtrace_host(void (writer)(int frame_no, void addr))	17✔
212	{
213	static void *backtrace_buf[SYSPROF_BACKTRACE_FRAME_MAX] = {};	17✔
214
215	struct sysprof *sp = &sysprof;	17✔
216	int max_depth = sp->opt.mode == LUAM_SYSPROF_LEAF	34✔
217	? SYSPROF_HANDLER_STACK_DEPTH + 1
218	: SYSPROF_BACKTRACE_FRAME_MAX;	17✔
219	const int depth = backtrace(backtrace_buf, max_depth);	17✔
220	int level;	17✔
221
222	lj_assertX(depth <= max_depth, "depth of C stack is too big");	17✔
223	for (level = SYSPROF_HANDLER_STACK_DEPTH; level < depth; ++level) {	144✔
224	if (!writer(level - SYSPROF_HANDLER_STACK_DEPTH + 1, backtrace_buf[level]))	110✔
225	return;
226	}
227	}
228
229	static void stream_backtrace_host(struct sysprof *sp)	17✔
230	{
231	lj_assertX(sp->backtracer != NULL, "uninitialized sysprof backtracer");	17✔
232	sp->backtracer(stream_frame_host);	17✔
233	lj_wbuf_addu64(&sp->out, (uintptr_t)LJP_FRAME_HOST_LAST);	17✔
234	}
235
236	#if LJ_HASJIT
237	static void stream_trace(struct sysprof *sp, uint32_t vmstate)	×
238	{
239	lj_wbuf_addbyte(&sp->out, (uint8_t)vmstate);	×
240	struct lj_wbuf *out = &sp->out;	×
241	uint32_t traceno = sp->g->vmstate;	×
242	jit_State *J = G2J(sp->g);	×
243	GCtrace *trace = traceref(J, traceno);	×
244
245	GCproto *startpt = gco2pt(gcref(trace->startpt));	×
246
247	lj_wbuf_addu64(out, traceno);	×
248	lj_wbuf_addu64(out, (uintptr_t)startpt);	×
249	lj_wbuf_addu64(out, startpt->firstline);	×
250	}	×
251	#endif
252
253	static void stream_guest(struct sysprof *sp, uint32_t vmstate)	10✔
254	{
255	lj_wbuf_addbyte(&sp->out, (uint8_t)vmstate);	10✔
256	stream_backtrace_lua(sp, vmstate);	10✔
257	stream_backtrace_host(sp);	10✔
258	}	10✔
259
260	static void stream_host(struct sysprof *sp, uint32_t vmstate)	7✔
261	{
262	lj_wbuf_addbyte(&sp->out, (uint8_t)vmstate);	7✔
263	stream_backtrace_host(sp);	7✔
264	}	7✔
265
266	typedef void (event_streamer)(struct sysprof sp, uint32_t vmstate);
267
268	static event_streamer event_streamers[] = {
269	/* XXX: order is important */
270	stream_host, /* LJ_VMST_INTERP */
271	stream_guest, /* LJ_VMST_LFUNC */
272	stream_guest, /* LJ_VMST_FFUNC */
273	stream_guest, /* LJ_VMST_CFUNC */
274	stream_host, /* LJ_VMST_GC */
275	stream_host, /* LJ_VMST_EXIT */
276	stream_host, /* LJ_VMST_RECORD */
277	stream_host, /* LJ_VMST_OPT */
278	stream_host, /* LJ_VMST_ASM */
279	#if LJ_HASJIT
280	stream_trace /* LJ_VMST_TRACE */
281	#endif
282	};
283
284	static void stream_event(struct sysprof *sp, uint32_t vmstate)
285	{
286	event_streamer stream = NULL;
287
288	lj_assertX(vmstfit4(vmstate), "vmstate don't fit in 4 bits");
289	stream = event_streamers[vmstate];
290	lj_assertX(stream != NULL, "uninitialized sysprof stream");
291	stream(sp, vmstate);
292	}
293
294	/* -- Signal handler ------------------------------------------------------ */
295
296	static void sysprof_record_sample(struct sysprof sp, siginfo_t info)
297	{
298	global_State *g = sp->g;
299	uint32_t _vmstate = ~(uint32_t)(g->vmstate);
300	uint32_t vmstate;
301
302	/* `g->vmstate` is 0 outside the VM. Hence, dump only the host stack. */
303	if (g->vmstate == 0)
304	vmstate = LJ_VMST_INTERP;
305	else if (_vmstate < LJ_VMST_TRACE)
306	vmstate = _vmstate;
307	else
308	vmstate = LJ_VMST_TRACE;
309
310	lj_assertX(pthread_self() == sp->thread,
311	"bad thread during sysprof record sample");
312
313	/* Caveat: order of counters must match vmstate order in <lj_obj.h>. */
314	((uint64_t *)&sp->counters)[vmstate]++;
315
316	sp->counters.samples++;
317
318	if (!stream_is_needed(sp))
319	return;
320
321	stream_event(sp, vmstate);
322	if (LJ_UNLIKELY(lj_wbuf_test_flag(&sp->out, STREAM_ERRIO\|STREAM_STOP))) {
323	sp->saved_errno = lj_wbuf_errno(&sp->out);
324	lj_wbuf_terminate(&sp->out);
325	sp->state = SPS_HALT;
326	}
327	}
328
329	static void sysprof_signal_handler(int sig, siginfo_t info, void ctx)	50✔
330	{
331	struct sysprof *sp = &sysprof;	50✔
332	UNUSED(sig);	50✔
333	UNUSED(ctx);	50✔
334
335	switch (sp->state) {	50✔
336	case SPS_PROFILE:	50✔
337	sysprof_record_sample(sp, info);	50✔
338	break;	50✔
339
340	case SPS_IDLE:
341	case SPS_HALT:
342	/* noop */
343	break;
344
345	default:
346	lj_assertX(0, "bad sysprof profiler state");
347	break;
348	}
349	}	50✔
350
351	/* -- Internal ------------------------------------------------------------ */
352
353	static int sysprof_validate(struct sysprof *sp,	19✔
354	const struct luam_Sysprof_Options *opt)
355	{
356	switch (sp->state) {	19✔
357	case SPS_IDLE:	17✔
358	if (opt->mode > LUAM_SYSPROF_CALLGRAPH) {	17✔
359	return PROFILE_ERRUSE;
360	} else if (opt->mode != LUAM_SYSPROF_DEFAULT &&	16✔
361	(opt->buf == NULL \|\| opt->len == 0 \|\| is_unconfigured(sp))) {	7✔
362	return PROFILE_ERRUSE;
363	} else if (opt->interval == 0) {	15✔
364	return PROFILE_ERRUSE;	1✔
365	}
366	break;
367
368	case SPS_PROFILE:
369	case SPS_HALT:
370	return PROFILE_ERRRUN;
371
372	default:
373	lj_assertX(0, "bad sysprof profiler state");
374	break;
375	}
376
377	return PROFILE_SUCCESS;
378	}
379
380	static int sysprof_init(struct sysprof sp, lua_State L,
381	const struct luam_Sysprof_Options *opt)
382	{
383	const int status = sysprof_validate(sp, opt);
384	if (PROFILE_SUCCESS != status)
385	return status;
386
387	/* Copy validated options to sysprof state. */
388	memcpy(&sp->opt, opt, sizeof(sp->opt));
389
390	/* Init general fields. */
391	sp->g = G(L);
392	sp->thread = pthread_self();
393
394	/* Reset counters. */
395	memset(&sp->counters, 0, sizeof(sp->counters));
396
397	/* Reset saved errno. */
398	sp->saved_errno = 0;
399
400	if (stream_is_needed(sp))
401	lj_wbuf_init(&sp->out, sp->writer, opt->ctx, opt->buf, opt->len);
402
403	return PROFILE_SUCCESS;
404	}
405
406	/* -- Public profiling API ------------------------------------------------ */
407
408	int lj_sysprof_set_writer(luam_Sysprof_writer writer) {	393✔
409	struct sysprof *sp = &sysprof;	393✔
410
411	if (sp->state != SPS_IDLE \|\| writer == NULL)	393✔
412	return PROFILE_ERRUSE;
413
414	sp->writer = writer;	393✔
415	if (!is_unconfigured(sp)) {	393✔
416	sp->state = SPS_IDLE;	1✔
417	}
418	return PROFILE_SUCCESS;
419	}
420
421	int lj_sysprof_set_on_stop(luam_Sysprof_on_stop on_stop) {	393✔
422	struct sysprof *sp = &sysprof;	393✔
423
424	if (sp->state != SPS_IDLE \|\| on_stop == NULL)	393✔
425	return PROFILE_ERRUSE;
426
427	sp->on_stop = on_stop;	393✔
428	if (!is_unconfigured(sp)) {	393✔
429	sp->state = SPS_IDLE;	1✔
430	}
431	return PROFILE_SUCCESS;
432	}
433
434	int lj_sysprof_set_backtracer(luam_Sysprof_backtracer backtracer) {	392✔
435	struct sysprof *sp = &sysprof;	392✔
436
437	if (sp->state != SPS_IDLE)	392✔
438	return PROFILE_ERRUSE;
439	if (backtracer == NULL) {	392✔
440	sp->backtracer = default_backtrace_host;	392✔
441	/*
442	** XXX: `backtrace` is not signal-safe, according to man,
443	** because it is lazy loaded on the first call, which triggers
444	** allocations. We need to call `backtrace` before starting profiling
445	** to avoid lazy loading.
446	*/
447	void *dummy = NULL;	392✔
448	backtrace(&dummy, 1);	392✔
449	}
450	else {
451	sp->backtracer = backtracer;	×
452	}
453	if (!is_unconfigured(sp)) {	392✔
454	sp->state = SPS_IDLE;	392✔
455	}
456	return PROFILE_SUCCESS;
457	}
458
459	int lj_sysprof_start(lua_State L, const struct luam_Sysprof_Options opt)	19✔
460	{
461	struct sysprof *sp = &sysprof;	19✔
462
463	int status = sysprof_init(sp, L, opt);	19✔
464	if (PROFILE_SUCCESS != status) {	19✔
465	if (NULL != sp->on_stop) {	5✔
466	/*
467	** Initialization may fail in case of unconfigured sysprof,
468	** so we cannot guarantee cleaning up resources in this case.
469	*/
470	sp->on_stop(opt->ctx, opt->buf);	5✔
471	}
472	return status;	5✔
473	}
474
475	sp->state = SPS_PROFILE;	14✔
476
477	if (stream_is_needed(sp)) {	14✔
478	stream_prologue(sp);	6✔
479	if (LJ_UNLIKELY(lj_wbuf_test_flag(&sp->out, STREAM_ERRIO\|STREAM_STOP))) {	6✔
480	/* on_stop call may change errno value. */
481	const int saved_errno = lj_wbuf_errno(&sp->out);	×
482	/* Ignore possible errors. mp->out.buf may be NULL here. */
483	sp->on_stop(opt->ctx, sp->out.buf);	×
484	lj_wbuf_terminate(&sp->out);	×
485	sp->state = SPS_IDLE;	×
486	errno = saved_errno;	×
487	return PROFILE_ERRIO;	×
488	}
489	}
490
491	sp->timer.opt.interval_msec = opt->interval;	14✔
492	sp->timer.opt.handler = sysprof_signal_handler;	14✔
493	lj_profile_timer_start(&sp->timer);	14✔
494
495	return PROFILE_SUCCESS;	14✔
496	}
497
498	int lj_sysprof_stop(lua_State *L)	405✔
499	{
500	struct sysprof *sp = &sysprof;	405✔
501	global_State *g = sp->g;	405✔
502	struct lj_wbuf *out = &sp->out;	405✔
503
504	if (SPS_IDLE == sp->state)	405✔
505	return PROFILE_ERRRUN;
506	else if (G(L) != g)	14✔
507	return PROFILE_ERRUSE;
508
509	lj_profile_timer_stop(&sp->timer);	14✔
510
511	if (SPS_HALT == sp->state) {	14✔
512	errno = sp->saved_errno;	×
513	sp->state = SPS_IDLE;	×
514	/* wbuf was terminated when error occurred. */
515	return PROFILE_ERRIO;	×
516	}
517
518	sp->state = SPS_IDLE;	14✔
519
520	if (stream_is_needed(sp)) {	14✔
521	int cb_status = 0;	6✔
522
523	stream_epilogue(sp);	6✔
524	lj_wbuf_flush(out);	6✔
525
526	cb_status = sp->on_stop(sp->opt.ctx, out->buf);	6✔
527	if (LJ_UNLIKELY(lj_wbuf_test_flag(out, STREAM_ERRIO \| STREAM_STOP)) \|\|	6✔
528	cb_status != 0) {
529	errno = lj_wbuf_errno(out);	×
530	lj_wbuf_terminate(out);	×
531	return PROFILE_ERRIO;	×
532	}
533
534	lj_wbuf_terminate(out);	6✔
535	}
536
537	return PROFILE_SUCCESS;
538	}
539
540	int lj_sysprof_report(struct luam_Sysprof_Counters *counters)	5✔
541	{
542	const struct sysprof *sp = &sysprof;	5✔
543	memcpy(counters, &sp->counters, sizeof(sp->counters));	5✔
544	return PROFILE_SUCCESS;	5✔
545	}
546
547	void lj_sysprof_add_proto(const struct GCproto *pt)	79,201✔
548	{
549	struct sysprof *sp = &sysprof;	79,201✔
550
551	if (sp->state != SPS_PROFILE \|\| sp->opt.mode == LUAM_SYSPROF_DEFAULT)	79,201✔
552	return;
553
554	/*
555	** XXX: Avoid sampling during the symtab extension. That shouldn't have any
556	** significant effect on profile precision, but if it does, it's better to
557	** implement an async-safe queue for the symtab events.
558	*/
559	sp->state = SPS_IDLE;	2✔
560	lj_wbuf_addbyte(&sp->out, LJP_SYMTAB_LFUNC_EVENT);	2✔
561	lj_symtab_dump_proto(&sp->out, pt);	2✔
562	sp->state = SPS_PROFILE;	2✔
563	}
564
565	#if LJ_HASJIT
566	void lj_sysprof_add_trace(const struct GCtrace *tr)	18,322✔
567	{
568	struct sysprof *sp = &sysprof;	18,322✔
569
570	if (sp->state != SPS_PROFILE \|\| sp->opt.mode == LUAM_SYSPROF_DEFAULT)	18,322✔
571	return;
572
573	/* See the comment about the sysprof state above. */
574	sp->state = SPS_IDLE;	×
575	lj_wbuf_addbyte(&sp->out, LJP_SYMTAB_TRACE_EVENT);	×
576	lj_symtab_dump_trace(&sp->out, tr);	×
577	sp->state = SPS_PROFILE;	×
578	}
579	#endif /* LJ_HASJIT */
580
581	#else /* LJ_HASSYSPROF */
582
583	int lj_sysprof_set_writer(luam_Sysprof_writer writer) {
584	UNUSED(writer);
585	return PROFILE_ERRUSE;
586	}
587
588	int lj_sysprof_set_on_stop(luam_Sysprof_on_stop on_stop) {
589	UNUSED(on_stop);
590	return PROFILE_ERRUSE;
591	}
592
593	int lj_sysprof_set_backtracer(luam_Sysprof_backtracer backtracer) {
594	UNUSED(backtracer);
595	return PROFILE_ERRUSE;
596	}
597
598	int lj_sysprof_start(lua_State L, const struct luam_Sysprof_Options opt)
599	{
600	UNUSED(L);
601	return PROFILE_ERRUSE;
602	}
603
604	int lj_sysprof_stop(lua_State *L)
605	{
606	UNUSED(L);
607	return PROFILE_ERRUSE;
608	}
609
610	int lj_sysprof_report(struct luam_Sysprof_Counters *counters)
611	{
612	UNUSED(counters);
613	return PROFILE_ERRUSE;
614	}
615
616	void lj_sysprof_add_proto(const struct GCproto *pt)
617	{
618	UNUSED(pt);
619	}
620
621	#if LJ_HASJIT
622	void lj_sysprof_add_trace(const struct GCtrace *tr)
623	{
624	UNUSED(tr);
625	}
626	#endif /* LJ_HASJIT */
627
628	#endif /* LJ_HASSYSPROF */

tarantool / luajit / 19798590866

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous