Separate hot counters

src/Makefile

@@ -426,6 +426,9 @@ ifneq (,$(findstring LJ_NO_UNWIND 1,$(TARGET_TESTARCH)))
   DASM_AFLAGS+= -D NO_UNWIND
   TARGET_ARCH+= -DLUAJIT_NO_UNWIND
 endif
+ifneq (,$(findstring LJ_SEPARATE_COUNTERS 1,$(TARGET_TESTARCH)))
+  DASM_AFLAGS+= -D SEPARATE_COUNTERS
+endif
 DASM_AFLAGS+= -D VER=$(subst LJ_ARCH_VERSION_,,$(filter LJ_ARCH_VERSION_%,$(subst LJ_ARCH_VERSION ,LJ_ARCH_VERSION_,$(TARGET_TESTARCH))))
 ifeq (Windows,$(TARGET_SYS))
   DASM_AFLAGS+= -D WIN

src/Makefile.dep

@@ -54,9 +54,9 @@ lj_asm.o: lj_asm.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_gc.h \
 lj_bc.o: lj_bc.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h lj_bc.h \
  lj_bcdef.h
 lj_bcread.o: lj_bcread.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
- lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_bc.h \
- lj_ctype.h lj_cdata.h lualib.h lj_lex.h lj_bcdump.h lj_state.h \
- lj_strfmt.h
+ lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_tab.h lj_bc.h lj_jit.h \
+ lj_ir.h lj_dispatch.h lj_ctype.h lj_cdata.h lualib.h lj_lex.h \
+ lj_bcdump.h lj_state.h lj_strfmt.h
 lj_bcwrite.o: lj_bcwrite.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_gc.h lj_buf.h lj_str.h lj_bc.h lj_ctype.h lj_dispatch.h lj_jit.h \
  lj_ir.h lj_strfmt.h lj_bcdump.h lj_lex.h lj_err.h lj_errmsg.h lj_vm.h
@@ -162,12 +162,11 @@ lj_opt_narrow.o: lj_opt_narrow.c lj_obj.h lua.h luaconf.h lj_def.h \
 lj_opt_sink.o: lj_opt_sink.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_ir.h lj_jit.h lj_iropt.h lj_target.h lj_target_*.h
 lj_opt_split.o: lj_opt_split.c lj_obj.h lua.h luaconf.h lj_def.h \
- lj_arch.h lj_err.h lj_errmsg.h lj_buf.h lj_gc.h lj_str.h lj_ir.h \
- lj_jit.h lj_ircall.h lj_iropt.h lj_dispatch.h lj_bc.h lj_vm.h
+ lj_arch.h
 lj_parse.o: lj_parse.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_gc.h lj_err.h lj_errmsg.h lj_debug.h lj_buf.h lj_str.h lj_tab.h \
  lj_func.h lj_state.h lj_bc.h lj_ctype.h lj_strfmt.h lj_lex.h lj_parse.h \
- lj_vm.h lj_vmevent.h
+ lj_vm.h lj_vmevent.h lj_dispatch.h lj_jit.h lj_ir.h
 lj_profile.o: lj_profile.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_buf.h lj_gc.h lj_str.h lj_frame.h lj_bc.h lj_debug.h lj_dispatch.h \
  lj_jit.h lj_ir.h lj_trace.h lj_traceerr.h lj_profile.h luajit.h
@@ -240,7 +239,7 @@ host/buildvm_fold.o: host/buildvm_fold.c host/buildvm.h lj_def.h lua.h \
  luaconf.h lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_ir.h lj_obj.h
 host/buildvm_lib.o: host/buildvm_lib.c host/buildvm.h lj_def.h lua.h luaconf.h \
  lj_arch.h lj_obj.h lj_def.h lj_arch.h lj_bc.h lj_lib.h lj_obj.h \
- host/buildvm_libbc.h
+ host/buildvm_libbc.h host/buildvm_libbc_counters.h
 host/buildvm_peobj.o: host/buildvm_peobj.c host/buildvm.h lj_def.h lua.h \
  luaconf.h lj_arch.h lj_bc.h lj_def.h lj_arch.h
 host/minilua.o: host/minilua.c

src/host/buildvm_lib.c

@@ -7,7 +7,12 @@
 #include "lj_obj.h"
 #include "lj_bc.h"
 #include "lj_lib.h"
+
+#if LJ_SEPARATE_COUNTERS
+#include "buildvm_libbc_counters.h"
+#else
 #include "buildvm_libbc.h"
+#endif
 
 /* Context for library definitions. */
 static uint8_t obuf[8192];

src/host/buildvm_libbc_counters.h

@@ -0,0 +1,56 @@
+/* This is a generated file. DO NOT EDIT! */
+
+static const int libbc_endian = 0;
+
+static const uint8_t libbc_code[] = {
+#if LJ_FR2
+0,1,2,0,0,1,2,24,1,0,0,76,1,2,0,241,135,158,166,3,220,203,178,130,4,0,1,2,0,
+0,1,2,24,1,0,0,76,1,2,0,243,244,148,165,20,198,190,199,252,3,0,1,2,0,0,0,3,
+16,0,5,0,21,1,0,0,76,1,2,0,0,2,10,0,0,0,16,16,0,12,0,16,1,9,0,41,2,1,0,21,3,
+0,0,41,4,1,0,77,2,8,128,18,6,1,0,18,8,5,0,59,9,5,0,66,6,3,2,10,6,0,0,89,7,1,
+128,76,6,2,0,79,2,248,127,88,0,60,0,75,0,1,0,0,2,11,0,0,0,17,16,0,12,0,16,1,
+9,0,43,2,0,0,18,3,0,0,41,4,0,0,89,5,7,128,18,7,1,0,18,9,5,0,18,10,6,0,66,7,
+3,2,10,7,0,0,89,8,1,128,76,7,2,0,70,5,3,3,82,5,247,127,88,0,60,0,75,0,1,0,0,
+1,2,0,0,0,3,16,0,12,0,21,1,0,0,76,1,2,0,0,2,10,0,0,2,31,16,0,12,0,21,2,0,0,
+11,1,0,0,89,3,7,128,8,2,0,0,89,3,24,128,59,3,2,0,43,4,0,0,64,4,2,0,76,3,2,0,
+89,3,19,128,16,1,14,0,41,3,1,0,3,3,1,0,89,3,15,128,3,1,2,0,89,3,13,128,59,3,
+1,0,22,4,1,1,18,5,2,0,41,6,1,0,77,4,4,128,23,8,1,7,59,9,7,0,64,9,8,0,79,4,252,
+127,88,0,60,0,43,4,0,0,64,4,2,0,76,3,2,0,75,0,1,0,0,2,0,5,12,0,0,0,37,16,0,
+12,0,16,1,14,0,16,2,14,0,16,3,14,0,11,4,0,0,89,5,1,128,18,4,0,0,16,4,12,0,3,
+1,2,0,89,5,26,128,33,5,1,3,0,2,3,0,89,6,4,128,2,3,1,0,89,6,2,128,4,4,0,0,89,
+6,10,128,18,6,1,0,18,7,2,0,41,8,1,0,77,6,4,128,32,10,5,9,59,11,9,0,64,11,10,
+4,79,6,252,127,88,0,60,0,89,6,9,128,18,6,2,0,18,7,1,0,41,8,255,255,77,6,4,128,
+32,10,5,9,59,11,9,0,64,11,10,4,79,6,252,127,88,0,60,0,76,4,2,0,0
+#else
+0,1,2,0,0,1,2,24,1,0,0,76,1,2,0,241,135,158,166,3,220,203,178,130,4,0,1,2,0,
+0,1,2,24,1,0,0,76,1,2,0,243,244,148,165,20,198,190,199,252,3,0,1,2,0,0,0,3,
+16,0,5,0,21,1,0,0,76,1,2,0,0,2,9,0,0,0,16,16,0,12,0,16,1,9,0,41,2,1,0,21,3,
+0,0,41,4,1,0,77,2,8,128,18,6,1,0,18,7,5,0,59,8,5,0,66,6,3,2,10,6,0,0,89,7,1,
+128,76,6,2,0,79,2,248,127,88,0,60,0,75,0,1,0,0,2,10,0,0,0,17,16,0,12,0,16,1,
+9,0,43,2,0,0,18,3,0,0,41,4,0,0,89,5,7,128,18,7,1,0,18,8,5,0,18,9,6,0,66,7,3,
+2,10,7,0,0,89,8,1,128,76,7,2,0,70,5,3,3,82,5,247,127,88,0,60,0,75,0,1,0,0,1,
+2,0,0,0,3,16,0,12,0,21,1,0,0,76,1,2,0,0,2,10,0,0,2,31,16,0,12,0,21,2,0,0,11,
+1,0,0,89,3,7,128,8,2,0,0,89,3,24,128,59,3,2,0,43,4,0,0,64,4,2,0,76,3,2,0,89,
+3,19,128,16,1,14,0,41,3,1,0,3,3,1,0,89,3,15,128,3,1,2,0,89,3,13,128,59,3,1,
+0,22,4,1,1,18,5,2,0,41,6,1,0,77,4,4,128,23,8,1,7,59,9,7,0,64,9,8,0,79,4,252,
+127,88,0,60,0,43,4,0,0,64,4,2,0,76,3,2,0,75,0,1,0,0,2,0,5,12,0,0,0,37,16,0,
+12,0,16,1,14,0,16,2,14,0,16,3,14,0,11,4,0,0,89,5,1,128,18,4,0,0,16,4,12,0,3,
+1,2,0,89,5,26,128,33,5,1,3,0,2,3,0,89,6,4,128,2,3,1,0,89,6,2,128,4,4,0,0,89,
+6,10,128,18,6,1,0,18,7,2,0,41,8,1,0,77,6,4,128,32,10,5,9,59,11,9,0,64,11,10,
+4,79,6,252,127,88,0,60,0,89,6,9,128,18,6,2,0,18,7,1,0,41,8,255,255,77,6,4,128,
+32,10,5,9,59,11,9,0,64,11,10,4,79,6,252,127,88,0,60,0,76,4,2,0,0
+#endif
+};
+
+static const struct { const char *name; int ofs; } libbc_map[] = {
+{"math_deg",0},
+{"math_rad",25},
+{"string_len",50},
+{"table_foreachi",69},
+{"table_foreach",140},
+{"table_getn",215},
+{"table_remove",234},
+{"table_move",367},
+{NULL,522}
+};
+

src/lib_jit.c

@@ -98,6 +98,55 @@ static void flagbits_to_strings(lua_State *L, uint32_t flags, uint32_t base,
 }
 #endif
 
+static GCproto *check_Lproto(lua_State *L, int nolua);
+
+LJLIB_CF(jit_sethot)
+{
+  GCproto *pt = check_Lproto(L, 0);
+  int32_t count = lj_lib_checkint(L, 2);
+  int32_t loopid = lj_lib_optint(L, 3, -1);
+  /*
+  ** Loops decrement the count by two instead of one like functions when using
+  ** shared hot counters.
+  */
+  if (!LJ_SEPARATE_COUNTERS && loopid != -1) {
+    count = count * 2;
+  }
+
+  if (count < 0 || count > 0xffff) {
+    luaL_error(L, "bad hot count value");
+  }
+
+  if (loopid == -1) {
+    int old = hotcount_get_pt(L2GG(L), pt, proto_bc(pt));
+    hotcount_set_pt(L2GG(L), pt, proto_bc(pt), count);
+    setintV(L->top-1, old);
+    return 1;
+  } else if (loopid > 0) {
+    BCIns *bc = proto_bc(pt);
+    MSize hci = 0, i = 0;
+    for (i = 0; i != pt->sizebc; i++) {
+#if LJ_SEPARATE_COUNTERS
+      int iscountbc = bc_op(bc[i]) == BC_LOOPHC;
+#else
+      int iscountbc = bc_op(bc[i]) == BC_FORL || bc_op(bc[i]) == BC_ITERL ||
+                      bc_op(bc[i]) == BC_LOOP;
+#endif
+      if (iscountbc) {
+        if (++hci == loopid) {
+          BCIns *hcbc = bc + i - LJ_SEPARATE_COUNTERS;
+          int old = hotcount_get_loop(L2GG(L), hcbc);
+          hotcount_set_loop(L2GG(L), hcbc, count);
+          setintV(L->top-1, old);
+          return 1;
+        }
+      }
+    }
+  }
+  lj_err_callerv(L, LJ_ERR_IDXRNG);
+  return 0;
+}
+
 LJLIB_CF(jit_status)
 {
 #if LJ_HASJIT

src/lj_arch.h

@@ -498,6 +498,12 @@
 #define LJ_HASJIT		1
 #endif
 
+#if !LUAJIT_NO_SEPARATE_COUNTERS && LJ_HASJIT && LJ_TARGET_X86ORX64
+#define LJ_SEPARATE_COUNTERS 1
+#else
+#define LJ_SEPARATE_COUNTERS 0
+#endif
+
 /* Disable or enable the FFI extension. */
 #if defined(LUAJIT_DISABLE_FFI) || defined(LJ_ARCH_NOFFI)
 #define LJ_HASFFI		0

src/lj_bc.h

@@ -56,6 +56,12 @@
   (((BCIns)(o))|((BCIns)(a)<<8)|((BCIns)(d)<<16))
 #define BCINS_AJ(o, a, j)	BCINS_AD(o, a, (BCPos)((int32_t)(j)+BCBIAS_J))
 
+#if LJ_SEPARATE_COUNTERS
+  #define BCDEF_LOOPHC(_)  _(LOOPHC, ___, ___, lit, ___)
+#else
+  #define BCDEF_LOOPHC(_) 
+#endif
+
 /* Bytecode instruction definition. Order matters, see below.
 **
 ** (name, filler, Amode, Bmode, Cmode or Dmode, metamethod)
@@ -183,6 +189,7 @@
   _(LOOP,	rbase,	___,	jump,	___) \
   _(ILOOP,	rbase,	___,	jump,	___) \
   _(JLOOP,	rbase,	___,	lit,	___) \
+  BCDEF_LOOPHC(_) \
   \
   _(JMP,	rbase,	___,	jump,	___) \
   \

src/lj_bcread.c

@@ -13,6 +13,8 @@
 #include "lj_str.h"
 #include "lj_tab.h"
 #include "lj_bc.h"
+#include "lj_jit.h"
+#include "lj_dispatch.h"
 #if LJ_HASFFI
 #include "lj_ctype.h"
 #include "lj_cdata.h"
@@ -381,6 +383,9 @@ GCproto *lj_bcread_proto(LexState *ls)
     setmref(pt->uvinfo, NULL);
     setmref(pt->varinfo, NULL);
   }
+#if LJ_SEPARATE_COUNTERS
+  pt->hotcount = L2J(ls->L)->param[JIT_P_hotfunc] - 1;
+#endif
   return pt;
 }
 

src/lj_dispatch.c

@@ -464,7 +464,7 @@ static int call_init(lua_State *L, GCfunc *fn)
 }
 
 /* Call dispatch. Used by call hooks, hot calls or when recording. */
-ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
+ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, BCIns *pc)
 {
   ERRNO_SAVE
   GCfunc *fn = curr_func(L);
@@ -480,7 +480,7 @@ ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
 #ifdef LUA_USE_ASSERT
     ptrdiff_t delta = L->top - L->base;
 #endif
-    pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
+    pc = (BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
     lj_trace_hot(J, pc);
     lua_assert(L->top - L->base == delta);
     goto out;

src/lj_dispatch.h

@@ -114,11 +114,28 @@ typedef struct GG_State {
 #define GG_DISP2HOT	(GG_OFS(hotcount) - GG_OFS(dispatch))
 #define GG_DISP2STATIC	(GG_LEN_DDISP*(int)sizeof(ASMFunction))
 
+#if LJ_SEPARATE_COUNTERS
+#define hotcount_get_pt(gg, pt, pc) ((pt)->hotcount)
+#define hotcount_set_pt(gg, pt, pc, value) ((pt)->hotcount = (value))
+
+#define hotcount_get_loop(gg, pc) \
+  ((HotCount)(((pc)[1]) >> 16))
+
+#define hotcount_set_loop(gg, pc, val) \
+  (pc)[1] = (((pc)[1] & 0xffff) | ((val) << 16))
+#else
 #define hotcount_get(gg, pc) \
   (gg)->hotcount[(u32ptr(pc)>>2) & (HOTCOUNT_SIZE-1)]
 #define hotcount_set(gg, pc, val) \
   (hotcount_get((gg), (pc)) = (HotCount)(val))
 
+#define hotcount_get_pt(gg, pt, pc) ((void)pt, hotcount_get(gg, (pc)+1))
+#define hotcount_set_pt(gg, pt, pc, val) (hotcount_set(gg, (pc)+1, val), (void)pt)
+
+#define hotcount_get_loop(gg, pc) hotcount_get(gg, (pc)+1)
+#define hotcount_set_loop(gg, pc, val) hotcount_set(gg, (pc)+1, val)
+#endif
+
 /* Dispatch table management. */
 LJ_FUNC void lj_dispatch_init(GG_State *GG);
 #if LJ_HASJIT
@@ -128,7 +145,7 @@ LJ_FUNC void lj_dispatch_update(global_State *g);
 
 /* Instruction dispatch callback for hooks or when recording. */
 LJ_FUNCA void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc);
-LJ_FUNCA ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns*pc);
+LJ_FUNCA ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, BCIns*pc);
 #if LJ_HASJIT
 LJ_FUNCA void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc);
 #endif

src/lj_jit.h

@@ -95,6 +95,17 @@
 #define JIT_P_sizemcode_DEFAULT		32
 #endif
 
+#define funchot_scale 2
+
+#if LJ_SEPARATE_COUNTERS
+#define JIT_PARAMDEF_COUNTERS(_) \
+  _(\007, hotfunc, (56*2))	        /* # of iter. to detect a hot function. */ \
+  _(\013, penaltyfunc, (36*2))      /* initial penalty hot counter back off value  */ \
+  _(\016, penaltymaxfunc, (25000*2)) /* max the function hot count backoff can be increased to  */ 
+#else
+#define JIT_PARAMDEF_COUNTERS(_)
+#endif
+
 /* Optimization parameters and their defaults. Length is a char in octal! */
 #define JIT_PARAMDEF(_) \
   _(\010, maxtrace,	1000)	/* Max. # of traces in cache. */ \
@@ -104,7 +115,9 @@
   _(\007, maxsnap,	500)	/* Max. # of snapshots for a trace. */ \
   _(\011, minstitch,	0)	/* Min. # of IR ins for a stitched trace. */ \
   \
-  _(\007, hotloop,	56)	/* # of iter. to detect a hot loop/call. */ \
+  _(\007, hotloop,	56)	/* # of iter. to detect a hot loop. */ \
+  _(\013, penaltyloop, 36) /*. initial penalty hot counter back off value */ \
+  _(\016, penaltymaxloop, 25000) /* max the loop hot count backoff can be increased to */ \
   _(\007, hotexit,	10)	/* # of taken exits to start a side trace. */ \
   _(\007, tryside,	4)	/* # of attempts to compile a side trace. */ \
   \
@@ -117,6 +130,7 @@
   _(\011, sizemcode,	JIT_P_sizemcode_DEFAULT) \
   /* Max. total size of all machine code areas (in KBytes). */ \
   _(\010, maxmcode,	512) \
+  JIT_PARAMDEF_COUNTERS(_) \
   /* End of list. */
 
 enum {
@@ -288,8 +302,6 @@ typedef struct HotPenalty {
 } HotPenalty;
 
 #define PENALTY_SLOTS	64	/* Penalty cache slot. Must be a power of 2. */
-#define PENALTY_MIN	(36*2)	/* Minimum penalty value. */
-#define PENALTY_MAX	60000	/* Maximum penalty value. */
 #define PENALTY_RNDBITS	4	/* # of random bits to add to penalty value. */
 
 /* Round-robin backpropagation cache for narrowing conversions. */

src/lj_obj.h

@@ -377,6 +377,10 @@ typedef struct GCproto {
   MRef lineinfo;	/* Compressed map from bytecode ins. to source line. */
   MRef uvinfo;		/* Upvalue names. */
   MRef varinfo;		/* Names and compressed extents of local variables. */
+#if LJ_HASJIT
+  uint16_t unused;
+  uint16_t hotcount;    /* Hot counter. */
+#endif
 } GCproto;
 
 /* Flags for prototype. */