Merge branch 'LuaJIT:v2.1' into v2.1

Author: magicaldave

Makefile

@@ -37,12 +37,13 @@ export MULTILIB= lib
 DPREFIX= $(DESTDIR)$(PREFIX)
 INSTALL_BIN=   $(DPREFIX)/bin
 INSTALL_LIB=   $(DPREFIX)/$(MULTILIB)
-INSTALL_SHARE= $(DPREFIX)/share
+INSTALL_SHARE_= $(PREFIX)/share
+INSTALL_SHARE= $(DESTDIR)$(INSTALL_SHARE_)
 INSTALL_DEFINC= $(DPREFIX)/include/luajit-$(MMVERSION)
 INSTALL_INC=   $(INSTALL_DEFINC)
 
-INSTALL_LJLIBD= $(INSTALL_SHARE)/luajit-$(MMVERSION)
-INSTALL_JITLIB= $(INSTALL_LJLIBD)/jit
+export INSTALL_LJLIBD= $(INSTALL_SHARE_)/luajit-$(MMVERSION)
+INSTALL_JITLIB= $(DESTDIR)$(INSTALL_LJLIBD)/jit
 INSTALL_LMODD= $(INSTALL_SHARE)/lua
 INSTALL_LMOD= $(INSTALL_LMODD)/$(ABIVER)
 INSTALL_CMODD= $(INSTALL_LIB)/lua
@@ -71,7 +72,7 @@ INSTALL_PC= $(INSTALL_PKGCONFIG)/$(INSTALL_PCNAME)
 
 INSTALL_DIRS= $(INSTALL_BIN) $(INSTALL_LIB) $(INSTALL_INC) $(INSTALL_MAN) \
   $(INSTALL_PKGCONFIG) $(INSTALL_JITLIB) $(INSTALL_LMOD) $(INSTALL_CMOD)
-UNINSTALL_DIRS= $(INSTALL_JITLIB) $(INSTALL_LJLIBD) $(INSTALL_INC) \
+UNINSTALL_DIRS= $(INSTALL_JITLIB) $(DESTDIR)$(INSTALL_LJLIBD) $(INSTALL_INC) \
   $(INSTALL_LMOD) $(INSTALL_LMODD) $(INSTALL_CMOD) $(INSTALL_CMODD)
 
 RM= rm -f
@@ -114,6 +115,7 @@ ifeq (Darwin,$(TARGET_SYS))
   INSTALL_SOSHORT1= $(INSTALL_DYLIBSHORT1)
   INSTALL_SOSHORT2= $(INSTALL_DYLIBSHORT2)
   LDCONFIG= :
+  SED_PC+= -e "s| -Wl,-E||"
 endif
 
 ##############################################################################

doc/ext_buffer.html

@@ -85,7 +85,7 @@ <h1>String Buffer Library</h1>
 </p>
 <p>
 The string buffer library also includes a high-performance
-<a href="serialize">serializer</a> for Lua objects.
+<a href="#serialize">serializer</a> for Lua objects.
 </p>
 
 <h2 id="use">Using the String Buffer Library</h2>
@@ -588,9 +588,9 @@ <h3 id="serialize_format">Serialization Format Specification</h3>
 tab       → 0x08                                   // Empty table
           | 0x09 h.U h*{object object}          // Key/value hash
           | 0x0a a.U a*object                    // 0-based array
-          | 0x0b a.U a*object h.U h*{object object}      // Mixed
+          | 0x0b a.U h.U a*object h*{object object}      // Mixed
           | 0x0c a.U (a-1)*object                // 1-based array
-          | 0x0d a.U (a-1)*object h.U h*{object object}  // Mixed
+          | 0x0d a.U h.U (a-1)*object h*{object object}  // Mixed
 tab_mt    → 0x0e (index-1).U tab          // Metatable dict entry
 
 int64     → 0x10 int.L                             // FFI int64_t

doc/ext_ffi_semantics.html

@@ -440,6 +440,19 @@ <h3 id="convert_vararg">Conversions for vararg C&nbsp;function arguments</h3>
 conversion rule applies. A vararg C&nbsp;function expecting an integer
 will see a garbled or uninitialized value.
 </p>
+<p>
+Note: this is the only place where creating a boxed scalar number type is
+actually useful. <b>Never use <tt>ffi.new("int")</tt>, <tt>ffi.new("float")</tt>
+etc. anywhere else!</b>
+</p>
+<p style="font-size: 8pt;">
+Ditto for <tt>ffi.cast()</tt>. Explicitly boxing scalars <b>does not</b>
+improve performance or force <tt>int</tt> or <tt>float</tt> arithmetic! It
+just adds costly boxing, unboxing and conversions steps. And it may lead
+to surprise results, because
+<a href="#cdata_arith">cdata arithmetic on scalar numbers</a>
+is always performed on 64 bit integers.
+</p>
 
 <h2 id="init">Initializers</h2>
 <p>

doc/extensions.html

@@ -160,13 +160,33 @@ <h3 id="xpcall"><tt>xpcall(f, err [,args...])</tt> passes arguments</h3>
 which is called in a protected context.
 </p>
 
-<h3 id="load"><tt>loadfile()</tt> etc. handle UTF-8 source code</h3>
+<h3 id="load"><tt>load*()</tt> handle UTF-8 source code</h3>
 <p>
 Non-ASCII characters are handled transparently by the Lua source code parser.
 This allows the use of UTF-8 characters in identifiers and strings.
 A UTF-8 BOM is skipped at the start of the source code.
 </p>
 
+<h3 id="load_mode"><tt>load*()</tt> add a mode parameter</h3>
+<p>
+As an extension from Lua 5.2, the functions <tt>loadstring()</tt>,
+<tt>loadfile()</tt> and (new) <tt>load()</tt> add an optional
+<tt>mode</tt> parameter.
+</p>
+<p>
+The default mode string is <tt>"bt"</tt>, which allows loading of both
+source code and bytecode. Use <tt>"t"</tt> to allow only source code
+or <tt>"b"</tt> to allow only bytecode to be loaded.
+</p>
+<p>
+By default, the <tt>load*</tt> functions generate the native bytecode format.
+For cross-compilation purposes, add <tt>W</tt> to the mode string to
+force the 32 bit format and <tt>X</tt> to force the 64 bit format.
+Add both to force the opposite format. Note that non-native bytecode
+generated by <tt>load*</tt> cannot be run, but can still be passed
+to <tt>string.dump</tt>.
+</p>
+
 <h3 id="tostring"><tt>tostring()</tt> etc. canonicalize NaN and &plusmn;Inf</h3>
 <p>
 All number-to-string conversions consistently convert non-finite numbers
@@ -186,26 +206,33 @@ <h3 id="tonumber"><tt>tonumber()</tt> etc. use builtin string to number conversi
 numbers (e.g. <tt>0x1.5p-3</tt>).
 </p>
 
-<h3 id="string_dump"><tt>string.dump(f [,strip])</tt> generates portable bytecode</h3>
+<h3 id="string_dump"><tt>string.dump(f [,mode])</tt> generates portable bytecode</h3>
 <p>
 An extra argument has been added to <tt>string.dump()</tt>. If set to
-<tt>true</tt>, 'stripped' bytecode without debug information is
-generated. This speeds up later bytecode loading and reduces memory
-usage. See also the
+<tt>true</tt> or to a string which contains the character <tt>s</tt>,
+'stripped' bytecode without debug information is generated. This speeds
+up later bytecode loading and reduces memory usage. See also the
 <a href="running.html#opt_b"><tt>-b</tt> command line option</a>.
 </p>
 <p>
 The generated bytecode is portable and can be loaded on any architecture
-that LuaJIT supports, independent of word size or endianess. However, the
-bytecode compatibility versions must match. Bytecode stays compatible
-for dot releases (x.y.0 &rarr; x.y.1), but may change with major or
-minor releases (2.0 &rarr; 2.1) or between any beta release. Foreign
-bytecode (e.g. from Lua 5.1) is incompatible and cannot be loaded.
+that LuaJIT supports. However, the bytecode compatibility versions must
+match. Bytecode only stays compatible within a major+minor version
+(x.y.aaa &rarr; x.y.bbb), except for development branches. Foreign bytecode
+(e.g. from Lua 5.1) is incompatible and cannot be loaded.
 </p>
 <p>
 Note: <tt>LJ_GC64</tt> mode requires a different frame layout, which implies
-a different, incompatible bytecode format for all 64 bit ports. This may be
-rectified in the future.
+a different, incompatible bytecode format between 32 bit and 64 bit ports.
+This may be rectified in the future. In the meantime, use the <tt>W</tt>
+and </tt>X</tt> <a href="#load_mode">modes of the <tt>load*</tt> functions</a>
+for cross-compilation purposes.
+</p>
+<p>
+Due to VM hardening, bytecode is not deterministic. Add <tt>d</tt> to the
+mode string to dump it in a deterministic manner: identical source code
+always gives a byte-for-byte identical bytecode dump. This feature is
+mainly useful for reproducible builds.
 </p>
 
 <h3 id="table_new"><tt>table.new(narray, nhash)</tt> allocates a pre-sized table</h3>
@@ -238,7 +265,7 @@ <h3 id="math_random">Enhanced PRNG for <tt>math.random()</tt></h3>
 LuaJIT uses a Tausworthe PRNG with period 2^223 to implement
 <tt>math.random()</tt> and <tt>math.randomseed()</tt>. The quality of
 the PRNG results is much superior compared to the standard Lua
-implementation, which uses the platform-specific ANSI rand().
+implementation, which uses the platform-specific ANSI <tt>rand()</tt>.
 </p>
 <p>
 The PRNG generates the same sequences from the same seeds on all
@@ -249,6 +276,10 @@ <h3 id="math_random">Enhanced PRNG for <tt>math.random()</tt></h3>
 preserve uniformity.
 </p>
 <p>
+Call <tt>math.randomseed()</tt> without any arguments to seed it from
+system entropy.
+</p>
+<p>
 Important: Neither this nor any other PRNG based on the simplistic
 <tt>math.random()</tt> API is suitable for cryptographic use.
 </p>
@@ -286,7 +317,7 @@ <h2 id="lua52">Extensions from Lua 5.2</h2>
 </p>
 <ul>
 <li><tt>goto</tt> and <tt>::labels::</tt>.</li>
-<li>Hex escapes <tt>'\x3F'</tt> and <tt>'\*'</tt> escape in strings.</li>
+<li>Hex escapes <tt>'\x3F'</tt> and <tt>'\z'</tt> escape in strings.</li>
 <li><tt>load(string|reader [, chunkname [,mode [,env]]])</tt>.</li>
 <li><tt>loadstring()</tt> is an alias for <tt>load()</tt>.</li>
 <li><tt>loadfile(filename [,mode [,env]])</tt>.</li>

doc/install.html

@@ -269,6 +269,7 @@ <h2 id="cross">Cross-compiling LuaJIT</h2>
 <li>Yes, you need a toolchain for both your host <em>and</em> your target!</li>
 <li>Both host and target architectures must have the same pointer size.</li>
 <li>E.g. if you want to cross-compile to a 32 bit target on a 64 bit host, you need to install the multilib development package (e.g. <tt>libc6-dev-i386</tt> on Debian/Ubuntu) and build a 32 bit host part (<tt>HOST_CC="gcc -m32"</tt>).</li>
+<li>On some distro versions, multilib conflicts with cross-compilers. The workaround is to install the x86 cross-compiler package <tt>gcc-i686-linux-gnu</tt> and use it to build the host part (<tt>HOST_CC=i686-linux-gnu-gcc</tt>).</li>
 <li>64 bit targets always require compilation on a 64 bit host.</li>
 </ul>
 <p>

doc/running.html

@@ -106,6 +106,9 @@ <h3 id="opt_b"><tt>-b[options] input output</tt></h3>
 <li><tt>-l</tt> &mdash; Only list bytecode.</li>
 <li><tt>-s</tt> &mdash; Strip debug info (this is the default).</li>
 <li><tt>-g</tt> &mdash; Keep debug info.</li>
+<li><tt>-W</tt> &mdash; Generate 32 bit (non-GC64) bytecode.</li>
+<li><tt>-X</tt> &mdash; Generate 64 bit (GC64) bytecode.</li>
+<li><tt>-d</tt> &mdash; Generate bytecode in deterministic manner.</li>
 <li><tt>-n name</tt> &mdash; Set module name (default: auto-detect from input name)</li>
 <li><tt>-t type</tt> &mdash; Set output file type (default: auto-detect from output name).</li>
 <li><tt>-a arch</tt> &mdash; Override architecture for object files (default: native).</li>
@@ -120,7 +123,8 @@ <h3 id="opt_b"><tt>-b[options] input output</tt></h3>
 </p>
 <ul>
 <li><tt>c</tt> &mdash; C source file, exported bytecode data.</li>
-<li><tt>h</tt> &mdash; C header file, static bytecode data.</li>
+<li><tt>cc</tt> &mdash; C++ source file, exported bytecode data.</li>
+<li><tt>h</tt> &mdash; C/C++ header file, static bytecode data.</li>
 <li><tt>obj</tt> or <tt>o</tt> &mdash; Object file, exported bytecode data
 (OS- and architecture-specific).</li>
 <li><tt>raw</tt> or any other extension &mdash; Raw bytecode file (portable).

dynasm/dasm_x86.lua

@@ -627,7 +627,11 @@ local function wputmrmsib(t, imark, s, vsreg, psz, sk)
 	werror("NYI: rip-relative displacement followed by immediate")
       end
       -- The previous byte in the action buffer cannot be 0xe9 or 0x80-0x8f.
-      wputlabel("REL_", disp[1], 2)
+      if disp[2] == "iPJ" then
+	waction("REL_A", disp[1])
+      else
+	wputlabel("REL_", disp[1], 2)
+      end
     else
       wputdarg(disp)
     end
@@ -744,9 +748,9 @@ local function dispexpr(expr)
     return imm*map_opsizenum[ops]
   end
   local mode, iexpr = immexpr(dispt)
-  if mode == "iJ" then
+  if mode == "iJ" or mode == "iPJ" then
     if c == "-" then werror("cannot invert label reference") end
-    return { iexpr }
+    return { iexpr, mode }
   end
   return expr -- Need to return original signed expression.
 end
@@ -1147,6 +1151,8 @@ local map_op = {
   rep_0 =	"F3",
   repe_0 =	"F3",
   repz_0 =	"F3",
+  endbr32_0 =	"F30F1EFB",
+  endbr64_0 =	"F30F1EFA",
   -- F4: *hlt
   cmc_0 =	"F5",
   -- F6: test... mb,i; div... mb

dynasm/dynasm.lua

@@ -75,7 +75,7 @@ local function wline(line, needindent)
   g_synclineno = g_synclineno + 1
 end
 
--- Write assembler line as a comment, if requestd.
+-- Write assembler line as a comment, if requested.
 local function wcomment(aline)
   if g_opt.comment then
     wline(g_opt.comment..aline..g_opt.endcomment, true)

src/Makefile

@@ -301,6 +301,9 @@ endif
 ifneq (,$(LMULTILIB))
   TARGET_XCFLAGS+= -DLUA_LMULTILIB=\"$(LMULTILIB)\"
 endif
+ifneq (,$(INSTALL_LJLIBD))
+  TARGET_XCFLAGS+= -DLUA_LJDIR=\"$(INSTALL_LJLIBD)\"
+endif
 
 ##############################################################################
 # Target system detection.
@@ -322,13 +325,13 @@ ifeq (Darwin,$(TARGET_SYS))
   endif
   TARGET_STRIP+= -x
   TARGET_XCFLAGS+= -DLUAJIT_UNWIND_EXTERNAL
-  TARGET_XSHLDFLAGS= -dynamiclib -single_module -undefined dynamic_lookup -fPIC
+  TARGET_XSHLDFLAGS= -dynamiclib -undefined dynamic_lookup -fPIC
   TARGET_DYNXLDOPTS=
   TARGET_XSHLDFLAGS+= -install_name $(TARGET_DYLIBPATH) -compatibility_version $(MAJVER).$(MINVER) -current_version $(MAJVER).$(MINVER).255
 else
 ifeq (iOS,$(TARGET_SYS))
   TARGET_STRIP+= -x
-  TARGET_XSHLDFLAGS= -dynamiclib -single_module -undefined dynamic_lookup -fPIC
+  TARGET_XSHLDFLAGS= -dynamiclib -undefined dynamic_lookup -fPIC
   TARGET_DYNXLDOPTS=
   TARGET_XSHLDFLAGS+= -install_name $(TARGET_DYLIBPATH) -compatibility_version $(MAJVER).$(MINVER) -current_version $(MAJVER).$(MINVER).255
   ifeq (arm64,$(TARGET_LJARCH))

src/Makefile.dep

@@ -32,7 +32,8 @@ lib_jit.o: lib_jit.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
  lj_target.h lj_target_*.h lj_trace.h lj_dispatch.h lj_traceerr.h \
  lj_vm.h lj_vmevent.h lj_lib.h luajit.h lj_libdef.h
 lib_math.o: lib_math.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h \
- lj_def.h lj_arch.h lj_lib.h lj_vm.h lj_prng.h lj_libdef.h
+ lj_def.h lj_arch.h lj_err.h lj_errmsg.h lj_lib.h lj_vm.h lj_prng.h \
+ lj_libdef.h
 lib_os.o: lib_os.c lua.h luaconf.h lauxlib.h lualib.h lj_obj.h lj_def.h \
  lj_arch.h lj_gc.h lj_err.h lj_errmsg.h lj_buf.h lj_str.h lj_lib.h \
  lj_libdef.h
@@ -97,7 +98,7 @@ lj_crecord.o: lj_crecord.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_cdata.h lj_cparse.h lj_cconv.h lj_carith.h lj_clib.h lj_ccall.h \
  lj_ff.h lj_ffdef.h lj_ir.h lj_jit.h lj_ircall.h lj_iropt.h lj_trace.h \
  lj_dispatch.h lj_traceerr.h lj_record.h lj_ffrecord.h lj_snap.h \
- lj_crecord.h lj_strfmt.h
+ lj_crecord.h lj_strfmt.h lj_strscan.h
 lj_ctype.o: lj_ctype.c lj_obj.h lua.h luaconf.h lj_def.h lj_arch.h \
  lj_gc.h lj_err.h lj_errmsg.h lj_str.h lj_tab.h lj_strfmt.h lj_ctype.h \
  lj_ccallback.h lj_buf.h

src/host/buildvm_asm.c

@@ -339,6 +339,10 @@ void emit_asm(BuildCtx *ctx)
     fprintf(ctx->fp, "\t.ident \"%s\"\n", ctx->dasm_ident);
     break;
   case BUILD_machasm:
+#if defined(__apple_build_version__) && __apple_build_version__ >= 15000000 && __apple_build_version__ < 15000300
+    /* Workaround for XCode 15.0 - 15.2. */
+    fprintf(ctx->fp, "\t.subsections_via_symbols\n");
+#endif
     fprintf(ctx->fp,
       "\t.cstring\n"
       "\t.ascii \"%s\\0\"\n", ctx->dasm_ident);