Run test262 tests in CI

Run test262 tests for Linux, macOS and Cosmopolitan

Add a Makefile `test2-bootstrap` helper to clone and patch test262 tests at
a particular commit. Running with the latest commit noticed a few tests were
failing and added them to the errors list.

The CI environment, especially for macOS, was a bit more unpredictable and was
failing in atomics tests due to lower timeouts in the `atomicsHelper` so double
the timeouts. The time didn't worsen too much. It still takes about 2-3 min.

.github/workflows/ci.yml

@@ -36,6 +36,10 @@ jobs:
       - name: Run microbench
         run: |
           make microbench
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make test2
 
   linux-lto:
     name: Linux LTO
@@ -67,6 +71,7 @@ jobs:
           submodules: true
       - name: Install gcc-multilib
         run: |
+          sudo apt update
           sudo apt install -y gcc-multilib
       - name: Build
         run: |
@@ -138,6 +143,10 @@ jobs:
       - name: Run built-in tests
         run: |
           make test
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make test2
 
   macos-asan:
     runs-on: macos-latest
@@ -173,6 +182,8 @@ jobs:
         uses: vmactions/freebsd-vm@v1
         with:
           usesh: true
+          copyback: false
+          mem: 16384
           prepare: |
             pkg install -y gmake
           run: |
@@ -202,6 +213,10 @@ jobs:
       - name: Run built-in tests
         run: |
           make CONFIG_COSMO=y test
+      - name: Run test262
+        run: |
+          make test2-bootstrap
+          make CONFIG_COSMO=y test2
 
   mingw-windows:
     name: MinGW Windows target

Makefile

@@ -54,6 +54,10 @@ PREFIX?=/usr/local
 # use UB sanitizer
 #CONFIG_UBSAN=y
 
+# TEST262 bootstrap config: commit id and shallow "since" parameter
+TEST262_COMMIT?=af3d908437b0912513a594e7167f17658e72d88b
+TEST262_SINCE?=2025-09-14
+
 OBJDIR=.obj
 
 ifdef CONFIG_ASAN
@@ -464,6 +468,15 @@ stats: qjs$(EXE)
 microbench: qjs$(EXE)
 	$(WINE) ./qjs$(EXE) --std tests/microbench.js
 
+ifeq ($(wildcard test262/features.txt),)
+test2-bootstrap:
+	git clone --single-branch --shallow-since=$(TEST262_SINCE) https://github.com/tc39/test262.git
+	(cd test262 && git checkout -q $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
+else
+test2-bootstrap:
+	(cd test262 && git fetch && git reset --hard $(TEST262_COMMIT) && patch -p1 < ../tests/test262.patch && cd ..)
+endif
+
 ifeq ($(wildcard test262o/tests.txt),)
 test2o test2o-update:
 	@echo test262o tests not installed

TODO

@@ -62,5 +62,5 @@ Optimization ideas:
 Test262o:   0/11262 errors, 463 excluded
 Test262o commit: 7da91bceb9ce7613f87db47ddd1292a2dda58b42 (es5-tests branch)
 
-Result: 54/79414 errors, 1637 excluded, 6821 skipped
+Result: 48/81760 errors, 1631 excluded, 5564 skipped
 Test262 commit: e7e136756cd67c1ffcf7c09d03aeb8ad5a6cec0c

quickjs-atom.h

@@ -78,6 +78,8 @@ DEF(await, "await")
 /* empty string */
 DEF(empty_string, "")
 /* identifiers */
+DEF(keys, "keys")
+DEF(size, "size")
 DEF(length, "length")
 DEF(fileName, "fileName")
 DEF(lineNumber, "lineNumber")
@@ -189,6 +191,7 @@ DEF(timed_out, "timed-out")
 DEF(ok, "ok")
 /* */
 DEF(toJSON, "toJSON")
+DEF(maxByteLength, "maxByteLength")
 /* class names */
 DEF(Object, "Object")
 DEF(Array, "Array")
@@ -228,6 +231,9 @@ DEF(Map, "Map")
 DEF(Set, "Set") /* Map + 1 */
 DEF(WeakMap, "WeakMap") /* Map + 2 */
 DEF(WeakSet, "WeakSet") /* Map + 3 */
+DEF(Iterator, "Iterator")
+DEF(IteratorHelper, "Iterator Helper")
+DEF(IteratorWrap, "Iterator Wrap")
 DEF(Map_Iterator, "Map Iterator")
 DEF(Set_Iterator, "Set Iterator")
 DEF(Array_Iterator, "Array Iterator")

release.sh

@@ -174,7 +174,7 @@ cp Makefile VERSION TODO Changelog readme.txt LICENSE \
 
 cp tests/*.js tests/*.patch tests/bjson.c $outdir/tests
 
-cp examples/*.js examples/*.c $outdir/examples
+cp examples/*.js examples/*.c examples/*.json $outdir/examples
 
 cp doc/quickjs.texi doc/quickjs.pdf doc/quickjs.html \
    $outdir/doc

test262.conf

@@ -64,12 +64,12 @@ Array.prototype.flatMap
 Array.prototype.includes
 Array.prototype.values
 ArrayBuffer
-arraybuffer-transfer=skip
+arraybuffer-transfer
 arrow-function
 async-functions
 async-iteration
 Atomics
-Atomics.pause=skip
+Atomics.pause
 Atomics.waitAsync=skip
 BigInt
 caller
@@ -116,6 +116,7 @@ for-of
 generators
 globalThis
 hashbang
+immutable-arraybuffer=skip
 import-attributes
 import-defer=skip
 import.meta
@@ -140,7 +141,7 @@ Intl.NumberFormat-v3=skip
 Intl.RelativeTimeFormat=skip
 Intl.Segmenter=skip
 IsHTMLDDA
-iterator-helpers=skip
+iterator-helpers
 iterator-sequencing=skip
 json-modules
 json-parse-with-source=skip
@@ -180,10 +181,10 @@ regexp-named-groups
 regexp-unicode-property-escapes
 regexp-v-flag
 RegExp.escape
-resizable-arraybuffer=skip
+resizable-arraybuffer
 rest-parameters
 Set
-set-methods=skip
+set-methods
 ShadowRealm=skip
 SharedArrayBuffer
 source-phase-imports-module-source=skip
@@ -304,14 +305,10 @@ test262/test/built-ins/String/prototype/split/cstm-split-on-string-primitive.js
 # spec updates it in this case)
 test262/test/staging/sm/Array/frozen-dense-array.js
 
+# does not match spec
+test262/test/staging/sm/Iterator/from/wrap-next-not-object-throws.js
+
 # not supported
-test262/test/staging/sm/Set/difference.js
-test262/test/staging/sm/Set/intersection.js
-test262/test/staging/sm/Set/is-disjoint-from.js
-test262/test/staging/sm/Set/is-subset-of.js
-test262/test/staging/sm/Set/is-superset-of.js
-test262/test/staging/sm/Set/symmetric-difference.js
-test262/test/staging/sm/Set/union.js
 test262/test/staging/sm/extensions/censor-strict-caller.js
 test262/test/staging/sm/JSON/parse-with-source.js
 

test262_errors.txt

@@ -5,8 +5,6 @@ test262/test/annexB/language/expressions/assignmenttargettype/callexpression-in-
 test262/test/annexB/language/expressions/assignmenttargettype/callexpression-in-prefix-update.js:27: SyntaxError: invalid increment/decrement operand
 test262/test/annexB/language/expressions/assignmenttargettype/callexpression.js:33: SyntaxError: invalid assignment left-hand side
 test262/test/annexB/language/expressions/assignmenttargettype/cover-callexpression-and-asyncarrowhead.js:20: SyntaxError: invalid assignment left-hand side
-test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: TypeError: ArrayBuffer is detached
-test262/test/built-ins/Atomics/notify/retrieve-length-before-index-coercion-non-shared-detached.js:34: strict mode: TypeError: ArrayBuffer is detached
 test262/test/staging/sm/Date/UTC-convert-all-arguments.js:75: Test262Error: index 1: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
 test262/test/staging/sm/Date/constructor-convert-all-arguments.js:75: Test262Error: index undefined: expected 42, got Error: didn't throw Expected SameValue(«Error: didn't throw», «42») to be true
 test262/test/staging/sm/Date/non-iso.js:76: Test262Error: Expected SameValue(«NaN», «-40071559730000») to be true
@@ -26,19 +24,15 @@ test262/test/staging/sm/RegExp/regress-613820-3.js:12: Test262Error: Actual [aab
 test262/test/staging/sm/RegExp/regress-613820-3.js:12: strict mode: Test262Error: Actual [aab, a, undefined, ab] and expected [aa, undefined, a, undefined] should have the same contents. 
 test262/test/staging/sm/TypedArray/constructor-buffer-sequence.js:73: Error: Assertion failed: expected exception ExpectedError, got Error: Poisoned Value
 test262/test/staging/sm/TypedArray/prototype-constructor-identity.js:17: Test262Error: Expected SameValue(«2», «6») to be true
-test262/test/staging/sm/TypedArray/set-detached-bigint.js:27: Error: Assertion failed: expected exception SyntaxError, got RangeError: invalid array length
-test262/test/staging/sm/TypedArray/set-detached.js:112: RangeError: invalid array length
 test262/test/staging/sm/TypedArray/sort_modifications.js:12: Test262Error: Int8Array at index 0 for size 4 Expected SameValue(«0», «1») to be true
 test262/test/staging/sm/async-functions/async-contains-unicode-escape.js:45: Error: Assertion failed: expected exception SyntaxError, no exception thrown
 test262/test/staging/sm/async-functions/await-error.js:12: Test262Error: Expected SameValue(«false», «true») to be true
 test262/test/staging/sm/async-functions/await-in-arrow-parameters.js:33: Error: Assertion failed: expected exception SyntaxError, no exception thrown - AsyncFunction:(a = (b = await/r/g) => {}) => {}
 test262/test/staging/sm/class/boundFunctionSubclassing.js:12: Test262Error: Expected SameValue(«false», «true») to be true
-test262/test/staging/sm/class/compPropNames.js:26: Error: Expected syntax error: ({[1, 2]: 3})
 test262/test/staging/sm/class/strictExecution.js:32: Error: Assertion failed: expected exception TypeError, no exception thrown
 test262/test/staging/sm/class/superPropOrdering.js:83: Error: Assertion failed: expected exception TypeError, no exception thrown
 test262/test/staging/sm/expressions/short-circuit-compound-assignment-const.js:97: TypeError: 'a' is read-only
 test262/test/staging/sm/expressions/short-circuit-compound-assignment-tdz.js:23: Error: Assertion failed: expected exception ReferenceError, got TypeError: 'a' is read-only
-test262/test/staging/sm/extensions/TypedArray-set-object-funky-length-detaches.js:55: RangeError: invalid array length
 test262/test/staging/sm/generators/syntax.js:30: Error: Assertion failed: expected SyntaxError, but no exception thrown - function* g() { (function* yield() {}); }
 test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-arguments.js:14: Test262Error: Expected SameValue(«"object"», «"function"») to be true
 test262/test/staging/sm/lexical-environment/block-scoped-functions-annex-b-eval.js:12: Test262Error: Expected SameValue(«"outer-gouter-geval-gtruefalseq"», «"outer-geval-gwith-gtruefalseq"») to be true

tests/test262.patch

@@ -14,9 +14,9 @@ index 9828b15..4a5919d 100644
 +//  small: 200,
 +//  long: 1000,
 +//  huge: 10000,
-+  yield: 20,
-+  small: 20,
-+  long: 100,
++  yield: 40,
++  small: 40,
++  long: 200,
 +  huge: 1000,
  };
  

tests/test_bjson.js

@@ -115,6 +115,41 @@ function bjson_test(a)
     }
 }
 
+function bjson_test_arraybuffer()
+{
+    var buf, array_buffer;
+
+    array_buffer = new ArrayBuffer(4);
+    assert(array_buffer.byteLength, 4);
+    assert(array_buffer.maxByteLength, 4);
+    assert(array_buffer.resizable, false);
+    buf = bjson.write(array_buffer);
+    array_buffer = bjson.read(buf, 0, buf.byteLength);
+    assert(array_buffer.byteLength, 4);
+    assert(array_buffer.maxByteLength, 4);
+    assert(array_buffer.resizable, false);
+
+    array_buffer = new ArrayBuffer(4, {maxByteLength: 4});
+    assert(array_buffer.byteLength, 4);
+    assert(array_buffer.maxByteLength, 4);
+    assert(array_buffer.resizable, true);
+    buf = bjson.write(array_buffer);
+    array_buffer = bjson.read(buf, 0, buf.byteLength);
+    assert(array_buffer.byteLength, 4);
+    assert(array_buffer.maxByteLength, 4);
+    assert(array_buffer.resizable, true);
+
+    array_buffer = new ArrayBuffer(4, {maxByteLength: 8});
+    assert(array_buffer.byteLength, 4);
+    assert(array_buffer.maxByteLength, 8);
+    assert(array_buffer.resizable, true);
+    buf = bjson.write(array_buffer);
+    array_buffer = bjson.read(buf, 0, buf.byteLength);
+    assert(array_buffer.byteLength, 4);
+    assert(array_buffer.maxByteLength, 8);
+    assert(array_buffer.resizable, true);
+}
+
 /* test multiple references to an object including circular
    references */
 function bjson_test_reference()
@@ -171,6 +206,7 @@ function bjson_test_all()
         assert(e instanceof TypeError);
     }
 
+    bjson_test_arraybuffer();
     bjson_test_reference();
 }