ParparVM: make the translator self-hosting, and fix what that exposed

.github/workflows/linux-build-run.yml

@@ -270,6 +270,25 @@ jobs:
           # itself is still stripped; only the companion grows, and it is uploaded beside
           # the core so the crash can be autopsied off the runner.
           CN1_LINUX_FULL_DEBUG: 'true'
+          # BiBOP page validation, on the x64 leg only.
+          #
+          # CN1_GC_VERIFY already ran here and came back CLEAN over 2213 epochs, so
+          # the sweep is not reclaiming anything a survivor still references -- the
+          # premise this investigation had been working from is wrong. And the
+          # generated Display.c is semantically identical to master's (14 lines
+          # differ, all of them this PR's class-init guards), so the code that
+          # writes the field is not at fault either.
+          #
+          # What remains is a pointer that LOOKS valid but addresses the wrong
+          # object, which is what a retired-and-reformatted page produces and what
+          # the verifier would not call reclaimed. This guard checks on every fast
+          # allocation that bibopCurrent[ci] is owned, matches its size class, and
+          # that the bumped slot lies inside the page -- and aborts at the
+          # allocation that breaks it. Its comment names "the intermittent x64
+          # cn1BibopFastAlloc crash"; this is an intermittent x64 crash.
+          #
+          # x64 only via the matrix arch: it is the leg that SIGSEGVs.
+          CN1_LINUX_EXTRA_DEFINES: ${{ matrix.arch == 'x64' && 'CN1_BIBOP_VALIDATE' || '' }}
           # Build the native ELF (and the demo) against an old glibc for portability.
           CN1_CC: /usr/local/bin/cn1-zig-cc
           # After the suite runs, the capture test relinks the same objects into a
@@ -283,7 +302,15 @@ jobs:
           # hang-stacks.txt with nothing but sample headers -- which is how four
           # occurrences of the suite stall ended up with no evidence at all. Still
           # best-effort: a runner without gdb must not fail the suite, it must say so.
-          bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed"
+          # $GITHUB_WORKSPACE, not a relative path: this step runs with
+          # working-directory: vm, so "scripts/ci/..." resolved to vm/scripts/ci and
+          # bash answered "No such file or directory". gdb was therefore NEVER
+          # installed on any Linux leg, and every hang-stacks.txt and crash-stacks.txt
+          # this job has ever produced was empty for that reason alone -- including
+          # the ones collected while chasing a SIGSEGV that dumped a 12GB core nobody
+          # could symbolise. The retry.sh call twenty lines below already had this
+          # right.
+          bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed"
           if command -v gdb >/dev/null 2>&1; then
             echo "gdb available: $(gdb --version | head -1)"
           else
@@ -324,11 +351,30 @@ jobs:
             mkdir -p "$(dirname "$CN1_APP_LOG_TEE")"
             {
               echo "=== post-mortem of $core (elf=$elf) ==="
+              # The event-stack arrays are dumped explicitly because the crash this
+              # keeps producing is a store into one of them from
+              # Display.edtLoopImpl, and -O3 optimises the array temporary out of
+              # the frame -- "array=<optimized out>" is all `info locals` gives.
+              # The Display object itself survives as frame 1's __cn1ThisObject, so
+              # the arrays are reachable THROUGH it, and what matters is whether
+              # each header's length and data agree: cn1_set_array_element_int
+              # bounds-checks against ->length and then stores through ->data, so a
+              # fault there means those two fields disagree, which is a corrupted or
+              # dangling header rather than a bad index. Harmless noise when the
+              # faulting frame is something else -- gdb just prints an error.
               gdb "$elf" "$core" -batch -ex 'set pagination off' \
                 -ex 'thread apply all bt' \
                 -ex 'thread 1' -ex 'bt full' \
-                -ex 'frame 0' -ex 'info args' \
+                -ex 'frame 0' -ex 'info args' -ex 'info registers' \
                 -ex 'frame 1' -ex 'info locals' -ex 'info args' \
+                -ex 'set $d = (struct obj__com_codename1_ui_Display*)__cn1ThisObject' \
+                -ex 'p $d->com_codename1_ui_Display_inputEventStackTmp' \
+                -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_inputEventStackTmp' \
+                -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_inputEventStack' \
+                -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_pointerMetaStackTmp' \
+                -ex 'p *(JAVA_ARRAY)$d->com_codename1_ui_Display_pointerMetaStack' \
+                -ex 'p $d->com_codename1_ui_Display_inputEventStackPointerTmp' \
+                -ex 'p $d->com_codename1_ui_Display_inputEventStackPointer' \
                 -ex 'p gcMarkWorklistTop' -ex 'p currentGcMarkValue' \
                 2>&1
             } >> "$(dirname "$CN1_APP_LOG_TEE")/crash-stacks.txt" || true
@@ -338,6 +384,12 @@ jobs:
           # frames (e.g. a stack overflow's recursion / the caller that ran CN1 on a small
           # native stack). The suite binary is not stripped, so addr2line resolves them.
           elf="$(/usr/bin/find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)"
+          # The binary and the cores are NOT copied here. "Package core dump for
+          # offline autopsy" below already ships the unstripped ELF (plus .debug)
+          # and zstd-compresses each core into its own artifact. Copying them into
+          # the screenshot directory as well would upload multi-gigabyte cores twice
+          # on exactly the runs that crashed -- the runs whose screenshots and
+          # backtrace most need to survive the disk and upload budget.
           if [ -n "$elf" ] && [ -f "$CN1_APP_LOG_TEE" ]; then
             {
               echo "=== addr2line of CN1 backtrace addresses (from app log) ==="
@@ -445,17 +497,29 @@ jobs:
           name: linux-suite-classes
       - name: Unpack suite classes
         run: tar xzf suite-classes.tgz
+      # The musl leg had no crash wiring at all: when the suite died mid-run the
+      # only artifact was a short screenshot directory, which is indistinguishable
+      # from a hang. core_pattern is global to the host kernel rather than per
+      # namespace, but the kernel writes the file in the CRASHING process's mount
+      # namespace -- so an absolute path under /cn1 lands inside the container, and
+      # /cn1 is the bind-mounted workspace, which means the core is uploaded with
+      # everything else.
+      - name: Enable core dumps for the container
+        run: |
+          mkdir -p artifacts/linux-port/raw-musl
+          sudo sysctl -w kernel.core_pattern='/cn1/artifacts/linux-port/raw-musl/core.%e.%p' || true
+
       - name: Translate + build + run the suite in Alpine (musl, end-to-end)
         run: |
-          docker run --rm \
+          docker run --rm --ulimit core=-1 \
             -v "$GITHUB_WORKSPACE":/cn1 -w /cn1 \
             -e CN1_SHOT_OUTPUT_DIR=/cn1/artifacts/linux-port/raw-musl \
             -e CN1_APP_LOG_TEE=/cn1/artifacts/linux-port/raw-musl/app-output.log \
             -e CN1_REQUIRE_SUITE=true \
             -e LIBGL_ALWAYS_SOFTWARE=1 \
             docker.io/library/alpine:3.20 sh -ec '
               sed -i "s|^#\(.*/community\)|\1|" /etc/apk/repositories
-              apk add --no-cache build-base cmake samurai pkgconf bash git openjdk8 openjdk17 maven \
+              apk add --no-cache build-base cmake samurai pkgconf bash git openjdk8 openjdk17 maven gdb \
                 gtk+3.0-dev cairo-dev pango-dev gdk-pixbuf-dev glib-dev fontconfig-dev freetype-dev \
                 curl-dev openssl-dev libepoxy-dev mesa-dev mesa-gles mesa-egl mesa-gbm mesa-dri-gallium \
                 webkit2gtk-4.1-dev gstreamer-dev gst-plugins-base-dev \
@@ -474,9 +538,31 @@ jobs:
               # core + Linux port (musl JDK8); the default cc on Alpine already links musl.
               cd /cn1/maven && mvn -B -pl linux -am -DskipTests -Dmaven.javadoc.skip=true -Plocal-dev-javase install
               cd /cn1/vm && mvn -B clean package -pl JavaAPI -am -DskipTests
+              ulimit -c unlimited || true
+              rc=0
               mvn -B test -pl tests -am \
                 -Dtest=CleanTargetLinuxIntegrationTest#capturesHelloSuiteOverWebSocketLinux \
-                -Dsurefire.failIfNoSpecifiedTests=false
+                -Dsurefire.failIfNoSpecifiedTests=false || rc=$?
+              # Keep the UNSTRIPPED binary next to any core: without it a core names
+              # addresses and nothing else, and the container is gone by the time the
+              # artifact is looked at.
+              out=/cn1/artifacts/linux-port/raw-musl
+              mkdir -p "$out"
+              elf="$(find /tmp -maxdepth 4 -name LinuxHelloMain -type f 2>/dev/null | head -1)"
+              if [ -n "$elf" ]; then cp "$elf" "$out/LinuxHelloMain" || true; fi
+              for core in "$out"/core.*; do
+                [ -f "$core" ] || continue
+                {
+                  echo "=== post-mortem of $core (elf=$elf) ==="
+                  gdb "$elf" "$core" -batch -ex "set pagination off" \
+                    -ex "thread apply all bt" -ex "thread 1" -ex "bt full" 2>&1
+                } >> "$out/crash-stacks.txt" || true
+              done
+              if [ ! -f "$out/crash-stacks.txt" ]; then
+                echo "no core file was written -- the suite stopped without a fatal signal" \
+                  > "$out/crash-stacks.txt"
+              fi
+              exit $rc
             '
       - name: Upload musl screenshots
         if: always()

.github/workflows/parparvm-selfhost.yml

@@ -0,0 +1,119 @@
+name: ParparVM Self-Hosting
+
+# Translates the ByteCodeTranslator with itself and compares the result against
+# the same translation run on a JVM.
+#
+# What this buys that the existing suites do not: ByteCodeTranslator is a 37.6k
+# line real program that hammers collections, strings, exceptions, file I/O and
+# the GC at a scale no unit test reaches, and the emitted C is a byte-exact
+# expected value that costs nothing to maintain -- it is whatever the JVM
+# produced from the same inputs. A VM defect that changes behaviour rather than
+# crashing (a wrong hash order, a dropped write barrier, a mis-mangled symbol)
+# shows up as a diff instead of passing silently.
+#
+# Gates, cheapest first:
+#   D  native vs native, two fresh processes, same input. If the native side is
+#      not self-consistent nothing else means anything, so it runs first.
+#   A  JVM vs native over the same corpus. The headline.
+#   Negative control: after a green comparison one emitted byte is flipped and
+#      the comparator MUST report exactly that file. A comparator nobody has
+#      watched fail is not a comparator.
+#
+# Not on the PR leg by default: a full run builds the translator twice and
+# translates a large corpus several times. It runs nightly, on demand, and on a
+# PR that opts in with the `selfhost` label.
+
+on:
+  schedule:
+    # 04:20 UTC daily, off the hour to avoid the runner rush.
+    - cron: '20 4 * * *'
+  workflow_dispatch:
+  pull_request:
+    types: [ opened, synchronize, reopened, labeled ]
+    paths:
+      - 'vm/**'
+      - '.github/workflows/parparvm-selfhost.yml'
+      - '!vm/**/README.md'
+      - '!vm/**/docs/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  CN1_NATIVE_VERIFY: strict
+
+jobs:
+  selfhost:
+    # On a pull_request only when the author asked for it; the schedule and
+    # workflow_dispatch legs always run.
+    if: >-
+      github.event_name != 'pull_request' ||
+      contains(github.event.pull_request.labels.*.name, 'selfhost')
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Install native build tools
+        run: |
+          bash scripts/ci/apt-get-update.sh
+          sudo apt-get install -y clang
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v5
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
+      - name: Save JDK 8 path
+        run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV
+
+      # The translator has to exist as classes before it can translate itself.
+      - name: Build the translator
+        run: scripts/ci/retry.sh mvn -q -B -pl ByteCodeTranslator -am package -DskipTests
+        working-directory: vm
+
+      - name: Resolve the ASM classpath
+        run: >-
+          scripts/ci/retry.sh mvn -q -B -pl ByteCodeTranslator
+          dependency:build-classpath
+          -Dmdep.outputFile=target/selfhost-asm-classpath.txt
+        working-directory: vm
+
+      # -O1: the diff gates care about the EMITTED C, not about how well clang
+      # optimised the binary that emitted it, and -O1 links several times faster.
+      # Mark threads are set explicitly rather than left to the source default,
+      # which resolves to a single marker and makes a large corpus take hours.
+      - name: Build the self-hosted translator
+        run: vm/selfhost/build-selfhost.sh
+        env:
+          CN1_SELFHOST_CFLAGS: -DCN1_GC_MARK_THREADS=4
+
+      # The corpus is the translator's OWN classes plus ASM. verify-selfhost.sh
+      # prepends vm/selfhost/target/javaapi-classes itself, so it is not repeated
+      # here. Absolute paths: the script runs both sides under `env -i` into one
+      # fixed output directory, so a relative path would not survive.
+      - name: Gate D and Gate A, with the negative control
+        run: |
+          vm/selfhost/verify-selfhost.sh \
+            "$PWD/vm/selfhost/target/asm-classes;$PWD/vm/selfhost/target/classes" \
+            com_codename1_tools_translator_ByteCodeTranslator \
+            com.codename1.tools.translator
+
+      # Both trees, so a divergence can be inspected rather than guessed at from
+      # a one-line summary.
+      - name: Upload the compared trees on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: selfhost-trees
+          path: |
+            vm/selfhost/target/verify/jvm-tree
+            vm/selfhost/target/verify/parpar1-tree
+            vm/selfhost/target/verify/parpar2-tree
+            vm/selfhost/target/verify/*.txt
+            vm/selfhost/target/verify/*.log
+          retention-days: 7
+          if-no-files-found: ignore

scripts/check-native-signatures.sh

@@ -36,7 +36,7 @@ for arg in "$@"; do
   esac
 done
 
-if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifier.class" ]]; then
+if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifierCli.class" ]]; then
   echo "check-native-signatures: building the translator" >&2
   (cd "$REPO_ROOT/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests)
 fi
@@ -95,7 +95,7 @@ for entry in "${PORTS[@]}"; do
 
   echo "== $name"
   if ! java -cp "$TRANSLATOR:$(cat "$ASM_CP_FILE")" \
-       com.codename1.tools.translator.NativeSignatureVerifier "${args[@]}"; then
+       com.codename1.tools.translator.NativeSignatureVerifierCli "${args[@]}"; then
     status=1
   fi
   checked=$((checked + 1))

scripts/copyright-header-exclusions.txt

@@ -33,3 +33,4 @@ vm/JavaAPI/src/java/util/Collections.java | Apache Harmony source retaining its
 vm/JavaAPI/src/java/util/HashMap.java | Apache Harmony source retaining its original Apache-2.0 notice
 vm/JavaAPI/src/java/util/Hashtable.java | Apache Harmony source retaining its original Apache-2.0 notice
 vm/JavaAPI/src/java/util/IdentityHashMap.java | Apache Harmony source retaining its original Apache-2.0 notice
+vm/JavaAPI/src/java/util/ArrayList.java | Apache Harmony source retaining its original Apache-2.0 notice

vm/ByteCodeTranslator/src/cn1_globals.h

@@ -2218,15 +2218,15 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int
 // because bibopCurrent[] is shared across all classes of the same size class).
 #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP)
 #define CN1_FAST_NEW(X) ({ \
-    if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \
+    if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \
     JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \
     if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \
     __cn1fo; })
 // No-body-zero variant (init-before-publish). The slow-path fallback __NEW_X
 // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full
 // path.
 #define CN1_FAST_NEW_NOZERO(X) ({ \
-    if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \
+    if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \
     JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \
     if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \
     __cn1fo; })
@@ -2860,6 +2860,34 @@ extern struct clazz class_array1__JAVA_DOUBLE;
 extern struct clazz class_array2__JAVA_DOUBLE;
 extern struct clazz class_array3__JAVA_DOUBLE;
 
+/**
+ * The nine scalar primitive class objects -- int.class, Integer.TYPE and friends.
+ *
+ * javac lowers a primitive class literal to a read of the boxed type's own TYPE
+ * field, so `TYPE = int.class` inside Integer's initializer compiles to
+ * `getstatic TYPE; putstatic TYPE` -- it reads the field it is initializing and
+ * leaves it null. Every wrapper that declared TYPE that way had a null one, and
+ * a Map keyed on them collapsed to a single entry, so a lookup for int answered
+ * with whatever type was stored last. Nothing threw. The wrappers now go through
+ * java_lang_Class_getPrimitiveClass, which hands back one of these.
+ *
+ * classId is CN1_PRIMITIVE_CLASS_ID for all nine: these never take part in an
+ * instanceof, and instanceofFunction indexes tables by classId, so the callers
+ * that could reach one (isAssignableFrom, isInstance) test primitiveType first
+ * rather than indexing with a value no table has a row for.
+ */
+#define CN1_PRIMITIVE_CLASS_ID (-1)
+
+extern struct clazz cn1_primitive_class_int;
+extern struct clazz cn1_primitive_class_long;
+extern struct clazz cn1_primitive_class_short;
+extern struct clazz cn1_primitive_class_byte;
+extern struct clazz cn1_primitive_class_char;
+extern struct clazz cn1_primitive_class_float;
+extern struct clazz cn1_primitive_class_double;
+extern struct clazz cn1_primitive_class_boolean;
+extern struct clazz cn1_primitive_class_void;
+
 extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]);
 /**
  * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of