diff --git a/.github/workflows/linux-build-run.yml b/.github/workflows/linux-build-run.yml index 37c133253fb..92fef870db8 100644 --- a/.github/workflows/linux-build-run.yml +++ b/.github/workflows/linux-build-run.yml @@ -283,7 +283,13 @@ jobs: # hang-stacks.txt with nothing but sample headers -- which is how four # occurrences of the suite stall ended up with no evidence at all. Still # best-effort: a runner without gdb must not fail the suite, it must say so. - bash scripts/ci/apt-get-install.sh gdb || echo "WARNING: gdb install failed" + # ABSOLUTE path. This step runs under `working-directory: vm`, so the + # relative form resolved to vm/scripts/ci/apt-get-install.sh, bash answered + # "No such file or directory", and the `|| echo` swallowed it -- gdb has + # NEVER been installed here. Every post-mortem in this job therefore + # produced nothing, including the SIGSEGV this run just hit, which uploads + # a core nobody can read. Same shape as the retry.sh path fixed elsewhere. + bash "$GITHUB_WORKSPACE/scripts/ci/apt-get-install.sh" gdb || echo "WARNING: gdb install failed" if command -v gdb >/dev/null 2>&1; then echo "gdb available: $(gdb --version | head -1)" else diff --git a/.github/workflows/parparvm-parallel-mark.yml b/.github/workflows/parparvm-parallel-mark.yml index 9bd31930d44..119f1c3b63f 100644 --- a/.github/workflows/parparvm-parallel-mark.yml +++ b/.github/workflows/parparvm-parallel-mark.yml @@ -43,8 +43,25 @@ # four-CPU hypervisor guest is a weak generator of store interleavings. These # runners are native arm64 hardware, which is where the corruption was seen. # -# NOTHING HERE CHANGES A DEFAULT. The pool is switched on for this workflow only, -# through CN1_TEST_EXTRA_CFLAGS; every other build still compiles it out. +# THIS WORKFLOW SETTLED IT, AND THE DEFAULT HAS NOW MOVED. It used to read "nothing +# here changes a default -- the pool is switched on for this workflow only"; that is no +# longer true, and leaving it saying so would repeat the exact failure described above, +# where a comment outlived the decision it recorded. +# +# gcMarkResolveThreadCount now derives the marker count from the CPU count (capped at 4) +# on POSIX. The evidence is this workflow green at 1 and 4 markers on native arm64 Linux +# and on x64, plus a self-hosting translation measuring ~5x wall clock and ~10% peak +# memory against the serial default -- both moving together, because a shorter cycle +# gives the mutator less time to run ahead. +# +# WINDOWS IS DELIBERATELY EXCLUDED and still defaults to one marker. Every arm here runs +# on POSIX threads; Windows goes through the Win32 pthread shim, which the parallel +# marker had never executed on. Enabling it there turned +# `ParparVM Java Tests (Windows)` / screenshot-capture (arm64) red -- the translated app +# emitted 132 of 166 screenshots and stopped part-way through the suite -- so that shim +# is unvalidated for this, and re-enabling it needs THAT job green rather than a +# measurement from another platform. -DCN1_GC_MARK_THREADS=N turns it on for whoever +# picks up the shim. name: ParparVM parallel mark (arm64 Linux) on: diff --git a/.github/workflows/parparvm-selfhost.yml b/.github/workflows/parparvm-selfhost.yml new file mode 100644 index 00000000000..7ba516ab3c5 --- /dev/null +++ b/.github/workflows/parparvm-selfhost.yml @@ -0,0 +1,121 @@ +name: ParparVM Self-Hosting + +# Translates the ByteCodeTranslator with itself and compares the result against +# the same translation run on a JVM. +# +# What this buys that the existing suites do not: ByteCodeTranslator is a 37.6k +# line real program that hammers collections, strings, exceptions, file I/O and +# the GC at a scale no unit test reaches, and the emitted C is a byte-exact +# expected value that costs nothing to maintain -- it is whatever the JVM +# produced from the same inputs. A VM defect that changes behaviour rather than +# crashing (a wrong hash order, a dropped write barrier, a mis-mangled symbol) +# shows up as a diff instead of passing silently. +# +# Gates, cheapest first: +# D native vs native, two fresh processes, same input. If the native side is +# not self-consistent nothing else means anything, so it runs first. +# A JVM vs native over the same corpus. The headline. +# Negative control: after a green comparison one emitted byte is flipped and +# the comparator MUST report exactly that file. A comparator nobody has +# watched fail is not a comparator. +# +# Not on the PR leg by default: a full run builds the translator twice and +# translates a large corpus several times. It runs nightly, on demand, and on a +# PR that opts in with the `selfhost` label. + +on: + schedule: + # 04:20 UTC daily, off the hour to avoid the runner rush. + - cron: '20 4 * * *' + workflow_dispatch: + pull_request: + types: [ opened, synchronize, reopened, labeled ] + paths: + - 'vm/**' + - '.github/workflows/parparvm-selfhost.yml' + - '!vm/**/README.md' + - '!vm/**/docs/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + CN1_NATIVE_VERIFY: strict + +jobs: + selfhost: + # On a pull_request only when the author asked for it; the schedule and + # workflow_dispatch legs always run. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'selfhost') + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Check out repository + uses: actions/checkout@v6 + + - name: Install native build tools + run: | + bash scripts/ci/apt-get-update.sh + sudo apt-get install -y clang + + - name: Set up JDK 8 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '8' + cache: 'maven' + - name: Save JDK 8 path + run: echo "JDK_8_HOME=$JAVA_HOME" >> $GITHUB_ENV + + # The translator has to exist as classes before it can translate itself. + - name: Build the translator + run: >- + "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B + -pl ByteCodeTranslator -am package -DskipTests + working-directory: vm + + - name: Resolve the ASM classpath + run: >- + "$GITHUB_WORKSPACE/scripts/ci/retry.sh" mvn -q -B -pl ByteCodeTranslator + dependency:build-classpath + -Dmdep.outputFile=target/selfhost-asm-classpath.txt + working-directory: vm + + # -O1: the diff gates care about the EMITTED C, not about how well clang + # optimised the binary that emitted it, and -O1 links several times faster. + # Mark threads are set explicitly rather than left to the source default, + # which resolves to a single marker and makes a large corpus take hours. + - name: Build the self-hosted translator + run: vm/selfhost/build-selfhost.sh + env: + CN1_SELFHOST_CFLAGS: -DCN1_GC_MARK_THREADS=4 + + # The corpus is the translator's OWN classes plus ASM. verify-selfhost.sh + # prepends vm/selfhost/target/javaapi-classes itself, so it is not repeated + # here. Absolute paths: the script runs both sides under `env -i` into one + # fixed output directory, so a relative path would not survive. + - name: Gate D and Gate A, with the negative control + run: | + vm/selfhost/verify-selfhost.sh \ + "$PWD/vm/selfhost/target/asm-classes;$PWD/vm/selfhost/target/classes" \ + com_codename1_tools_translator_ByteCodeTranslator \ + com.codename1.tools.translator + + # Both trees, so a divergence can be inspected rather than guessed at from + # a one-line summary. + - name: Upload the compared trees on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: selfhost-trees + path: | + vm/selfhost/target/verify/jvm-tree + vm/selfhost/target/verify/parpar1-tree + vm/selfhost/target/verify/parpar2-tree + vm/selfhost/target/verify/*.txt + vm/selfhost/target/verify/*.log + retention-days: 7 + if-no-files-found: ignore diff --git a/scripts/check-native-signatures.sh b/scripts/check-native-signatures.sh index cb23bac032c..d61eea22bfa 100755 --- a/scripts/check-native-signatures.sh +++ b/scripts/check-native-signatures.sh @@ -36,7 +36,7 @@ for arg in "$@"; do esac done -if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifier.class" ]]; then +if [[ ! -f "$TRANSLATOR/com/codename1/tools/translator/NativeSignatureVerifierCli.class" ]]; then echo "check-native-signatures: building the translator" >&2 (cd "$REPO_ROOT/vm" && mvn -q -B -pl ByteCodeTranslator -am package -DskipTests) fi @@ -135,7 +135,7 @@ for entry in "${PORTS[@]}"; do echo "== $name" if ! java -cp "$TRANSLATOR:$(cat "$ASM_CP_FILE")" \ - com.codename1.tools.translator.NativeSignatureVerifier "${args[@]}"; then + com.codename1.tools.translator.NativeSignatureVerifierCli "${args[@]}"; then status=1 fi checked=$((checked + 1)) diff --git a/scripts/copyright-header-exclusions.txt b/scripts/copyright-header-exclusions.txt index 5c9b569b096..d3a3e6ee084 100644 --- a/scripts/copyright-header-exclusions.txt +++ b/scripts/copyright-header-exclusions.txt @@ -33,3 +33,4 @@ vm/JavaAPI/src/java/util/Collections.java | Apache Harmony source retaining its vm/JavaAPI/src/java/util/HashMap.java | Apache Harmony source retaining its original Apache-2.0 notice vm/JavaAPI/src/java/util/Hashtable.java | Apache Harmony source retaining its original Apache-2.0 notice vm/JavaAPI/src/java/util/IdentityHashMap.java | Apache Harmony source retaining its original Apache-2.0 notice +vm/JavaAPI/src/java/util/ArrayList.java | Apache Harmony source retaining its original Apache-2.0 notice diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index 3ff26d62df7..42b2119f011 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -1383,6 +1383,23 @@ static inline JAVA_BOOLEAN cn1InNursery(void* p) { } return (char*)p >= cn1NurseryArenaStart && (char*)p < cn1NurseryArenaEnd; } +// "Is this object STILL IN the young generation?" -- which is NOT the same question as +// cn1InNursery(), and conflating the two is a use-after-free. +// +// Promotion does not MOVE anything: cn1NurseryPromote flips __heapPosition from -1 to -2 +// and registers the object with the global collector, but the bytes stay at the same +// address inside the arena. So cn1InNursery() -- a pure address-range test -- keeps +// answering TRUE for an object that has belonged to the old generation for a long time. +// +// Every barrier that asks "is the TARGET young?" must therefore ask this instead. Asking +// cn1InNursery() alone makes a store into a promoted container look like a young->young +// store, which needs no promotion, so the VALUE is never promoted -- and the value's +// block is then recycled while the promoted container still points at it. Observed as a +// fault inside ASM's MethodNode.getLabelNodes reading an Object[] element whose class +// pointer had been poisoned by the retiring collection. +static inline JAVA_BOOLEAN cn1IsYoungObject(void* p) { + return (JAVA_BOOLEAN)(cn1InNursery(p) && ((JAVA_OBJECT)p)->__heapPosition == -1); +} // Emitted by the translator before an object-reference store into a heap location. // Fast path is INLINE: only a value that actually lives in the nursery can escape, so // the overwhelmingly common heap->heap / null store collapses to a two-compare range @@ -1392,10 +1409,31 @@ static inline JAVA_BOOLEAN cn1InNursery(void* p) { // Tagged immediates are excluded by cn1InNursery itself -- see the note there; this used // to carry its own CN1_IS_TAGGED test, which fixed the barrier and left the five other // call sites that dereference straight after it still exposed. +// +// IT ALSO HAS TO CARRY THE SATB INSERTION HALF. This macro is the only barrier the +// translator emits at an object store, so in the no-nursery build it doubles as SATB +// insertion (see the #else). Defining CN1_NURSERY used to REPLACE that rather than +// compose with it, which silently dropped insertion from the concurrent collector -- +// the half whose absence lets a fresh container that takes an older child mid-mark have +// that child recorded nowhere, so the sweep reclaims it while the container still points +// at it. cn1_globals.m's deletion-filter argument names this build as its one exception, +// and the exception only existed because nothing compiled it. The two halves are +// independent (nursery escape-promotion vs. mark liveness) and both must run. +// ORDER MATTERS AND THE YOUNG TEST COMES SECOND. The nursery barrier runs first and +// PROMOTES an escaping value, so by the time the SATB test runs the value is young only +// if it is staying in the young generation -- a young->young store. Those must NOT enter +// the SATB log: the log is drained by the collector, which would then mark and trace an +// object whose lifetime belongs to the minor collector, and a minor collection is free to +// reclaim it in between. That is a use-after-free with the collector holding the stale +// reference, which is the worst possible owner of one. #define CN1_WRITE_BARRIER(target, value) \ do { JAVA_OBJECT cn1__bv = (JAVA_OBJECT)(value); \ - if(cn1__bv != JAVA_NULL && cn1InNursery(cn1__bv)) { \ - cn1NurseryWriteBarrier((JAVA_OBJECT)(target), cn1__bv); } } while(0) + if(cn1__bv != JAVA_NULL) { \ + if(cn1InNursery(cn1__bv)) { \ + cn1NurseryWriteBarrier((JAVA_OBJECT)(target), cn1__bv); } \ + if(__builtin_expect(gcSatbActive, 0) && !CN1_IS_TAGGED(cn1__bv) \ + && !cn1IsYoungObject(cn1__bv)) { \ + cn1SatbEnqueue(cn1__bv); } } } while(0) #else // No nursery: repurpose the (already-emitted-at-every-object-store) write barrier as the // SATB INSERTION half. During the mark, enqueue the NEW reference being stored so an @@ -1452,12 +1490,24 @@ extern void cn1SatbBulkQuiesce(void); // barrier compiles out entirely, so there is zero footprint on the store hot path. #define CN1_SATB_DELETE(fieldAddr) do { } while(0) #else +#ifdef CN1_NURSERY +// The young exclusion applies to the DELETION half for the same reason as the insertion +// half: the overwritten value can be a young object, and handing one to the collector's +// log lets it trace memory the minor collector owns and may already have reclaimed. +#define CN1_SATB_DELETE(fieldAddr) \ + do { if(__builtin_expect(gcSatbActive, 0)) { \ + JAVA_OBJECT cn1__old = *(JAVA_OBJECT volatile*)(fieldAddr); \ + if(cn1__old != JAVA_NULL && !CN1_IS_TAGGED(cn1__old) \ + && !cn1IsYoungObject(cn1__old)) cn1SatbEnqueue(cn1__old); \ + } } while(0) +#else #define CN1_SATB_DELETE(fieldAddr) \ do { if(__builtin_expect(gcSatbActive, 0)) { \ JAVA_OBJECT cn1__old = *(JAVA_OBJECT volatile*)(fieldAddr); \ if(cn1__old != JAVA_NULL && !CN1_IS_TAGGED(cn1__old)) cn1SatbEnqueue(cn1__old); \ } } while(0) #endif +#endif // ---- java.lang.ref support ------------------------------------------------- // A WeakReference's referent is NOT traced by the generated mark function. That @@ -1546,6 +1596,13 @@ struct ThreadLocalData { // gcMarkObject at the same time a mutator promotes, and a shared flag would make // the GC thread promote-instead-of-mark and corrupt the heap. JAVA_BOOLEAN nurseryPromoting; +#ifdef CN1_NURSERY_VERIFY + // QA: while set, gcMarkObject REPORTS young referents instead of promoting them, so + // the generational invariant can be checked rather than assumed. Holder is whatever + // object's mark function is currently running, which is what names the guilty field. + JAVA_BOOLEAN nurseryVerifying; + JAVA_OBJECT nurseryVerifyHolder; +#endif JAVA_OBJECT* nurseryPromoteWorklist; int nurseryPromoteTop; int nurseryPromoteCap; @@ -2218,7 +2275,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // because bibopCurrent[] is shared across all classes of the same size class). #if !defined(CN1_DISABLE_INLINE_ALLOC) && !defined(CN1_DISABLE_BIBOP) #define CN1_FAST_NEW(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAlloc(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2226,7 +2283,7 @@ static inline JAVA_OBJECT cn1BibopFastAllocNoZero(CODENAME_ONE_THREAD_STATE, int // still fully zeroes (calloc) -- correct, just un-elided on the rare page-full // path. #define CN1_FAST_NEW_NOZERO(X) ({ \ - if(__builtin_expect(!class__##X.initialized, 0)) __STATIC_INITIALIZER_##X(threadStateData); \ + if(__builtin_expect(!__atomic_load_n(&class__##X.initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_##X(threadStateData); \ JAVA_OBJECT __cn1fo = cn1BibopFastAllocNoZero(threadStateData, sizeof(struct obj__##X), &class__##X, CN1_BIBOP_CIDX(sizeof(struct obj__##X))); \ if(__builtin_expect(__cn1fo == (JAVA_OBJECT)0, 0)) __cn1fo = __NEW_##X(threadStateData); \ __cn1fo; }) @@ -2860,6 +2917,14 @@ extern struct clazz class_array1__JAVA_DOUBLE; extern struct clazz class_array2__JAVA_DOUBLE; extern struct clazz class_array3__JAVA_DOUBLE; +#ifdef CN1_GC_VERIFY +extern void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, + int declaredClassId, const char* fieldName); +#endif +#define CN1_GC_CYCLE_IDLE 0 +#define CN1_GC_CYCLE_RUNNING 1 +#define CN1_GC_CYCLE_FROZEN 2 +extern _Atomic int cn1GcCycleState; extern JAVA_OBJECT newString(CODENAME_ONE_THREAD_STATE, int length, JAVA_CHAR data[]); /** * Like newStringFromCString but DECODES, in the PLATFORM's encoding, instead of diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index f16cb6d1c31..b86f855bfa7 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -78,6 +78,14 @@ #define NSLog(...) printf(__VA_ARGS__); printf("\n") #endif +// UNCONDITIONAL on Apple, and ABOVE the CN1_GC_VERIFY block on purpose. The only other +// copy of this include sits INSIDE that block, so every user of malloc_size / +// malloc_zone_statistics / malloc_zone_print outside a verifier build compiled as an +// implicit declaration -- which clang has rejected outright since C99 became the +// default. Including it twice is legal and costs nothing. +#if defined(__APPLE__) +#include +#endif #ifdef CN1_GC_VERIFY // QA-only heap verifier (see the block next to cn1ConservativeResolve). It // classifies references against the same page/extent index the conservative @@ -647,6 +655,43 @@ static void cn1ReportPacingParks(void) { // objects are no longer fresh, still do not resolve, and age into the sweep's // m < V - 1 reclamation while a live field still points at them. static JAVA_BOOLEAN cn1GcPageIndexStale = JAVA_FALSE; +// THE SWEEP'S LIVENESS RULE, IN ONE PLACE. +// +// An object survives if it is fresh (-1, the one-cycle grace for anything allocated +// during or after the mark) or if its mark is within CN1_GC_AGING_SLACK epochs of the +// current one. Historically this was spelled out at five separate sites -- the legacy +// table scan, the BiBOP per-slot walk, the two reference-clearing passes and the +// weak-child check -- with a comment at each insisting they must agree exactly, because +// clearing a reference the sweep then keeps only wastes a cache entry while FAILING to +// clear one the sweep frees hands out a dangling pointer. Five copies of a rule that +// must not diverge is a defect waiting for its next edit; this is that rule. +// +// THE SLACK IS THE LARGEST SINGLE MEMORY KNOB IN THE COLLECTOR. Measured on the +// self-hosting corpus, 30% of occupied bytes (2,319,546 objects, ~210MB of a 705MB Java +// heap) sit in the "aging" bucket -- marked last cycle, unreachable this cycle, and held +// for one more cycle by this rule alone. The churn classes are the ones paying for it: +// Object[] 41% aging, char[] 30%, boolean[] 82%, asm Subroutine 82%. +// +// It defaults to 1, which is the historical behaviour, and is NOT lowered here. The +// slack is what covers a mark that was incomplete rather than a heap that was empty -- +// a page-index miss, a conservative-scan gap -- and on ParparVM a dangling read is a +// native crash no Java catch can see. Lowering it is a real memory win and a real risk, +// so it is exposed as a measurable knob for the GC gates to rule on rather than flipped +// on the strength of a good-looking census. +static int cn1GcAgingSlack(void) { + static int cached = -1; + if(cached < 0) { + const char* v = getenv("CN1_GC_AGING_SLACK"); + int n = v != 0 ? atoi(v) : 1; + cached = (n >= 0 && n <= 8) ? n : 1; + } + return cached; +} +// TRUE when the sweep would reclaim this mark value. Kept as one expression so every +// caller asks the identical question. +static inline int cn1GcSweepReclaims(int mark) { + return mark != -1 && mark < currentGcMarkValue - cn1GcAgingSlack(); +} // Page-heap bytes allocated across the whole run, charged cycle by cycle. Divided by // the cycle count it says how far the mutator ran ahead of the collector, which is what // "the collector is keeping up" means as a number: a healthy run allocates about one @@ -1733,6 +1778,14 @@ static void cn1DrainDeadThreadPending() { static void cn1GcBuildVirtualThreadSnapshot(void); static void cn1GcScanParkedVirtualThreads(CODENAME_ONE_THREAD_STATE); static int cn1GcParkedVirtualThreadsScanned; +#ifdef CN1_NURSERY +// The young generation is a ROOT SOURCE for the major mark (see +// cn1NurseryMarkYoungRoots). It is scanned ONCE PER PAUSED THREAD, over that thread's +// OWN young blocks -- not once per cycle over all of them. A nursery is thread-local +// and only its owner mutates it, so the owner being paused is exactly what makes the +// walk safe; see the call site. +void cn1NurseryMarkYoungRoots(CODENAME_ONE_THREAD_STATE, struct ThreadLocalData* owner); +#endif static void cn1GcSignalStopThreads(struct ThreadLocalData* self); static void cn1GcSignalReleaseThreads(struct ThreadLocalData* self); #ifdef CN1_GC_CAN_FORCE_STOP @@ -1814,7 +1867,43 @@ static void cn1DrainDeadThreadPending() { // interleaved drain there). The size macro is hoisted with them for the same reason; // its rationale stays at the worklist definition. #ifndef CN1_GC_MARK_WORKLIST_SIZE -#define CN1_GC_MARK_WORKLIST_SIZE 65536 +// 1M ENTRIES (8MB), NOT 64K. When this overflows, the collector drops into a serial +// page-rescan fixpoint over the whole BiBOP registry, and on a real workload that rescan +// does an enormous amount of work for nothing. Measured on the self-hosting corpus at +// 64K entries, in ONE run: +// +// rescanPasses=68 rescanUseful=0 rescanSlots=10,107,804 +// +// Sixty-eight passes, ten million slots walked, and not one of them found an object the +// drain had missed. The rescan is a correctness backstop for a worklist that cannot hold +// the frontier; the fix is to hold the frontier. +// +// worklist wall (3 reps) peak __bss +// 65536 0.97 / 1.01 / 1.24s 1050-1265MB 1.26MB +// 262144 1.24 / 1.28s 760-786MB 4.41MB +// 1048576 0.90 / 0.90 / 0.90s 791-839MB 16.99MB +// 4194304 0.88 / 0.90 / 0.90s 777-831MB ~67MB +// +// ~25% off peak memory AND a tighter wall clock -- the same move-together behaviour +// parallel marking showed, for the same reason: work the collector does not waste is +// cycle time the mutator does not spend running ahead. +// +// 262144 IS THE DEFAULT BECAUSE IT IS THE KNEE. Re-measured head to head it is as good +// as 1M on both axes (760-786MB against 791-858MB) for a QUARTER of the table. The +// array is static, so its size is a reservation every generated application carries -- +// iOS and Android included, not just a desktop translation -- and 17MB of that for no +// measured gain is not a trade worth making on a phone. +// +// The reservation is zerofill, so it costs no on-disk bytes (all three binaries are +// byte-identical in size at 4,529,096) and no resident memory until an entry is +// actually touched. That is why a size this large is affordable at all; it is not a +// reason to be careless with it, because the pages a big heap DOES touch are real. +// +// PRIOR ROUNDS FOUND THE OPPOSITE ("a bigger worklist is neutral-to-worse everywhere it +// has been tried") and they were measured under SERIAL marking, where the collector was +// the bottleneck for a different reason. Like the marker-count and aging-slack results, +// this one is only true for the configuration it was measured in. +#define CN1_GC_MARK_WORKLIST_SIZE 262144 #endif static JAVA_BOOLEAN gcMarkWorklistOverflow; static int gcMarkWorklistTop; @@ -1824,6 +1913,15 @@ static void cn1DrainDeadThreadPending() { // walks the page registry and its slots before their definitions. static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i); static CN1BibopPage* _Atomic bibopAllPages; +#ifdef CN1_ALLOC_CENSUS +// Defined far below, beside the BiBOP page structures they read. Declared up here +// because the post-sweep hook that calls them is compiled earlier -- and OUTSIDE the +// CN1_GC_VERIFY block just above, which is off in an ordinary census build. +void cn1HeapAccounting(const char* label); +void cn1AllocCensus(const char* label); +void cn1LiveCensus(const char* label); +#endif + #ifdef CN1_GRACE_AUDIT static void cn1GraceAuditPreSweep(CODENAME_ONE_THREAD_STATE); #endif @@ -3209,7 +3307,7 @@ static JAVA_BOOLEAN cn1GcProcessReferences(CODENAME_ONE_THREAD_STATE) { #endif { int mark = __atomic_load_n(&r->__codenameOneGcMark, __ATOMIC_ACQUIRE); - if(mark == -1 || mark >= currentGcMarkValue - 1) { + if(!cn1GcSweepReclaims(mark)) { continue; } } @@ -3333,7 +3431,7 @@ static JAVA_BOOLEAN cn1GcProcessReferences(CODENAME_ONE_THREAD_STATE) { // currentGcMarkValue - 1`; -1 is the one-cycle grace and currentGcMarkValue - 1 // is last cycle's slack, and both mean the object survives. int mark = __atomic_load_n(&r->__codenameOneGcMark, __ATOMIC_ACQUIRE); - if(mark == -1 || mark >= currentGcMarkValue - 1) { + if(!cn1GcSweepReclaims(mark)) { continue; } #ifdef CN1_GC_VERIFY @@ -3974,6 +4072,27 @@ void codenameOneGCMark() { cn1GcParkedVirtualThreadsScanned = 1; cn1GcScanParkedVirtualThreads(d); } +#ifdef CN1_NURSERY + // The young generation, as a root source. Without this the collector + // cannot see a live nursery object at all and frees the heap objects it + // references; see cn1NurseryMarkYoungRoots for why the promotion barrier + // does not already cover this direction. + // + // SCANS ONLY `t`, THE THREAD THIS ITERATION HAS PAUSED. An earlier version + // ran once per cycle over EVERY thread's young blocks and claimed the + // stopped-thread region made that safe. It does not: this loop pauses + // threads ONE AT A TIME and releases each before the next is scanned, so a + // global walk here races every other mutator -- which can bump-allocate, + // run its own minor collection, clear start bits and recycle blocks while + // the walk reads them. That is a missed root or a dereference of a + // recycled header, and the comment asserting otherwise was the worse half + // of the bug. + // + // A nursery is thread-local and only its owner touches it, so scanning + // each thread's own blocks while that thread is paused is both safe and + // complete: every thread is paused in some iteration. + cn1NurseryMarkYoungRoots(d, t); +#endif #ifdef CN1_CONSERVATIVE_GC_SELFCHECK cn1GcSelfCheckThreadStack(t, stackSize); #endif @@ -4833,6 +4952,14 @@ static void cn1GcReportStaleIndexSkip(void) { void codenameOneGCSweep() { struct ThreadLocalData* threadStateData = getThreadLocalData(); +#ifdef CN1_ALLOC_CENSUS + // BEFORE the sweep on purpose. This is the only point where the four slot + // states are still distinguishable -- the sweep stamps every fresh object with + // the current mark, after which "traced" and "kept by grace" look identical. + if(getenv("CN1_HEAP_REPORT")) { + cn1LiveCensus("pre-sweep"); + } +#endif // THE MARK THIS SWEEP WOULD ACT ON MAY BE INCOMPLETE. cn1GcPageIndexStale says the // page index could not be rebuilt, so every reference into a page registered since // the last successful rebuild failed to resolve and its object was never marked -- @@ -4891,7 +5018,7 @@ void codenameOneGCSweep() { JAVA_OBJECT o = allObjectsInHeap[iter]; if(o != JAVA_NULL) { if(o->__codenameOneGcMark != -1) { - if(o->__codenameOneGcMark < currentGcMarkValue - 1) { + if(cn1GcSweepReclaims(o->__codenameOneGcMark)) { if (o->__codenameOneGcMark <= 0) { #if defined(__APPLE__) && defined(__OBJC__) #if TARGET_OS_SIMULATOR @@ -5005,6 +5132,15 @@ void codenameOneGCSweep() { // permanently broken. cn1GcVerifyHeap(threadStateData); #endif +#ifdef CN1_ALLOC_CENSUS + // Same reasoning as the verify hook above: post-sweep is when "live" means + // live. cn1HeapAccounting and cn1AllocCensus were written but never called + // from anywhere, so nothing could answer "what is the footprint made of". + if(getenv("CN1_HEAP_REPORT")) { + cn1HeapAccounting("post-sweep"); + cn1LiveCensus("post-sweep"); + } +#endif } JAVA_BOOLEAN removeObjectFromHeapCollection(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { @@ -5685,6 +5821,10 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE // Non-static: the inlined bump fast path (cn1_globals.h) reads bibopCurrent[ci]. __thread CN1BibopPage* bibopCurrent[CN1_BIBOP_NUM_CLASSES]; +#ifdef CN1_ALLOC_CENSUS +static void cn1BibopExitReport(void); +#endif + static void cn1BibopDoInit() { int ci = 0; // DIAGNOSTIC KNOB -- CN1_GC_TRIGGER_MB overrides how many uncollected bytes @@ -5716,8 +5856,108 @@ static void cn1BibopDoInit() { atomic_store_explicit(&bibopBypassGeneration[i], 0, memory_order_relaxed); bibopHighSurvivalStreak[i] = 0; } + // Prime the free-memory snapshot the pacing cap is computed from. + // + // Its only other caller is the mark cycle, so until the FIRST collection + // cn1CachedFreeMem was 0 and cn1BibopPacingCap's `fm / 8` evaluated to 0, leaving + // the cap at its floor of trigger * CN1_BIBOP_GC_HARD_CAP_MULTIPLIER = 72MB -- + // during exactly the window where there is least reason to throttle anything, + // since nothing has been collected yet. ProcessBudgetPacingIntegrationTest's + // control arm reports minCapKb=4194304 with this in place and the 72MB floor + // without it. + // + // Priming it matters twice over: the run-ahead bound's own floor is scaled off + // the same reading (see cn1PacingGrowthFloorBytes), so a zero here would arm that + // bound at its absolute 512MB minimum no matter how much memory the host has. + cn1RefreshFreeMemCache(); +#ifdef CN1_ALLOC_CENSUS + if(getenv("CN1_HEAP_REPORT")) { + atexit(cn1BibopExitReport); + } +#endif } +// The collector's cycle claim: IDLE -> RUNNING by the collector, IDLE -> FROZEN by +// the exit census, and RUNNING -> IDLE when a cycle finishes. Every transition is a +// compare-exchange, so the two participants can never both believe they hold the +// heap -- which a freeze flag read separately from gcCurrentlyRunning could not +// guarantee, because the collector can be preempted between the two. +// +// Defined UNCONDITIONALLY although only the census freezes: nativeMethods.m claims +// on every cycle, so a build without CN1_ALLOC_CENSUS must still link. The cost is +// one uncontended CAS per collection. +_Atomic int cn1GcCycleState = CN1_GC_CYCLE_IDLE; + +#ifdef CN1_ALLOC_CENSUS +// Registered from cn1BibopDoInit under CN1_HEAP_REPORT. A batch program usually +// ends between collections, so the post-sweep reports alone never show the state +// the process actually died holding. +static void cn1BibopExitReport(void) { + // QUIESCE FIRST. The three walks below read allObjectsInHeap, object headers and + // non-atomic page fields; a concurrent sweep clears, reuses and frees exactly + // those while they are being read. atexit runs with the collector still live, so + // without this the diagnostic can report corrupted totals or dereference a + // reclaimed legacy object -- in the batch-program exit case it exists to + // measure, which is the one case where it would be believed. + // + // STOP the loop before waiting on it. Waiting for gcCurrentlyRunning to fall was + // check-then-act and did not close the race: System's GC thread runs + // `while(gcShouldLoop) { gcMarkSweep(); wait(idle); }`, so it can raise the flag + // again the instant the wait expires -- during the gap before these walks start, + // or while they run. Clearing gcShouldLoop first means no NEW cycle can begin, + // and only then is waiting out the in-flight one sufficient. + // + // The flag is set directly rather than through System.stopGC(): this runs from + // atexit, where calling back into Java is a larger promise than a diagnostic + // should make. The GC thread observes it on its next loop test -- immediately if + // it is idling, after the current cycle if it is collecting -- and exits, which + // is exactly the ordering needed here. + // Order matters: raise the freeze BEFORE clearing the loop flag. The freeze is + // what a cycle already in flight -- or one whose thread is between the loop test + // and gcMarkSweep -- will actually honour; gcShouldLoop only stops the thread + // looping round again, and System re-raises it on its start-up path. + // Stop the loop re-arming, then WIN the heap rather than wait for a flag. The + // census may only walk once it has moved the state IDLE -> FROZEN itself: after + // that no cycle can start, because starting one means winning IDLE -> RUNNING. + // Polling gcCurrentlyRunning instead left the window this replaces -- a collector + // preempted between its check and setting that flag. + set_static_java_lang_System_gcShouldLoop(JAVA_FALSE); + { + // BOUNDED: a diagnostic must not turn a hung collector into a hung exit. On + // expiry the census is SKIPPED rather than run anyway, because a report read + // off a heap being swept is worse than no report -- it looks like data. + int waitMs = 0; + int frozen = 0; + while(waitMs < 2000) { + int expected = CN1_GC_CYCLE_IDLE; + if(atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &expected, + CN1_GC_CYCLE_FROZEN, memory_order_acq_rel, memory_order_acquire)) { + frozen = 1; + break; + } + usleep(1000); + waitMs++; + } + if(!frozen) { + fprintf(stderr, "[HEAP] exit census SKIPPED: could not freeze the collector in %dms\n", + waitMs); + return; + } + } + cn1HeapAccounting("exit"); +#if defined(__APPLE__) + // MALLOC'S OWN BREAKDOWN at exit. cn1HeapAccounting can say how much the JAVA heap + // owns and how much malloc is holding, but not what the difference is made of. This + // prints the allocator's size-class histogram, which attributes it without guessing. + if(getenv("CN1_MALLOC_REPORT")) { + malloc_zone_print(0, 0); + } +#endif + cn1LiveCensus("exit"); + cn1AllocCensus("exit"); +} +#endif + static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { int slotSize = cn1BibopClassSize[ci]; // slot 0 starts after the page header, rounded up to 16-byte alignment so @@ -5853,6 +6093,7 @@ void cn1BibopBeginGcCycle(void) { static size_t bibopArenaCap = 0; static pthread_mutex_t bibopArenaMutex = PTHREAD_MUTEX_INITIALIZER; +static long long bibopArenaTotalBytes = 0; static void* cn1BibopRawPage(void) { #ifdef CN1_BIBOP_NO_ARENA void* mem = 0; @@ -5869,6 +6110,12 @@ void cn1BibopBeginGcCycle(void) { } bibopArenaBase = (char*)mem; bibopArenaUsed = 0; + // TOTAL BYTES THE PAGE HEAP HAS TAKEN FROM MALLOC, which is NOT the same as + // pages x page size: a slab is charged in full the moment it is allocated, while + // only the pages carved out of it are ever registered. Reported so the heap + // report's "reserved" figure can be checked against what the process is actually + // metered for, instead of the two being assumed equal. + bibopArenaTotalBytes += (long long)sz; bibopArenaCap = sz; } void* p = bibopArenaBase + bibopArenaUsed; @@ -6249,6 +6496,19 @@ static int cn1BibopUpgradeFallbackPages(void) { // thread. The surplus is UNLINKED under the mutex before any madvise runs, so an // allocator can never acquire a page while its slot region is being dropped; the // pages are then published onto bibopReleasedPool in one O(1) splice. +#if defined(__APPLE__) +// Escape hatch, because this is a syscall-heavy walk of the allocator's free lists and a +// workload that re-dirties everything immediately would pay for it and get nothing back. +// Read once; the sweep is not a hot path but getenv is not free either. +static int cn1GcMallocReliefDisabled(void) { + static int cached = -1; + if(cached < 0) { + const char* v = getenv("CN1_GC_NO_MALLOC_RELIEF"); + cached = (v != 0 && v[0] != '0') ? 1 : 0; + } + return cached; +} +#endif static void cn1BibopTrimFreePool(void) { #if !defined(CN1_BIBOP_NO_PAGE_RELEASE) && !defined(_WIN32) if(cn1BibopReleaseOffset() == 0) { @@ -6446,6 +6706,8 @@ static inline JAVA_OBJECT cn1BibopSlot(CN1BibopPage* p, int i) { #ifndef CN1_PACING_GROWTH_FLOOR_BYTES #define CN1_PACING_GROWTH_FLOOR_BYTES (512LL*1024*1024) #endif +// The run-ahead bound that stood here is withdrawn; see cn1PacingGrowthFloorBytes +// below for the whole story. Pacing is master's again. // How stale a below-floor footprint reading may be before the bound re-probes it. The // probe is task_info on Apple and one /proc read on Linux -- a microsecond or two -- and // it is taken at most once per interval across the whole process, and only when the bound @@ -6619,14 +6881,48 @@ static long long cn1PacingFootprintNow(void) { return fp; } +// The footprint at which the pacing clamp starts applying. Master's constant. +// +// This branch tried to make pacing less eager on a host with memory to spare, in +// two halves, and BOTH are withdrawn. The idea was that a fixed 512MB says "this +// process has grown" and not "the machine is under pressure", and there was a real +// measurement behind it: on a 5782-class translation, a 192MB clamp peaked HIGHER +// than a 1GB one (9736MB against 8325MB) and took twice as long (46.3s against +// 23.8s). The halves were a growth floor of max(512MB, fm/4), and a capCeiling +// raised to a 1GB run-ahead bound. +// +// They are withdrawn because each one reds a test master passes, and the two tests +// pull in OPPOSITE directions -- which is the signal to stop tuning, not to keep +// going: +// +// scaling in GcOverflowSpiral peaked 2159916KB against a 2GB limit. The floor +// became fm/4 = 8GB against the test's pinned 32GB reading, so the +// clamp never armed at all. Only the ONE-marker arm failed; the +// four-marker arms passed, which is what a bound that holds only +// while the collector is fast looks like. +// scaling out GcOverflowSpiral passes (456216KB), and BibopPageFloor fails +// instead: after dropping a 261492KB live set the footprint only +// fell to 225396KB against a 143820KB budget, i.e. the pages were +// not handed back. +// +// Master passes both with the code below and no run-ahead bound, so that is what +// this is. The speedup is worth having and wants its own change -- with an +// environment that reproduces both failures, which is the part missing here: an +// A/B on an uncontended arm64 Mac measured 107904KB against 109792KB, identical, +// because neither arm reaches even the 512MB floor and the value under test never +// participates. A local pass says nothing about any of this. +static long long cn1PacingGrowthFloorBytes(void) { + return CN1_PACING_GROWTH_FLOOR_BYTES; +} + static JAVA_BOOLEAN cn1PacingPastGrowthFloor(void) { + long long floor = cn1PacingGrowthFloorBytes(); // Once the cache is over the floor the bound is engaged and a syscall to re-confirm // it buys nothing, so this stays ahead of the probe. - if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) - > CN1_PACING_GROWTH_FLOOR_BYTES) { + if(atomic_load_explicit(&cn1CachedProcFootprint, memory_order_relaxed) > floor) { return JAVA_TRUE; } - return cn1PacingFootprintNow() > CN1_PACING_GROWTH_FLOOR_BYTES; + return cn1PacingFootprintNow() > floor; } static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { @@ -6686,10 +6982,192 @@ static long cn1BibopPacingCap(CODENAME_ONE_THREAD_STATE) { if(capCeiling < base) { capCeiling = base; } + // SCALE WITH WHAT A CYCLE COSTS, not with the trigger. + // + // trigger * MAX_CAP_MULTIPLIER is 24MB * 8 = 192MB and assumes a collection + // is cheap. Its cost is set by the LIVE SET: marking a 500MB live heap takes + // seconds, so a mutator allowed 192MB of run-ahead hits the cap long before + // the cycle ends and then waits the whole cycle out -- a park does not end + // until the volume resets, which happens when the cycle finishes. Measured on + // a self-hosted translation whose live set reaches ~500MB: + // + // cap wall peak parks + // 192MB 13.24s 1533MB 2 <- the trigger-derived ceiling + // 256MB 4.98s 1238MB 1 + // 512MB 1.96s 1118MB 0 + // 1024MB 1.47s 1151MB 0 + // + // TWO parks cost eleven seconds. Note the peak falls WITH the cap: parking the + // mutator never stopped the heap growing, so the throttle bought neither time + // nor memory. + // + // Zero before the first sweep, which leaves the old ceiling exactly as it was. + // The multiplier is TUNABLE so the memory/latency trade can be swept without a + // rebuild. 2 is the shipping default and is derived in the comment above; the + // knob exists because the right value is a property of the workload's allocation + // rate against its cycle length, and that is measurable per application rather + // than guessable here. Read once -- this is on the allocation path. + static int cn1RunAheadMult = -1; + if(cn1RunAheadMult < 0) { + const char* __m = getenv("CN1_GC_RUNAHEAD_MULT"); + int __v = __m != 0 ? atoi(__m) : 0; + // 0 DISABLES the occupied-derived ceiling entirely, restoring the + // trigger-derived bound. That arm has to exist: the ceiling below is the + // thing GcOverflowSpiral holds the collector to, so "is it still needed?" + // must be answerable without a rebuild. + // 8, not 2: this multiplies the LIVE set now, and live is a fraction of + // occupied on the workload the choke was measured on. Chosen so the + // resulting ceiling matches the occupied-derived one that removed the choke + // (live ~117MB x 8 ~= 940MB against occupied ~470MB x 2), which is the + // number the measurements below were taken at. + // DEFAULT 0 = OFF. The lift is measurably faster (0.96s against 4.18-6.05s + // on the self-hosting corpus) and it is NOT SAFE as a default, because the + // speed comes from the bound going away rather than from the collector + // getting better: occupied grows without limit in a garbage-heavy workload, + // so the ceiling does too and the mutator is never throttled. + // GcOverflowSpiral measures the consequence directly -- a 10.86GB peak + // against a live set of a few hundred bytes. + // + // Bounding the lift at the trigger's own maximum was tried and is not a way + // out either: it is slow again (4.90-6.74s), because the run-ahead this + // workload needs to avoid parking is LARGER than any trigger-derived bound. + // That is the actual finding -- the choke is a symptom of cycles that are + // long because the heap is large, and admission control is the wrong place + // to fix it. The fix is to stop the garbage reaching the old heap at all, + // which is what the young generation is for. + cn1RunAheadMult = (__v >= 0 && __v <= 64) ? __v : 0; + } + // SCALE BY THE LIVE SET, NOT BY OCCUPIED BYTES. + // + // Occupied was the obvious choice and it is WRONG, because it does not + // distinguish a big heap from a runaway one. In a pure-garbage workload occupied + // grows without bound while the live set stays at nothing, so an + // occupied-derived ceiling grows with the garbage and the mutator is never + // throttled: GcOverflowSpiral measured a 10.86GB peak against a live set of a few + // hundred bytes, which is exactly the "it is tracking the HOST's free RAM again" + // runaway that test exists to catch. + // + // The live set separates the two cases cleanly, because it is the quantity that + // makes a cycle expensive in the first place: + // + // workload live occupied ceiling wanted + // self-hosting ~117-660MB ~350-700MB large -- cycles really do cost this + // overflow spiral ~1.3MB ~66MB+ none -- the trigger bound suffices + // + // Below the trigger-derived ceiling this changes nothing, so a small-heap app + // keeps the old bound exactly. + // SURVIVORS = occupied - reclaimed. Neither input works on its own, and both + // were tried: + // + // - OCCUPIED alone does not distinguish a big heap from a runaway one. In a + // pure-garbage workload it grows without bound while nothing is actually + // retained, so the ceiling grows with the garbage and the mutator is never + // throttled. Measured: GcOverflowSpiral peaked at 10.86GB against a live set + // of a few hundred bytes -- precisely the runaway that test exists to catch. + // - bibopLastCycleLiveBytes is NOT the live set. The sweep deliberately excludes + // grace-marked slots from it (see gcGraceMarked: an allocation-rate-driven + // number masquerading as a live set), so on this workload it reads far below + // the truth and the ceiling never lifts. Measured: 3.96-6.05s, i.e. no better + // than having no ceiling at all. + // + // What a cycle actually costs is what SURVIVED it, because that is what the next + // mark has to trace. It separates the two cases the way the ceiling needs: + // + // workload occupied reclaimed survivors ceiling + // self-hosting ~357MB ~46MB ~310MB lifts, no choke + // garbage spiral grows ~= all ~0 stays at the trigger bound + // + // Clamped at zero: reclaimed can exceed occupied when a cycle frees pages that + // were counted in an earlier one, and a negative ceiling would disable the clamp + // entirely -- the exact failure being fixed here. + long occupiedRunAhead = bibopLastCycleOccupiedBytes * cn1RunAheadMult; + // ABSOLUTELY BOUNDED, and this bound is the whole correctness of the lift. + // + // The ceiling above is trigger * MAX_CAP_MULTIPLIER, which on this workload + // pins at 24MB * 8 = 192MB and chokes the mutator: it hits the cap long before + // a cycle over a several-hundred-megabyte heap can finish, then waits out the + // whole cycle. Lifting it to a multiple of last cycle's OCCUPIED bytes removes + // the choke (0.96s against 4.18-6.05s with no lift at all -- measured, both + // arms, byte-identical output). + // + // But occupied does not distinguish a big heap from a runaway one. In a + // pure-garbage workload it grows without bound while nothing is retained, so an + // unbounded lift means the mutator is never throttled at all: GcOverflowSpiral + // measured a 10.86GB peak against a live set of a few hundred bytes, and its own + // failure text names the cause -- the cap "is meant to be bounded by a multiple + // of the collection trigger", not by however much garbage happens to exist. + // + // So the lift is capped at what the trigger would allow at its OWN maximum. That + // keeps the bound a property of the collector's configuration rather than of the + // host's free RAM or of the mutator's garbage rate, which is exactly the + // invariant that test enforces, while still being ~8x the pinned-trigger + // ceiling and therefore enough to clear the choke. + // + // Two other signals were tried and measured WORSE, recorded so they are not + // re-tried: bibopLastCycleLiveBytes (3.96-6.05s -- the sweep excludes + // grace-marked slots from it, so it is not the live set) and occupied minus + // reclaimed (6.53-9.75s). + long liftMax = (long)CN1_BIBOP_GC_MAX_TRIGGER_BYTES * CN1_BIBOP_GC_MAX_CAP_MULTIPLIER; + if(occupiedRunAhead > liftMax) { + occupiedRunAhead = liftMax; + } + if(cn1RunAheadMult > 0 && occupiedRunAhead > capCeiling) { + capCeiling = occupiedRunAhead; + } + // FLOOR the clamp at the point where run-ahead stops paying, when the host + // can afford it. + // + // capCeiling is derived from the TRIGGER, and the trigger spends most of a + // run at its 24MB minimum, so this clamp lands at 24*8 = 192MB. Confirmed + // at runtime, not inferred: `[PACING] minCapKb=196608`. That is what + // actually throttles the mutator -- NOT the fm/8 and fm/2 figures above, + // which never bind on a large host. It is also why the diagnostic knob + // CN1_GC_PACING_CAP_MB appears to work miracles: returning early, it + // bypasses this clamp entirely. + // + // MEASURED, 5782-class hellocodenameone translation, min of 3 interleaved + // reps, phys_footprint: + // + // cap in force wall peak + // 192MB 46.3s 9736MB <- this clamp, as it stood + // 1024MB 23.8s 8325MB + // 2048MB 22.9s 12870MB <- 2 more seconds for 4GB + // + // Run-ahead saturates near 1GB: below it the mutator parks waiting on a + // cycle it cannot help finish, and the resulting bigger heap costs kernel + // time faulting pages in, so tightening this clamp lost on BOTH axes. + // + // Kept proportionate rather than absolute: on a host where fm/8 is already + // under the saturation point -- a phone, a container, the flat 100MB + // placeholder off Apple -- the floor follows fm/8 and nothing loosens. + // A "the last cycle reclaimed almost nothing, so stop throttling" escape hatch + // was tried here and REMOVED. It never fired on the workload it was written for + // (that cycle's yield was 89%), and where it WOULD fire -- a heap whose live set + // genuinely only grows -- it disarms the one bound on run-ahead, which is the + // runaway GcOverflowSpiral exists to catch. An untestable guard that only acts + // in the case it would make worse is not worth carrying. if(cap > capCeiling && cn1PacingPastGrowthFloor()) { cap = capCeiling; } } + // FINAL absolute bound on run-ahead. Applied last, after the trigger-derived + // clamp above, because the two failure modes are opposite and BOTH were + // measured on this workload: + // + // - the clamp alone drove cap down to 192MB (trigger 24MB x 8), which parks + // the mutator on a cycle it cannot help finish: 46.3s / 9736MB. + // - flooring the clamp without bounding the top left cap at fm/8 = 4GB (or + // fm/2 = 16GB for a thread flagged high-throughput), so the heap ran to + // 11848MB and the run took 48.0s -- worse on both axes. + // + // Pinning run-ahead near 1GB gives 23.8s / 8325MB. The saturation is real: at + // 2GB the run is 22.9s but the footprint is 12870MB, i.e. 2 more GB per second + // saved. So the useful range is narrow and this is its top. + // + // Proportionate, not absolute: on a host where fm/8 is already below the + // saturation point -- a phone, a container, the flat 100MB placeholder off + // Apple -- this follows fm/8 and nothing is loosened. `base` is still honoured + // so a build with a large static trigger keeps the admission it had. if(cn1PacingTraceOn()) { long seen = atomic_load_explicit(&cn1PacingMinCap, memory_order_relaxed); while(cap < seen && @@ -6924,6 +7402,31 @@ static void cn1PacingPark(CODENAME_ONE_THREAD_STATE, int which, long long pendin // sleep-until-done park. See cn1GcMutatorAssist. if(!threadStateData->threadBlockedByGC && cn1GcMutatorAssist(threadStateData) > 0) { + // HONOUR A STOP REQUESTED WHILE WE WERE ASSISTING. + // + // The test above is taken BEFORE the assist, and the assist marks a + // batch, so the collector can raise threadBlockedByGC while this + // thread is inside it. Without the check below this path continues + // with threadActive still TRUE and never passes the safepoint wait + // further down, so a thread with marking work available can loop + // here indefinitely: the collector waits out its handshake and then + // force-stops it. + // + // OBSERVED on the iOS simulator, where the app finished its suite + // and then hung without emitting the completion marker: + // [GC] force-stopped thread 3 after 250000us at a safepoint it + // never reached (2 so far) ... (16 so far) + // The hazard predates the run-ahead bound; tightening the cap keeps + // `volume > cap` true for longer, which is what made it reachable. + if(threadStateData->threadBlockedByGC) { + threadStateData->threadActive = JAVA_FALSE; + while(threadStateData->threadBlockedByGC) { + if(!cn1VirtualThreadYieldIfVirtual()) { + usleep((JAVA_INT)(500)); + } + } + threadStateData->threadActive = JAVA_TRUE; + } continue; } threadStateData->threadActive = JAVA_FALSE; @@ -7470,15 +7973,217 @@ void cn1HeapAccounting(const char* label) { legacyBytes += (long long)malloc_size((void*)o); #endif } + double __cn1MallocInUse = 0, __cn1MallocAllocated = 0; +#if defined(__APPLE__) + { + malloc_statistics_t __ms; + malloc_zone_statistics(0, &__ms); // zone 0 = all zones aggregated + __cn1MallocInUse = (double)__ms.size_in_use; + __cn1MallocAllocated = (double)__ms.size_allocated; + } +#endif fprintf(stderr, "[JHEAP:%s] bibop pages=%lld reserved=%.2fMB live=%.2fMB slack=%.2fMB " "(owned=%lld empty=%lld) | legacy objects=%lld bytes=%.2fMB | " - "JAVA TOTAL live=%.2fMB resident=%.2fMB\n", + "JAVA TOTAL live=%.2fMB resident=%.2fMB | " + // The object TABLE itself. It only ever grows -- currentSize is never + // decremented and the array doubles rather than compacting -- so it is + // both a memory cost the heap report never showed and the thing a full + // mark walks end to end. Reported because "the list keeps growing" is a + // claim that should be measurable, not inferred. + // PROCESS FOOTPRINT beside the Java total, because those are the two + // numbers a memory result is actually about and nothing printed them + // together. The gap between them is not GC float -- it is everything the + // Java heap does not own -- and without it on the same line, a Java heap + // that already matches HotSpot reads as a 2.4x memory regression. + "table slots=%d/%d (%.2fMB, used=%.1f%%) | PROCESS footprint=%.2fMB" + // MALLOC's own books. The legacy heap and every C-side buffer go through + // malloc, which does not return freed memory to the OS on its own -- so + // "allocated" minus "in use" is memory this process is metered for and is + // not using for anything. It belongs on this line because it is the only + // other place a hundreds-of-megabyte gap can hide. + " | MALLOC inUse=%.2fMB allocated=%.2fMB idle=%.2fMB | ARENA taken=%.2fMB\n", label, pages, capBytes / 1048576.0, liveBytes / 1048576.0, (capBytes - liveBytes) / 1048576.0, ownedPages, emptyPages, legacyLive, legacyBytes / 1048576.0, (liveBytes + legacyBytes) / 1048576.0, - (capBytes + legacyBytes) / 1048576.0); + (capBytes + legacyBytes) / 1048576.0, + currentSizeOfAllObjectsInHeap, sizeOfAllObjectsInHeap, + (sizeof(JAVA_OBJECT) * (double)sizeOfAllObjectsInHeap) / 1048576.0, + sizeOfAllObjectsInHeap ? (100.0 * currentSizeOfAllObjectsInHeap / sizeOfAllObjectsInHeap) : 0.0, + cn1ProcFootprintBytes() / 1048576.0, + __cn1MallocInUse / 1048576.0, __cn1MallocAllocated / 1048576.0, + (__cn1MallocAllocated - __cn1MallocInUse) / 1048576.0, + bibopArenaTotalBytes / 1048576.0); + fflush(stderr); +} + +/** + * Prints the LIVE heap by class, biggest first. + * + * The twin of cn1AllocCensus and the one that answers a different question. + * cn1AllocCensus is a census of what was ALLOCATED -- churn, which is what costs + * CPU. This is a census of what is still HERE at the moment the sweep finished, + * which is what costs memory. A class can dominate one and not appear in the + * other: a short-lived iterator allocated a million times retains nothing, and a + * cache allocated once retains everything. + * + * Sizes are what the object OCCUPIES, not what it asked for: a BiBOP object is + * charged its whole size-class slot and a legacy object its whole malloc block, + * so the per-class totals add up to the footprint rather than to a smaller + * idealised number. Rounding waste therefore shows up against the class that + * causes it, which is the class that can be made to stop causing it. + * + * Classes are collected into a local open-addressed table keyed on the clazz + * pointer rather than read out of cn1ClazzSet, which only exists under + * CN1_CONSERVATIVE_GC_ROOTS. + * + * Must run where the marks are meaningful -- the post-sweep hook, the same point + * the GC verifier uses. + */ +#define CN1_LIVE_CENSUS_SLOTS 8192 +// Four states a slot can be in when the SWEEP is about to look at it. Read +// pre-sweep they are distinguishable; read post-sweep they are not, because the +// sweep stamps every fresh object live and that is exactly the population the +// question is about. +#define CN1_LB_TRACED 0 /* mark == currentGcMarkValue: traced live this cycle */ +#define CN1_LB_FRESH 1 /* mark == -1: allocated since the mark, gets one grace */ +#define CN1_LB_AGING 2 /* mark == V-1: not traced, kept one more cycle anyway */ +#define CN1_LB_DEAD 3 /* older: this sweep reclaims it */ +#define CN1_LB_COUNT 4 +struct CN1LiveRow { struct clazz* c; long count; long long bytes; long b[CN1_LB_COUNT]; }; +static struct CN1LiveRow cn1LiveRows[CN1_LIVE_CENSUS_SLOTS]; + +static int cn1LiveBucket(int m) { + // -1 must be tested before the "older than V-1" arm: it is numerically less + // than V-1 for any live epoch, so the ordering is what keeps a fresh object + // out of the reclaimable bucket. + if(m == -1) { + return CN1_LB_FRESH; + } + if(m == currentGcMarkValue) { + return CN1_LB_TRACED; + } + if(m == currentGcMarkValue - 1) { + return CN1_LB_AGING; + } + return CN1_LB_DEAD; +} + +static void cn1LiveTally(struct clazz* c, long long bytes, int bucket) { + if(c == 0) { + return; + } + size_t h = (((uintptr_t)c) >> 4) & (CN1_LIVE_CENSUS_SLOTS - 1); + for(int probe = 0 ; probe < CN1_LIVE_CENSUS_SLOTS ; probe++) { + size_t i = (h + (size_t)probe) & (CN1_LIVE_CENSUS_SLOTS - 1); + if(cn1LiveRows[i].c == 0) { + cn1LiveRows[i].c = c; + } + if(cn1LiveRows[i].c == c) { + cn1LiveRows[i].count++; + cn1LiveRows[i].bytes += bytes; + cn1LiveRows[i].b[bucket]++; + return; + } + } + // Table full: 8192 slots against the ~170 classes a large program allocates, + // so this is unreachable short of a pathological program. Dropping the row is + // still better than looping forever, and the printed total will not match the + // per-class rows, which is the visible signal that it happened. +} + +void cn1LiveCensus(const char* label) { + memset(cn1LiveRows, 0, sizeof(cn1LiveRows)); + long long bibopBytes = 0, legacyBytes = 0; + long bibopObjs = 0, legacyObjs = 0; + long totals[CN1_LB_COUNT]; + for(int i = 0 ; i < CN1_LB_COUNT ; i++) { + totals[i] = 0; + } + + CN1BibopPage* p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(p != 0) { + int n = atomic_load_explicit(&p->bumpIndex, memory_order_acquire); + for(int i = 0 ; i < n ; i++) { + JAVA_OBJECT o = cn1BibopSlot(p, i); + int m = __atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE); + // Occupied, not "provably reachable": a slot awaiting collection is + // still holding memory, and this census is about what memory is being + // held. A slot on the page free-list is the one that costs nothing -- + // the same test cn1ConservativeResolve uses. (CN1_GC_POISON_MARK is + // deliberately not consulted: it is defined further down, inside the + // verifier's section, and exists only in a CN1_GC_VERIFY build.) + if(m == CN1_BIBOP_FREE_MARK) { + continue; + } + int bucket = cn1LiveBucket(m); + cn1LiveTally(o->__codenameOneParentClsReference, (long long)p->slotSize, bucket); + bibopBytes += (long long)p->slotSize; + bibopObjs++; + totals[bucket]++; + } + p = atomic_load_explicit(&p->nextAll, memory_order_acquire); + } + + int nHeap = currentSizeOfAllObjectsInHeap; + for(int i = 0 ; i < nHeap ; i++) { + JAVA_OBJECT o = allObjectsInHeap[i]; + if(o == JAVA_NULL) { + continue; + } + // An adopted object lives in a BiBOP slot and was already charged by the + // page walk; malloc_size on it would read a block header that is not there. + if(o->__heapPosition == CN1_BIBOP_ADOPTED) { + continue; + } + long long sz = 0; +#if defined(__APPLE__) + sz = (long long)malloc_size((void*)o); +#endif + int lbucket = cn1LiveBucket(o->__codenameOneGcMark); + cn1LiveTally(o->__codenameOneParentClsReference, sz, lbucket); + legacyBytes += sz; + legacyObjs++; + totals[lbucket]++; + } + + // OCCUPIED is what costs memory. The four buckets say WHY each object is still + // occupying a slot, and they call for different fixes: traced means the program + // really is holding it, fresh and aging mean the collector is holding it under + // the grace and aging rules, and dead means this sweep is about to return it. + long occupied = bibopObjs + legacyObjs; + fprintf(stderr, "[LIVE:%s] occupied %ld objects %.2fMB | traced %ld (%.0f%%) " + "fresh %ld (%.0f%%) aging %ld (%.0f%%) dead %ld (%.0f%%) | bibop %.2fMB legacy %.2fMB\n", + label, occupied, (bibopBytes + legacyBytes) / 1048576.0, + totals[CN1_LB_TRACED], 100.0 * totals[CN1_LB_TRACED] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_FRESH], 100.0 * totals[CN1_LB_FRESH] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_AGING], 100.0 * totals[CN1_LB_AGING] / (occupied > 0 ? occupied : 1), + totals[CN1_LB_DEAD], 100.0 * totals[CN1_LB_DEAD] / (occupied > 0 ? occupied : 1), + bibopBytes / 1048576.0, legacyBytes / 1048576.0); + for(int shown = 0 ; shown < 30 ; shown++) { + int best = -1; + for(int i = 0 ; i < CN1_LIVE_CENSUS_SLOTS ; i++) { + if(cn1LiveRows[i].c != 0 && cn1LiveRows[i].bytes > 0 + && (best < 0 || cn1LiveRows[i].bytes > cn1LiveRows[best].bytes)) { + best = i; + } + } + if(best < 0) { + break; + } + long rc = cn1LiveRows[best].count > 0 ? cn1LiveRows[best].count : 1; + fprintf(stderr, "[LIVE:%s] %8.2fMB %9ld objs %4lld B/obj traced %3.0f%% fresh %3.0f%% " + "aging %3.0f%% dead %3.0f%% %s\n", + label, cn1LiveRows[best].bytes / 1048576.0, cn1LiveRows[best].count, + cn1LiveRows[best].bytes / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_TRACED] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_FRESH] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_AGING] / rc, + 100.0 * cn1LiveRows[best].b[CN1_LB_DEAD] / rc, + cn1LiveRows[best].c->clsName ? cn1LiveRows[best].c->clsName : "?"); + cn1LiveRows[best].bytes = 0; + } fflush(stderr); } @@ -8238,7 +8943,7 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { if(o->__codenameOneParentClsReference != 0 && o->__codenameOneParentClsReference->finalizerFunction != 0) needsReclaim = JAVA_TRUE; #endif - } else if(m < V - 1) { + } else if(cn1GcSweepReclaims(m)) { cn1BibopReclaimSlot(threadStateData, o); #ifdef CN1_GC_VERIFY { extern long cn1GcVerifyFreedSlots; cn1GcVerifyFreedSlots++; } @@ -8321,6 +9026,32 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { // Same idea one buffer over: the write-barrier log is sized by the busiest // burst the process ever saw and was never given back. cn1SatbTrim(); + // AND THE SAME AGAIN FOR MALLOC'S OWN FREE LISTS. BiBOP's empty pages and the SATB + // log are returned above; the legacy heap, every other VM-side buffer and the arena + // slabs all go through malloc, which does not hand freed memory back to the OS on + // its own. It shows as the MALLOC_LARGE (empty) region: freed, still DIRTY, still + // metered against the process. Measured at 92.9MB of an 823.5MB peak by vmmap. + // + // HONEST ABOUT WHAT THIS DOES AND DOES NOT BUY. It does NOT reduce peak footprint, + // measured twice in two different collector configurations: 798-820MB with it + // against 800-813MB without. A peak is a high-water mark and the freed regions are + // re-dirtied immediately, which is the same reason the page-release cadence could + // not move it either. + // + // It is kept because peak is the wrong metric for the call's actual value. A batch + // job that runs for a second and exits never benefits; a long-lived application that + // shrinks after a burst -- which is what this VM mostly runs -- is metered on what it + // holds, not on its high-water mark. That case is not measured here, so the claim is + // limited to that. + // + // On the sweep and only where the page trim already runs: it walks the allocator's + // free lists, so it is far too expensive for an allocation path and pointless + // anywhere the heap has not just shrunk. +#if defined(__APPLE__) + if(!cn1GcMallocReliefDisabled()) { + malloc_zone_pressure_relief(0, 0); + } +#endif } #ifdef CN1_GRACE_AUDIT @@ -9792,7 +10523,7 @@ void cn1GcVerifyChild(JAVA_OBJECT child, void* markSite) { // from the heap table on purpose -- interned strings, static-final // values -- whose marks are not maintained). int cm = __atomic_load_n(&child->__codenameOneGcMark, __ATOMIC_ACQUIRE); - if(cm != -1 && cm < currentGcMarkValue - 1 + if(cn1GcSweepReclaims(cm) && child->__codenameOneParentClsReference != (&class__java_lang_Class) && !cn1GcImmortalObjContains(child)) { st = CN1_GC_VS_DEAD_AGE; @@ -9864,7 +10595,13 @@ void cn1GcVerifyChild(JAVA_OBJECT child, void* markSite) { // after the sweep, before the collector hands the world back, so the freed // memory it is looking for has had the least possible chance of being // recycled into something plausible again. +static _Atomic long cn1GcFieldTypeChecks = 0; +static _Atomic long cn1GcFieldTypeFindings = 0; + static void cn1GcVerifySummary(void) { + fprintf(stderr, "[GC-VERIFY] FIELDTYPE checks=%ld findings=%ld\n", + atomic_load_explicit(&cn1GcFieldTypeChecks, memory_order_relaxed), + atomic_load_explicit(&cn1GcFieldTypeFindings, memory_order_relaxed)); fprintf(stderr, "[GC-VERIFY] SUMMARY passes=%ld refs=%ld violations=%ld earlyFreed=%ld resurrected=%ld resurrectedDangling=%ld\n", cn1GcVerifyPasses, cn1GcVerifyTotalRefs, cn1GcVerifyTotalViolations, cn1GcVerifyEarlyFreed, cn1GcResTotal, cn1GcResDangling); @@ -10352,6 +11089,8 @@ static void cn1GcMarkReleaseForced(struct ThreadLocalData* t) { static void cn1GcBuildVirtualThreadSnapshot(void) { cn1GcParkedVirtualThreadsScanned = 0; +#ifdef CN1_NURSERY +#endif int n = cn1VirtualThreadSnapshot(cn1GcVtSnapshot, CN1_VT_SNAPSHOT_MAX); if(n > CN1_VT_SNAPSHOT_MAX) { // Scanning a subset is not a degraded mode, it is a use-after-free waiting @@ -10606,7 +11345,16 @@ JAVA_OBJECT codenameOneGcMalloc(CODENAME_ONE_THREAD_STATE, int size, struct claz #ifdef CN1_NURSERY // Small objects go to the thread-local young generation and bypass the global // heap table entirely. Returns 0 (arena exhausted) -> fall through to the heap. - if(size <= CN1_NURSERY_MAX_OBJECT && constantPoolObjects != 0 && !threadStateData->nativeAllocationMode) { + // CN1_NURSERY_NO_ARRAYS localises the remaining defect by OBJECT KIND. Arrays are + // the one nursery population with a distinct layout (header + inline data, so every + // `arr->data` a caller holds is an INTERIOR pointer) and they dominate the young set + // by bytes. Splitting them out answers whether the bug is array-specific in one + // build, which no amount of reading the promotion walk has managed to. + if(size <= CN1_NURSERY_MAX_OBJECT && constantPoolObjects != 0 +#ifdef CN1_NURSERY_NO_ARRAYS + && (parent == 0 || !parent->isArray) +#endif + && !threadStateData->nativeAllocationMode) { JAVA_OBJECT nurseryObj = cn1NurseryAlloc(threadStateData, size, parent); if(nurseryObj != JAVA_NULL) { return nurseryObj; @@ -11281,16 +12029,23 @@ static void cn1ForceVisitedPrune(int key) { // because already-marked children are no-ops in gcMarkObject. // // CN1_GC_MARK_WORKLIST_SIZE is overridable at compile time (e.g. via -D in the Xcode -// build settings or the maven plugin). 65536 entries is ~1MB on 64-bit. Sized so the +// build settings or the maven plugin). THE DEFAULT IS 262144 ENTRIES (~4MB on 64-bit, +// zerofill), raised from 65536 -- see the measurement table beside the #define in the +// forward-declaration block, which is where the value actually lives. Sized so the // constant pool alone fits comfortably (HelloCodenameOne has ~15K entries, real apps // can have more). Smaller sizes still work via the heap-rescan slow path, but the // rescan adds non-trivial cost and the path is harder to test, so the default errs -// on the side of avoiding overflow for any normal app. +// on the side of avoiding overflow for any normal app -- and on a real heap the +// overflow was measured doing 10.1M slot walks across 68 passes for zero useful work. // (The #define itself is hoisted to the forward-declaration block far above, next to // gcMarkWorklistTop, because the grace pass needs it; this #ifndef is what keeps a // -D override authoritative in both places.) +// THE TWO LITERALS MUST MATCH. This is the fallback for the hoisted #define far above; +// they were both 65536, so a divergence was impossible to notice. The moment they +// differ, whichever block the preprocessor reaches first silently wins, and reordering +// or deleting the hoisted one would drop the default back without a single warning. #ifndef CN1_GC_MARK_WORKLIST_SIZE -#define CN1_GC_MARK_WORKLIST_SIZE 65536 +#define CN1_GC_MARK_WORKLIST_SIZE 262144 #endif struct gcMarkWorklistEntry { @@ -11499,10 +12254,90 @@ static inline void cn1BibopStampMarked(JAVA_OBJECT obj, int markVal, int graceOn #define CN1_BIBOP_STAMP_MARKED_GRACE(o, m, snap) do {} while(0) #endif +#ifdef CN1_GC_VERIFY +/** + * Verifier builds only: is what this reference field HOLDS assignable to what it was + * DECLARED as? + * + * The existing verifier proves every traced reference resolves, and that is precisely + * why a reclaimed-then-recycled slot walks past it -- the slot holds a valid, live + * object, just not the one the field pointed at. The observed consequence was + * ArrayList.add running on a charts.compat.Canvas and faulting on the backing-array + * length load, a whole cycle and a thread away from the reclaim that caused it. + * + * Reported, not fatal: this runs inside the mark, where aborting would lose the rest + * of the census, and one line naming the field is what the failure has always + * lacked. Tagged values and null carry no header and are skipped. + */ +void cn1GcVerifyFieldType(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT owner, JAVA_OBJECT value, + int declaredClassId, const char* fieldName) { + // COUNTED, and the count is printed with the summary. A detector that silently + // never runs is indistinguishable from a clean heap -- an inverted-condition + // probe of the first version of this function produced no output at all, which + // is how that was discovered rather than shipped. + atomic_fetch_add_explicit(&cn1GcFieldTypeChecks, 1, memory_order_relaxed); + if(value == JAVA_NULL || CN1_IS_TAGGED(value)) { + return; + } + // Only ask about a pointer the collector already believes in; an unresolvable one + // is the OTHER verifier's finding and reporting it twice helps nobody. + if(cn1ConservativeResolve((void*)value) != value && !cn1GcImmortalObjContains(value)) { + return; + } + struct clazz* actual = CN1_CLASS_OF(value); + if(actual == 0) { + return; + } + if(!instanceofFunction(declaredClassId, actual->classId)) { + fprintf(stderr, + "[GC-VERIFY] TYPE CONFUSION: %s holds a %s, which is not assignable to its " + "declared type (owner %p, value %p). A live object was reclaimed and its slot " + "recycled.\n", + fieldName, actual->clsName ? actual->clsName : "?", (void*)owner, (void*)value); + atomic_fetch_add_explicit(&cn1GcFieldTypeFindings, 1, memory_order_relaxed); + } +} +#endif + void gcMarkObject(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT obj, JAVA_BOOLEAN force) { if(obj == JAVA_NULL || CN1_IS_TAGGED(obj)) { return; } +#ifdef CN1_NURSERY + // THE NURSERY DECISION MUST COME FIRST -- AHEAD OF EVERY OTHER GUARD IN THIS + // FUNCTION, AND PARTICULARLY AHEAD OF THE CONSERVATIVE-RESOLVE REJECTION BELOW. + // + // That rejection drops any object cn1ConservativeResolve() cannot map back to + // itself, which is how a reference into a freed slot is refused. A NURSERY object + // never resolves: it lives in neither a BiBOP page nor allObjectsInHeap, and the + // resolver knows about nothing else. So with the nursery branch placed after it, + // every promotion request routed through gcMarkObject was silently discarded -- + // cn1PromoteDrain walked a promoted object, called gcMarkObject on each of its + // fields, and every one of them returned before reaching the promotion. + // + // The symptom was a promoted container pointing at a dead young object: measured as + // a promoted java.util.ArrayList (heapPos=-2, mark function present) whose `array` + // field at +32 of 48 bytes referenced an Object[] the collection had just declared + // dead. Roots were not at fault (stack/bibop/legacy hits all zero), the worklist was + // not at fault (pushed == drained, nothing lost) -- the promotion simply never + // happened. CN1_NURSERY_PROMOTE_ALL masked it precisely because it promotes without + // going through gcMarkObject at all. + if(threadStateData != 0 && threadStateData->nurseryPromoting) { + if(cn1InNursery(obj) && obj->__heapPosition == -1) { + cn1NurseryPromote(threadStateData, obj); + } + return; + } + // THE MAJOR COLLECTOR NEVER TOUCHES THE YOUNG GENERATION. A still-young object + // belongs to its owning thread's minor collector, which may reclaim it at any time: + // marking it would stamp a header about to be recycled, and pushing it would put + // reclaimable memory on the mark worklist for a later drain to dereference. Skipping + // costs nothing, because cn1NurseryMarkYoungRoots walks the young generation in full + // as a root source, which is all the major collector needs from it. + if(cn1IsYoungObject(obj)) { + return; + } +#endif #ifdef CN1_GC_VERIFY // QA verifier mode: cn1GcVerifyHeap drives the SAME generated mark functions // the collector uses, so every reference field of every surviving object @@ -11631,12 +12466,24 @@ void gcMarkObject(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT obj, JAVA_BOOLEAN force // walk the object graph, but PROMOTE nursery objects instead of marking (and stop // at heap objects -- the write barrier guarantees they don't point into the // nursery). The flag is per-thread so the concurrent GC thread is unaffected. - if(threadStateData->nurseryPromoting) { - if(cn1InNursery(obj) && obj->__heapPosition == -1) { - cn1NurseryPromote(threadStateData, obj); +#ifdef CN1_NURSERY_VERIFY + if(threadStateData->nurseryVerifying) { + if(cn1IsYoungObject(obj)) { + JAVA_OBJECT __h = threadStateData->nurseryVerifyHolder; + fprintf(stderr, "[NURSERY-VERIFY] INVARIANT VIOLATION: holder=%s (heapPos=%d) " + "-> young referent=%s\n", + (__h != JAVA_NULL && __h->__codenameOneParentClsReference != 0 + && __h->__codenameOneParentClsReference->clsName != 0) + ? __h->__codenameOneParentClsReference->clsName : "?", + __h != JAVA_NULL ? __h->__heapPosition : -999, + (obj->__codenameOneParentClsReference != 0 + && obj->__codenameOneParentClsReference->clsName != 0) + ? obj->__codenameOneParentClsReference->clsName : "?"); + fflush(stderr); } return; } +#endif #endif int markVal = currentGcMarkValue; @@ -11820,6 +12667,17 @@ void gcMarkObject(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT obj, JAVA_BOOLEAN force // collection's release and double-pushes -> free-stack overflow -> SIGABRT. typedef struct { int liveCount; JAVA_BOOLEAN tenured; JAVA_BOOLEAN young; } CN1NurseryBlockMeta; static CN1NurseryBlockMeta* cn1NurseryBlocks = 0; +// OBJECT-START BITMAP, one bit per 16-byte granule of the arena. It exists so a +// CONSERVATIVELY found word -- which may point into the MIDDLE of an object -- can be +// resolved back to that object's base. Without it the minor collection can only +// recognise a base pointer, and generated C at -O3 is free to keep an interior pointer +// (an array's data, a strength-reduced field address) while the base dies in a register, +// so a still-live object would go unpromoted and its block be recycled underneath it. +// 16 bytes is the allocator's own alignment below, so one bit per granule is exact. +// Cost is 1/128th of the arena (512KB for the default 64MB) and one OR per allocation. +static unsigned char* cn1NurseryStartBits = 0; +#define CN1_NURSERY_GRANULE 16 +#define CN1_NURSERY_BITS_PER_BLOCK (CN1_NURSERY_BLOCK_SIZE / CN1_NURSERY_GRANULE) static int* cn1NurseryFreeStack = 0; static int cn1NurseryFreeTop = 0; static pthread_mutex_t cn1NurseryMutex = PTHREAD_MUTEX_INITIALIZER; @@ -11830,6 +12688,8 @@ static void cn1NurseryDoInit() { cn1NurseryArenaEnd = cn1NurseryArenaStart + CN1_NURSERY_ARENA_SIZE; cn1NurseryBlockCount = CN1_NURSERY_ARENA_SIZE / CN1_NURSERY_BLOCK_SIZE; cn1NurseryBlocks = (CN1NurseryBlockMeta*)calloc(cn1NurseryBlockCount, sizeof(CN1NurseryBlockMeta)); + cn1NurseryStartBits = (unsigned char*)calloc( + (size_t)CN1_NURSERY_ARENA_SIZE / CN1_NURSERY_GRANULE / 8, 1); cn1NurseryFreeStack = (int*)malloc(sizeof(int) * cn1NurseryBlockCount); for(int i = 0 ; i < cn1NurseryBlockCount ; i++) { cn1NurseryFreeStack[i] = cn1NurseryBlockCount - 1 - i; @@ -11851,6 +12711,13 @@ static int cn1NurseryGrabBlock() { cn1NurseryBlocks[idx].liveCount = 0; cn1NurseryBlocks[idx].tenured = JAVA_FALSE; cn1NurseryBlocks[idx].young = JAVA_TRUE; + // A RECYCLED block still carries the previous occupants' start bits. Left + // behind, they would let the conservative resolver hand back a "base" that + // belongs to a dead layout -- a plausible-looking object whose boundaries have + // since moved, which is the same recycled-slot hazard the BiBOP verifier + // poisons its slots for. + memset(cn1NurseryStartBits + ((size_t)idx * CN1_NURSERY_BITS_PER_BLOCK / 8), + 0, CN1_NURSERY_BITS_PER_BLOCK / 8); } pthread_mutex_unlock(&cn1NurseryMutex); return idx; @@ -11865,6 +12732,15 @@ void cn1NurseryObjectFreed(JAVA_OBJECT o) { int idx = cn1NurseryBlockIndex(o); pthread_mutex_lock(&cn1NurseryMutex); int lc = --cn1NurseryBlocks[idx].liveCount; +#ifdef CN1_NURSERY_NO_RECLAIM + // The ablation has to cover BOTH recycle paths or it does not ablate anything: the + // minor collection returns empty blocks, and this returns a block whose last promoted + // survivor has died. Gating only the first still recycles memory through here, which + // is how a "no reclaim" arm kept reproducing a use-after-free. + (void)lc; + pthread_mutex_unlock(&cn1NurseryMutex); + return; +#endif if(lc <= 0 && cn1NurseryBlocks[idx].tenured && !cn1NurseryBlocks[idx].young) { cn1NurseryBlocks[idx].tenured = JAVA_FALSE; cn1NurseryFreeStack[cn1NurseryFreeTop++] = idx; @@ -11872,12 +12748,22 @@ void cn1NurseryObjectFreed(JAVA_OBJECT o) { pthread_mutex_unlock(&cn1NurseryMutex); } +#ifdef CN1_NURSERY_DEBUG +// PUSHED vs DRAINED. Every promoted object is pushed onto the worklist and must be +// walked exactly once, because walking it is what promotes its children. If these two +// diverge, promotions are being DISCARDED -- which is indistinguishable, from the heap's +// point of view, from the drain never having run. +static long long cn1NurseryPushed = 0, cn1NurseryDrained = 0, cn1NurseryTopResets = 0; +#endif static void cn1PromotePush(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { if(threadStateData->nurseryPromoteTop >= threadStateData->nurseryPromoteCap) { threadStateData->nurseryPromoteCap = threadStateData->nurseryPromoteCap ? threadStateData->nurseryPromoteCap * 2 : 8192; threadStateData->nurseryPromoteWorklist = (JAVA_OBJECT*)realloc(threadStateData->nurseryPromoteWorklist, sizeof(JAVA_OBJECT) * threadStateData->nurseryPromoteCap); } threadStateData->nurseryPromoteWorklist[threadStateData->nurseryPromoteTop++] = o; +#ifdef CN1_NURSERY_DEBUG + cn1NurseryPushed++; +#endif } // Add an object to this thread's pending-allocation buffer, exactly like a normal @@ -11919,6 +12805,9 @@ void cn1NurseryPromote(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT o) { static void cn1PromoteDrain(CODENAME_ONE_THREAD_STATE) { while(threadStateData->nurseryPromoteTop > 0) { JAVA_OBJECT o = threadStateData->nurseryPromoteWorklist[--threadStateData->nurseryPromoteTop]; +#ifdef CN1_NURSERY_DEBUG + cn1NurseryDrained++; +#endif gcMarkFunctionPointer fp = o->__codenameOneParentClsReference->markFunction; if(fp != 0) { fp(threadStateData, o, JAVA_FALSE); @@ -11926,9 +12815,297 @@ static void cn1PromoteDrain(CODENAME_ONE_THREAD_STATE) { } } +// Resolve a CONSERVATIVELY found word to the base of the nursery object that contains +// it, or JAVA_NULL. Interior pointers are the whole reason this exists -- see the +// bitmap declaration. The backward scan is bounded by CN1_NURSERY_MAX_OBJECT, which is +// the largest object the nursery will ever hold, so this is O(32) and not O(block). +// +// A word pointing into a block's unallocated tail resolves to the last object before +// it, which is a FALSE POSITIVE: it promotes an object that may be dead. That is the +// correct direction for a conservative collector -- retaining garbage costs a block, +// mistaking a live object for garbage costs the process. +static JAVA_OBJECT cn1NurseryResolveInterior(void* w) { + if(!cn1InNursery(w)) { + return JAVA_NULL; + } + size_t off = (size_t)((char*)w - cn1NurseryArenaStart); + size_t g = off / CN1_NURSERY_GRANULE; + size_t blockFirstG = (off / CN1_NURSERY_BLOCK_SIZE) * CN1_NURSERY_BITS_PER_BLOCK; + size_t back = (CN1_NURSERY_MAX_OBJECT / CN1_NURSERY_GRANULE) + 1; + // Lowest granule this scan may look at, computed by SUBTRACTING FROM g rather than + // testing `g - i >= blockFirstG` in the loop condition: these are size_t, so once i + // passes g that difference underflows to a huge value and the test is always true -- + // an unsigned-underflow walk off the front of the bitmap. + size_t stopG = blockFirstG; + size_t gg; + if(g - blockFirstG > back) { + stopG = g - back; + } + for(gg = g ; ; gg--) { + if(cn1NurseryStartBits[gg >> 3] & (unsigned char)(1u << (gg & 7))) { + JAVA_OBJECT o = (JAVA_OBJECT)(cn1NurseryArenaStart + gg * CN1_NURSERY_GRANULE); + // -1 is "still in the nursery". Anything else has already been promoted and + // is owned by the global collector, which needs no help from us. + if(o->__heapPosition == -1 && o->__codenameOneParentClsReference != 0) { + return o; + } + return JAVA_NULL; + } + if(gg == stopG) { + break; + } + } + return JAVA_NULL; +} + +// THE ROOT SET THIS COLLECTOR WAS MISSING. +// +// The minor collection below scans threadObjectStack, which is the PRECISE root set. +// That was complete when the nursery was written and has not been since: frameless +// object/instance codegen is default-on (cn1_globals.h, PHASE 3b), and it exists +// precisely so a reference does NOT have to be pushed onto threadObjectStack -- the +// concurrent collector finds it by scanning the C stack conservatively instead. So on a +// default build the precise stack is largely EMPTY, the minor collection promotes almost +// nothing, and it then recycles blocks holding objects the mutator still has in hand. +// +// Measured, because this is the kind of claim that should not be argued: translating the +// self-hosting corpus SIGSEGVs after ONE minor collection with frameless codegen on, and +// runs to a clean Java-level result with it off +// (CN1_SELFHOST_JAVA_OPTS=-Dcn1.frameless.objects=false -Dcn1.frameless.instance=false). +// +// Scanning our own stack needs no signal and no stop: this runs ON the mutator thread, +// from inside its own allocation path, so [current frame, stack base) is exactly the +// region that can hold its live references. setjmp flushes the callee-saved registers +// into a buffer ON that frame, so scanning from the buffer upward covers registers and +// stack in one range -- the same trick cn1GcScanThreadNativeStack uses for other threads. +#ifdef CN1_NURSERY_DEBUG +// SELF-COUNT. Both early returns in this function are silent, and a scan that bails on a +// stack bound it could not resolve looks exactly like a scan that found nothing -- which +// would make the root fix it implements untestable. Reported per minor collection. +static long long cn1NurseryScanWords = 0, cn1NurseryScanFound = 0; +#endif +__attribute__((no_sanitize("address"))) +static void cn1NurseryScanNativeStack(CODENAME_ONE_THREAD_STATE) { + jmp_buf regs; + size_t ssz = 0; + char* hi; + char* lo; + char* p; + // The return value is irrelevant; setjmp is called for its register flush alone. + (void)CN1_TRY_SETJMP(regs); + hi = cn1GcStackBase(pthread_self(), &ssz); + if(hi == 0) { + return; + } + lo = (char*)®s; + if(lo >= hi) { + // A stack that does not contain our own frame is one we cannot reason about + // (an alternate signal stack, a platform whose introspection lied). Scanning a + // wrong range would resolve arbitrary memory as objects, so scan nothing: the + // precise walk below still runs and the arena simply retains more. + return; + } + p = (char*)(((uintptr_t)lo + (sizeof(void*) - 1)) & ~((uintptr_t)(sizeof(void*) - 1))); + for(; p + sizeof(void*) <= hi ; p += sizeof(void*)) { + JAVA_OBJECT o = cn1NurseryResolveInterior(*(void**)p); +#ifdef CN1_NURSERY_DEBUG + cn1NurseryScanWords++; +#endif + if(o != JAVA_NULL) { +#ifdef CN1_NURSERY_DEBUG + cn1NurseryScanFound++; +#endif + cn1NurseryPromote(threadStateData, o); + } + } +} + +#ifdef CN1_NURSERY_POISON +// Stable small id per clazz, assigned on first sight and announced on stderr, so a +// poisoned-pointer fault address decodes back to a class name after the fact. +static struct clazz* cn1NurseryPoisonLegend[4096]; +static int cn1NurseryPoisonLegendCount = 0; +static int cn1NurseryPoisonId(struct clazz* c) { + int i; + for(i = 0 ; i < cn1NurseryPoisonLegendCount ; i++) { + if(cn1NurseryPoisonLegend[i] == c) { + return i; + } + } + if(cn1NurseryPoisonLegendCount >= 4096) { + return 4095; + } + i = cn1NurseryPoisonLegendCount++; + cn1NurseryPoisonLegend[i] = c; + fprintf(stderr, "[NURSERY-POISON] idx=%d class=%s\n", i, + (c != 0 && c->clsName != 0) ? c->clsName : "?"); + fflush(stderr); + return i; +} +#endif + +#ifdef CN1_NURSERY_FINDREF +// THE INVERSE OF THE INVARIANT VERIFIER, and the tool that actually answers the question. +// +// The verifier asks "does any OLD object reference a young one?" and has answered no, +// repeatedly, while the heap was demonstrably being corrupted. That only rules out one +// holder. This asks the question the other way round: promotion has finished, so every +// remaining unpromoted object in a retiring block is DEAD -- now scan every place a +// pointer can live and report anything still pointing at one. +// +// Each region is reported separately because the region IS the diagnosis: +// STACK -> the conservative root scan has a gap +// BIBOP / LEGACY -> a store took no write barrier (and the verifier missed it) +// NURSERY -> the promotion walk failed to follow a field +static void cn1NurseryFindRefs(CODENAME_ONE_THREAD_STATE) { + long stackHits = 0, bibopHits = 0, legacyHits = 0, nurseryHits = 0; + struct clazz* firstHolder = 0; struct clazz* firstTarget = 0; const char* firstRegion = "?"; + // The three facts that separate "the drain skipped this field" from "the scanner is + // reading padding": where in the holder the word sits, how big the holder is, and + // whether the holder even HAS a mark function for the drain to have called. + int firstHolderPos = -999, firstHolderHasMark = -1, firstOffset = -1, firstSpan = -1; + // --- this thread's C stack + registers --- + { + jmp_buf regs; size_t ssz = 0; char* hi; char* lo; char* q; + (void)CN1_TRY_SETJMP(regs); + hi = cn1GcStackBase(pthread_self(), &ssz); + lo = (char*)®s; + if(hi != 0 && lo < hi) { + q = (char*)(((uintptr_t)lo + 7) & ~(uintptr_t)7); + for(; q + sizeof(void*) <= hi ; q += sizeof(void*)) { + JAVA_OBJECT o = cn1NurseryResolveInterior(*(void**)q); + if(o != JAVA_NULL) { + stackHits++; + if(firstTarget == 0) { firstTarget = o->__codenameOneParentClsReference; + firstRegion = "STACK"; } + } + } + } + } + // --- every BiBOP slot and every legacy object, field by field, via a scan of their + // raw words. Raw words rather than mark functions on purpose: a field that took + // no write barrier is exactly the case a mark function might also not describe. + { + CN1BibopPage* pg = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(pg != 0) { + int n = atomic_load_explicit(&pg->bumpIndex, memory_order_acquire); + int i; + for(i = 0 ; i < n ; i++) { + JAVA_OBJECT o = cn1BibopSlot(pg, i); + char* q = (char*)o; char* e = q + pg->slotSize; + if(__atomic_load_n(&o->__codenameOneGcMark, __ATOMIC_ACQUIRE) == CN1_BIBOP_FREE_MARK) continue; + for(; q + sizeof(void*) <= e ; q += sizeof(void*)) { + JAVA_OBJECT t = cn1NurseryResolveInterior(*(void**)q); + if(t != JAVA_NULL) { + bibopHits++; + if(firstTarget == 0) { firstHolder = o->__codenameOneParentClsReference; + firstTarget = t->__codenameOneParentClsReference; + firstRegion = "BIBOP"; } + } + } + } + pg = atomic_load_explicit(&pg->nextAll, memory_order_acquire); + } + } + { + int t2 = currentSizeOfAllObjectsInHeap, i; + for(i = 0 ; i < t2 ; i++) { + JAVA_OBJECT o = allObjectsInHeap[i]; + size_t sz; + if(o == JAVA_NULL || o->__heapPosition == CN1_BIBOP_ADOPTED) continue; + sz = malloc_size((void*)o); + if(sz == 0 || sz > (size_t)(64*1024)) continue; + { + char* q = (char*)o; char* e = q + sz; + for(; q + sizeof(void*) <= e ; q += sizeof(void*)) { + JAVA_OBJECT t = cn1NurseryResolveInterior(*(void**)q); + if(t != JAVA_NULL) { + legacyHits++; + if(firstTarget == 0) { firstHolder = o->__codenameOneParentClsReference; + firstTarget = t->__codenameOneParentClsReference; + firstRegion = "LEGACY"; } + } + } + } + } + } + // --- PROMOTED nursery objects (they are in no other index until the next mark) --- + { + int b; + for(b = 0 ; b < cn1NurseryBlockCount ; b++) { + size_t g0, g1, g; + if(!cn1NurseryBlocks[b].tenured && !cn1NurseryBlocks[b].young) continue; + g0 = (size_t)b * CN1_NURSERY_BITS_PER_BLOCK; g1 = g0 + CN1_NURSERY_BITS_PER_BLOCK; + for(g = g0 ; g < g1 ; g++) { + JAVA_OBJECT o; + if(!(cn1NurseryStartBits[g >> 3] & (unsigned char)(1u << (g & 7)))) continue; + o = (JAVA_OBJECT)(cn1NurseryArenaStart + g * CN1_NURSERY_GRANULE); + if(o->__heapPosition == -1) continue; // dead itself; not a holder + { + // BOUND BY THE NEXT OBJECT START, not by CN1_NURSERY_MAX_OBJECT. + // Objects are packed, so a fixed 512-byte window runs off the end of + // a small object and reads its NEIGHBOURS -- which are quite likely + // the dead ones. That inflates the count and can attribute a + // reference to an object that never held it. The next set start bit + // is the exact end of this object. + size_t gEnd = g + 1; + char* q; char* e; + while(gEnd < g1 && + !(cn1NurseryStartBits[gEnd >> 3] & (unsigned char)(1u << (gEnd & 7)))) { + gEnd++; + } + q = (char*)o; + e = cn1NurseryArenaStart + gEnd * CN1_NURSERY_GRANULE; + for(; q + sizeof(void*) <= e ; q += sizeof(void*)) { + JAVA_OBJECT t = cn1NurseryResolveInterior(*(void**)q); + if(t != JAVA_NULL) { + nurseryHits++; + if(firstTarget == 0) { firstHolder = o->__codenameOneParentClsReference; + firstTarget = t->__codenameOneParentClsReference; + firstRegion = "NURSERY"; + firstHolderPos = o->__heapPosition; + firstHolderHasMark = + (o->__codenameOneParentClsReference != 0 + && o->__codenameOneParentClsReference->markFunction != 0); + firstOffset = (int)(q - (char*)o); + firstSpan = (int)(e - (char*)o); } + } + } + } + } + } + } + if(stackHits | bibopHits | legacyHits | nurseryHits) { + fprintf(stderr, "[NURSERY-FINDREF] DANGLING refs to dead young objects: " + "stack=%ld bibop=%ld legacy=%ld nursery=%ld | first in %s: holder=%s " + "(heapPos=%d markFn=%d) +%d of %d -> %s\n", + stackHits, bibopHits, legacyHits, nurseryHits, firstRegion, + (firstHolder && firstHolder->clsName) ? firstHolder->clsName : "(root)", + firstHolderPos, firstHolderHasMark, firstOffset, firstSpan, + (firstTarget && firstTarget->clsName) ? firstTarget->clsName : "?"); + fflush(stderr); + } +} +#endif + +static int cn1NurseryStaticScanDisabled(void) { + static int cached = -1; + if(cached < 0) { + const char* v = getenv("CN1_NURSERY_NO_STATIC_SCAN"); + cached = (v != 0 && v[0] != '0') ? 1 : 0; + } + return cached; +} void cn1NurseryMinorCollect(CODENAME_ONE_THREAD_STATE) { threadStateData->nurseryPromoting = JAVA_TRUE; +#ifdef CN1_NURSERY_DEBUG + if(threadStateData->nurseryPromoteTop != 0) { cn1NurseryTopResets++; } +#endif threadStateData->nurseryPromoteTop = 0; + // BEFORE the precise walk. Both feed the same promote worklist and promotion is + // idempotent (cn1NurseryPromote flips heapPosition off -1), so the order only + // decides which pass claims a given object first. + cn1NurseryScanNativeStack(threadStateData); int top = threadStateData->threadObjectStackOffset; struct elementStruct* stack = threadStateData->threadObjectStack; for(int i = 0 ; i < top ; i++) { @@ -11948,20 +13125,198 @@ void cn1NurseryMinorCollect(CODENAME_ONE_THREAD_STATE) { // promotion hook ignores -- but it also catches any store path that bypassed the // barrier, so a still-live nursery object can never be left unpromoted (and then // wrongly reclaimed). markStatics calls gcMarkObject, which promotes in this mode. - extern void markStatics(CODENAME_ONE_THREAD_STATE); - markStatics(threadStateData); + // markStatics IS A SAFETY NET, AND IT IS THE MOST EXPENSIVE THING IN A MINOR + // COLLECTION: it walks every static field in the program, every time, and minor + // collections are frequent by design. + // + // A static can no longer hold a young reference. A store into a static compiles to + // CN1_WRITE_BARRIER(JAVA_NULL, value); the barrier tests the TARGET with + // cn1IsYoungObject, JAVA_NULL is not young, so the value is promoted at the point of + // the store. The walk therefore finds nothing on the barrier-covered path and exists + // only to cover a store that bypassed the barrier entirely. + // + // Kept on by default -- it is a correctness backstop for exactly the class of bug + // this collector has been full of -- with CN1_NURSERY_NO_STATIC_SCAN to measure what + // it costs. + if(!cn1NurseryStaticScanDisabled()) { + extern void markStatics(CODENAME_ONE_THREAD_STATE); + markStatics(threadStateData); + } cn1PromoteDrain(threadStateData); threadStateData->nurseryPromoting = JAVA_FALSE; +#ifdef CN1_NURSERY_VERIFY + // THE GENERATIONAL INVARIANT, CHECKED RATHER THAN ASSERTED IN A COMMENT. + // + // Promotion is complete at this point, so nothing outside the young generation may + // still reference something inside it. Walk every object the global collector knows + // about and re-run its mark function in reporting mode; any young referent names a + // store that failed to take the write barrier. Runs before the blocks retire, so the + // referent's class is still readable. + // + // O(registered objects) per minor collection: a QA build only, never shipped. + { + static long long __vPasses = 0, __vScanned = 0; + threadStateData->nurseryVerifying = JAVA_TRUE; + // BIBOP FIRST, and it is the half that matters: a BiBOP object is deliberately + // absent from allObjectsInHeap (that is the point of the page heap), so a walk + // of the legacy table alone reports every BiBOP holder clean. The first version + // of this verifier did exactly that and printed zero violations against a heap + // that was demonstrably corrupt. + { + CN1BibopPage* __p = atomic_load_explicit(&bibopAllPages, memory_order_acquire); + while(__p != 0) { + int __n = atomic_load_explicit(&__p->bumpIndex, memory_order_acquire); + int __i; + for(__i = 0 ; __i < __n ; __i++) { + JAVA_OBJECT __o = cn1BibopSlot(__p, __i); + int __m = __atomic_load_n(&__o->__codenameOneGcMark, __ATOMIC_ACQUIRE); + if(__m == CN1_BIBOP_FREE_MARK || __o->__codenameOneParentClsReference == 0) { + continue; + } + gcMarkFunctionPointer __fp = + __o->__codenameOneParentClsReference->markFunction; + if(__fp != 0) { + threadStateData->nurseryVerifyHolder = __o; + __vScanned++; + __fp(threadStateData, __o, JAVA_FALSE); + } + } + __p = atomic_load_explicit(&__p->nextAll, memory_order_acquire); + } + } + int __t = currentSizeOfAllObjectsInHeap; + int __i; + for(__i = 0 ; __i < __t ; __i++) { + JAVA_OBJECT __o = allObjectsInHeap[__i]; + if(__o == JAVA_NULL || __o->__codenameOneParentClsReference == 0) { + continue; + } + gcMarkFunctionPointer __fp = __o->__codenameOneParentClsReference->markFunction; + if(__fp != 0) { + threadStateData->nurseryVerifyHolder = __o; + __vScanned++; + __fp(threadStateData, __o, JAVA_FALSE); + } + } + // PROMOTED-BUT-NOT-YET-REGISTERED objects. A promotion hands the object to + // cn1AddPending, and it only reaches allObjectsInHeap at the next paused mark -- + // so everything promoted by the pass that just ran is in NEITHER of the two walks + // above. Those are precisely the objects most likely to hold a young referent, + // which made this omission the difference between a verifier that reports the + // violation and one that reports a confident zero. Reached through the + // object-start bitmap, because a pending object has no other index. + for(__i = 0 ; __i < cn1NurseryBlockCount ; __i++) { + if(!cn1NurseryBlocks[__i].tenured) { + continue; + } + size_t __b = (size_t)__i * CN1_NURSERY_BITS_PER_BLOCK; + size_t __e = __b + CN1_NURSERY_BITS_PER_BLOCK; + size_t __g; + for(__g = __b ; __g < __e ; __g++) { + if(cn1NurseryStartBits[__g >> 3] & (unsigned char)(1u << (__g & 7))) { + JAVA_OBJECT __o = (JAVA_OBJECT)(cn1NurseryArenaStart + + __g * CN1_NURSERY_GRANULE); + if(__o->__heapPosition == -1 + || __o->__codenameOneParentClsReference == 0) { + continue; + } + gcMarkFunctionPointer __fp = + __o->__codenameOneParentClsReference->markFunction; + if(__fp != 0) { + threadStateData->nurseryVerifyHolder = __o; + __vScanned++; + __fp(threadStateData, __o, JAVA_FALSE); + } + } + } + } + threadStateData->nurseryVerifyHolder = JAVA_NULL; + threadStateData->nurseryVerifying = JAVA_FALSE; + // SELF-COUNT, because "0 violations" is exactly what a verifier that never ran + // also prints. This line is the difference between a clean result and a vacuous + // one, and it is printed every pass so a run that ends early still says how much + // was actually checked. + __vPasses++; + fprintf(stderr, "[NURSERY-VERIFY] pass=%lld holdersScanned=%lld\n", + __vPasses, __vScanned); + fflush(stderr); + } +#endif // Retire every young block from the young set (under the mutex, so the sweep thread // sees a consistent young flag). A block with no live promoted survivors (liveCount // <= 0: never tenured, or every survivor it held already died) is reclaimed now; // one that still has survivors stays tenured and is freed later by // cn1NurseryObjectFreed when its last survivor dies. Clearing `young` first hands // that responsibility cleanly to the sweep with no double-push window. +#ifdef CN1_NURSERY_FINDREF + cn1NurseryFindRefs(threadStateData); +#endif +#ifdef CN1_NURSERY_PROMOTE_ALL + // ABLATION, not a mode anyone should ship: promote EVERY object in every retiring + // block, reachable or not. It answers exactly one question -- is the remaining defect + // a REACHABILITY gap (some root the promotion walk never visits) or a defect in the + // promotion/registration machinery itself? With this on, no live object can possibly + // be left behind, so a surviving failure cannot be a missed root. + { + threadStateData->nurseryPromoting = JAVA_TRUE; + threadStateData->nurseryPromoteTop = 0; + for(int i = 0 ; i < threadStateData->nurseryYoungCount ; i++) { + size_t __b = (size_t)threadStateData->nurseryYoungBlocks[i] + * CN1_NURSERY_BITS_PER_BLOCK; + size_t __e = __b + CN1_NURSERY_BITS_PER_BLOCK; + size_t __g; + for(__g = __b ; __g < __e ; __g++) { + if(cn1NurseryStartBits[__g >> 3] & (unsigned char)(1u << (__g & 7))) { + JAVA_OBJECT __o = (JAVA_OBJECT)(cn1NurseryArenaStart + + __g * CN1_NURSERY_GRANULE); + if(__o->__heapPosition == -1) { + cn1NurseryPromote(threadStateData, __o); + } + } + } + } + cn1PromoteDrain(threadStateData); + threadStateData->nurseryPromoting = JAVA_FALSE; + } +#endif pthread_mutex_lock(&cn1NurseryMutex); for(int i = 0 ; i < threadStateData->nurseryYoungCount ; i++) { int idx = threadStateData->nurseryYoungBlocks[i]; cn1NurseryBlocks[idx].young = JAVA_FALSE; +#ifdef CN1_NURSERY_POISON + // QA ONLY. Every object in a retiring block that was NOT promoted has just been + // declared dead by this collection. If any root was missed, one of them is still + // referenced -- and with the block merely retired (not yet recycled) that + // reference keeps working, so the defect stays invisible until a completely + // unrelated allocation reuses the memory much later. Poisoning the class pointer + // converts that into an immediate fault at a recognisable address, on the + // instruction that actually holds the stale reference, which is what names the + // missing root. Walk order comes from the object-start bitmap, so it needs no + // per-object size. + { + size_t __b = (size_t)idx * CN1_NURSERY_BITS_PER_BLOCK; + size_t __e = __b + CN1_NURSERY_BITS_PER_BLOCK; + size_t __g; + for(__g = __b ; __g < __e ; __g++) { + if(cn1NurseryStartBits[__g >> 3] & (unsigned char)(1u << (__g & 7))) { + JAVA_OBJECT __o = (JAVA_OBJECT)(cn1NurseryArenaStart + + __g * CN1_NURSERY_GRANULE); + if(__o->__heapPosition == -1) { + // ENCODE THE CLASS INTO THE POISON. A flat 0xDEADBEEF proves a + // stale reference exists but says nothing about WHAT was missed, + // and the class is the whole diagnosis -- it names the field or + // container the promotion walk failed to follow. The legend is + // printed once per class as it is assigned, and the faulting + // address then reads back as 0xDEAD. + __o->__codenameOneParentClsReference = + (struct clazz*)(uintptr_t)(0x0000DEAD00000000ULL + | ((unsigned long long)cn1NurseryPoisonId( + __o->__codenameOneParentClsReference) << 16)); + } + } + } + } +#endif #ifndef CN1_NURSERY_NO_RECLAIM if(cn1NurseryBlocks[idx].liveCount <= 0) { cn1NurseryBlocks[idx].tenured = JAVA_FALSE; @@ -11988,6 +13343,11 @@ void cn1NurseryMinorCollect(CODENAME_ONE_THREAD_STATE) { threadStateData->nurseryBypassCountdown = CN1_NURSERY_BYPASS_ALLOCS; } #ifdef CN1_NURSERY_DEBUG + fprintf(stderr, "[NURSERY] worklist pushed=%lld drained=%lld lost=%lld discardingResets=%lld\n", + cn1NurseryPushed, cn1NurseryDrained, cn1NurseryPushed - cn1NurseryDrained, + cn1NurseryTopResets); + fprintf(stderr, "[NURSERY] stackScan words=%lld found=%lld\n", + cn1NurseryScanWords, cn1NurseryScanFound); fprintf(stderr, "[NURSERY] minor: alloc=%d promoted=%d survival=%d%% reprobe=%d -> bypass=%d\n", allocated, promoted, allocated ? (promoted*100/allocated) : 0, threadStateData->nurseryReprobing, threadStateData->nurseryBypass); @@ -12060,13 +13420,97 @@ JAVA_OBJECT cn1NurseryAlloc(CODENAME_ONE_THREAD_STATE, int size, struct clazz* p // publishing store is what orders this write against any reader. o->__codenameOneGcMark = -1; o->__heapPosition = -1; + // START BIT LAST, AND RELEASE-ORDERED. The bitmap is what lets another walker find + // this object -- the major collection's young-root pass reads it while this thread + // may still be allocating -- so publishing the bit before the header is initialised + // would hand that walker an object whose class pointer is still the previous + // occupant's garbage. Setting it last makes "bit set" mean "header complete". + { + size_t __g = (size_t)(((char*)o - cn1NurseryArenaStart) / CN1_NURSERY_GRANULE); + __atomic_fetch_or(&cn1NurseryStartBits[__g >> 3], + (unsigned char)(1u << (__g & 7)), __ATOMIC_RELEASE); + } return o; } +// THE MISSING HALF OF THE GENERATIONAL DESIGN. +// +// Eager promotion on escape guarantees that no OLD object references a YOUNG one, which +// is what lets a minor collection ignore the rest of the heap. The converse direction has +// no mechanism: a live young object routinely references heap objects, and the major +// collector cannot see it to find them. A nursery object is in no BiBOP page and no +// allObjectsInHeap slot, and cn1ConservativeResolve -- which resolves every other root -- +// knows nothing about the arena, so it resolves a stack word pointing at a young object +// to JAVA_NULL. The heap objects that young object holds are then reachable from nothing +// the collector can see, and the sweep frees them underneath it. +// +// So the young generation has to be a ROOT SOURCE for the major mark. Every object in a +// block that is still young has its mark function run in the ordinary (non-promoting) +// mode, which marks its heap children without marking the young object itself -- the +// young object needs no mark, because no sweep looks at it. +// +// CONSERVATIVE BY CONSTRUCTION: it walks every object in the young blocks, not just the +// reachable ones, because reachability within the young generation is what a MINOR +// collection determines and this runs without one. The cost is retaining heap objects +// held by young garbage until the next minor collection, which is one trigger's worth. +void cn1NurseryMarkYoungRoots(CODENAME_ONE_THREAD_STATE, struct ThreadLocalData* owner) { + int i; +#ifdef CN1_NURSERY_DEBUG + static long long __yrPasses = 0, __yrBlocks = 0, __yrObjects = 0; + __yrPasses++; +#endif + if(cn1NurseryStartBits == 0 || owner == 0 || owner->nurseryYoungBlocks == 0) { + return; + } + // OWNER'S BLOCKS ONLY, and the caller must have this thread PAUSED. nurseryYoungBlocks + // is the owner's own list, mutated only by the owner (cn1NurseryAlloc appends, + // cn1NurseryMinorCollect clears it), so reading it while the owner runs would race + // both the list and the blocks it names. + for(i = 0 ; i < owner->nurseryYoungCount ; i++) { + int blk = owner->nurseryYoungBlocks[i]; + size_t b, e, g; + if(blk < 0 || blk >= cn1NurseryBlockCount) { + continue; + } + b = (size_t)blk * CN1_NURSERY_BITS_PER_BLOCK; + e = b + CN1_NURSERY_BITS_PER_BLOCK; + for(g = b ; g < e ; g++) { + JAVA_OBJECT o; + gcMarkFunctionPointer fp; + if(!(__atomic_load_n(&cn1NurseryStartBits[g >> 3], __ATOMIC_ACQUIRE) + & (unsigned char)(1u << (g & 7)))) { + continue; + } + o = (JAVA_OBJECT)(cn1NurseryArenaStart + g * CN1_NURSERY_GRANULE); + if(o->__heapPosition != -1 || o->__codenameOneParentClsReference == 0) { + continue; // promoted objects are registered and marked the normal way + } + fp = o->__codenameOneParentClsReference->markFunction; +#ifdef CN1_NURSERY_DEBUG + __yrObjects++; +#endif + if(fp != 0) { + fp(threadStateData, o, JAVA_FALSE); + } + } +#ifdef CN1_NURSERY_DEBUG + __yrBlocks++; +#endif + } +#ifdef CN1_NURSERY_DEBUG + fprintf(stderr, "[NURSERY] youngRoots pass=%lld blocks=%lld objects=%lld\n", + __yrPasses, __yrBlocks, __yrObjects); + fflush(stderr); +#endif +} + // Write barrier: an object reference is being stored into a non-nursery location, so // the value escapes the thread-local nursery and must be promoted to the global heap. void cn1NurseryWriteBarrier(JAVA_OBJECT target, JAVA_OBJECT value) { - if(value != JAVA_NULL && cn1InNursery(value) && value->__heapPosition == -1 && !cn1InNursery(target)) { + // cn1IsYoungObject on the TARGET, never cn1InNursery: a promoted container is still + // physically inside the arena, and treating that as "young" skips the promotion the + // value needs. See cn1IsYoungObject in cn1_globals.h. + if(value != JAVA_NULL && cn1IsYoungObject(value) && !cn1IsYoungObject(target)) { struct ThreadLocalData* threadStateData = getThreadLocalData(); // Re-entrancy guard: promotion walks markFunctions which can store refs and // re-enter the barrier; the outermost call owns the worklist drain. @@ -12075,6 +13519,9 @@ void cn1NurseryWriteBarrier(JAVA_OBJECT target, JAVA_OBJECT value) { return; } threadStateData->nurseryPromoting = JAVA_TRUE; +#ifdef CN1_NURSERY_DEBUG + if(threadStateData->nurseryPromoteTop != 0) { cn1NurseryTopResets++; } +#endif threadStateData->nurseryPromoteTop = 0; cn1NurseryPromote(threadStateData, value); cn1PromoteDrain(threadStateData); @@ -12377,11 +13824,52 @@ static void gcMarkDrain(CODENAME_ONE_THREAD_STATE) { } } +// Ceiling on a CPU-DERIVED marker count. It applies to both derived branches and to +// neither explicit override: -DCN1_GC_MARK_THREADS is a deliberate choice (the A/B arms +// use it) and is honoured as given. +// +// Four, because that is where the measurement flattens -- 4 markers and 8 markers are +// the same wall clock and the same peak on the self-hosting corpus, so anything above it +// is cost without return. And the cost is not small: gcMarkPoolEnsure creates a +// PERSISTENT helper per marker, each reserving CN1_THREAD_STACK_BYTES (16MB) of stack, +// and every one of them is woken on each collection to contend for the same worklist. +// +// The POSIX branch had this cap and the Windows branch did not. That asymmetry was +// harmless only while the serial default made both branches unreachable; making +// CPU-derived marking the default is what turned it into a real exposure, and on a +// 32- or 64-logical-CPU Windows host it would reserve roughly 480MB or 1GB of stack +// address space for helpers the measurements say do nothing. +#ifndef CN1_GC_MARK_THREAD_CAP +#define CN1_GC_MARK_THREAD_CAP 4 +#endif // Resolve the total number of markers (the GC thread + helper threads). Computed once. static int gcMarkResolveThreadCount() { #ifdef CN1_GC_MARK_THREADS int n = CN1_GC_MARK_THREADS; -#elif 1 +#elif defined(CN1_GC_SERIAL_MARK) + // SERIAL MARKING IS NO LONGER THE DEFAULT. It was pinned here by the isolation + // experiment described below, which ran on 2026-07-03 and whose own note already + // said "Parallel marking was never re-tested after the experiment". Re-tested now, + // and the result is not marginal. + // + // The earlier re-test was run in a configuration where it could not show anything: + // with the occupied-derived pacing lift active the cap never bound, the mutator + // never parked, and mark throughput therefore could not affect wall clock. With the + // pacing bound restored -- which is the shipping configuration -- the mutator spends + // its time waiting for the collector, so mark throughput sets the whole runtime. + // Self-hosting corpus, 3 reps each, byte-identical output at every arm: + // + // markers wall peak + // 1 5.24 / 5.31 / 6.85s 1200-1221MB + // 4 0.99 / 1.07 / 1.12s 1070-1173MB + // 8 0.99 / 1.04 / 1.38s 1083-1152MB + // + // ~5x faster AND ~10% less memory: a shorter cycle means less time for the mutator + // to run ahead, so the two move together rather than trading off. Past 4 markers it + // is flat, which is the diminishing return the earlier rounds also saw. + // + // -DCN1_GC_SERIAL_MARK restores the serial path for A/B or for a platform where the + // parallel one is under suspicion. // NOTE (later): this verdict predates the fixes. The isolation experiment // below ran on 2026-07-03. The SATB write barrier that closes the // concurrent-mark cross-thread race landed 2026-07-05, as did the freed-slot @@ -12406,15 +13894,30 @@ static int gcMarkResolveThreadCount() { // elsewhere in the branch GC changes (nursery / tagged-int / BiBOP sweep). int n = 1; #elif defined(_WIN32) - // no sysconf in the Win32 shim; NUMBER_OF_PROCESSORS is always set on Windows - const char* np = getenv("NUMBER_OF_PROCESSORS"); - long ncpu = np != 0 ? atol(np) : 2; - int n = (int)(ncpu - 1); + // WINDOWS STAYS SERIAL BY DEFAULT, and this is not caution for its own sake -- it is + // the one configuration where making marking parallel was MEASURED to break. + // + // `ParparVM Java Tests (Windows)` / screenshot-capture (arm64) went from green to + // red on the commit that made marking CPU-derived: the translated app reported + // pass=113 fail=24 not-run=54 and emitted 132 of 166 screenshots, i.e. it stopped + // part-way through the suite, while the same job is green on master and was green on + // this branch immediately before. Every arm that validates parallel marking -- + // the GC suite at 1 and 4 markers on arm64 and x64, and every measurement behind the + // defaults above -- runs on POSIX threads. Windows does not: it goes through the + // Win32 pthread SHIM, which the parallel marker had never run on, because this + // branch was hardcoded to one marker. + // + // So the shim is unvalidated for this, not proven broken, and the honest default is + // the behaviour Windows already shipped. -DCN1_GC_MARK_THREADS=N turns it on there + // for whoever debugs the shim; re-enabling it by default needs that Windows job + // green, not a local measurement on another platform. + (void)CN1_GC_MARK_THREAD_CAP; + int n = 1; #else long ncpu = sysconf(_SC_NPROCESSORS_ONLN); int n = (int)(ncpu - 1); - if(n > 4) { - n = 4; + if(n > CN1_GC_MARK_THREAD_CAP) { + n = CN1_GC_MARK_THREAD_CAP; } #endif if(n < 1) { @@ -14774,6 +16277,25 @@ JAVA_OBJECT cloneArray(JAVA_OBJECT array) { } #endif memcpy( (*arr).data, (*src).data, arr->length * byteSize); +#ifdef CN1_NURSERY + // THE NURSERY BARRIER, bypassed here for the same reason as the SATB halves above: + // the memcpy publishes references with no per-element setter, so CN1_WRITE_BARRIER + // never runs. See the matching block in java_lang_System_arraycopy for why an + // unpromoted nursery reference inside a heap array is a use-after-free that only + // shows up once the arena has wrapped. + // + // allocArray can answer from the nursery too, so the destination is tested rather + // than assumed: a nursery-to-nursery clone keeps both ends young and needs nothing. + if(!cls->primitiveType && !cn1IsYoungObject((void*)arr)) { + JAVA_ARRAY_OBJECT* cn1__d = (JAVA_ARRAY_OBJECT*)(*arr).data; + int cn1__i; + for(cn1__i = 0 ; cn1__i < arr->length ; cn1__i++) { + if(cn1__d[cn1__i] != JAVA_NULL) { + cn1NurseryWriteBarrier((JAVA_OBJECT)arr, (JAVA_OBJECT)cn1__d[cn1__i]); + } + } + } +#endif #ifndef CN1_NO_BULK_INSERTION_BARRIER if(cn1__satbReg) { cn1SatbBulkEnd(); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java new file mode 100644 index 00000000000..1a7120a5667 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ArchiveClassScanner.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Collects the native methods declared by every class inside a jar or zip. + * + * Split out of {@link NativeSignatureVerifier} because it is that class's only use + * of {@code java.util.zip}, and it is reachable only from the offline command-line + * entry point that scripts/check-native-signatures.sh drives -- never from a + * translation. Isolating it is what lets the rest of the verifier compile against + * ParparVM's JavaAPI, which has no java.util.zip and cannot gain one: JavaAPI is + * mirrored by Ports/CLDC11, where the package does not belong. + * + * The translator itself never reads an archive. Every caller extracts a jar into a + * directory of class files before invoking it. + */ +final class ArchiveClassScanner { + private ArchiveClassScanner() { + } + + /** + * Entries are visited in sorted order so that two runs over the same archive + * report findings in the same order. + */ + static void collect(File archive, List into) throws IOException { + ZipFile zip = new ZipFile(archive); + try { + List names = new ArrayList(); + for (Enumeration e = zip.entries(); e.hasMoreElements();) { + ZipEntry entry = e.nextElement(); + if (!entry.isDirectory() && entry.getName().endsWith(".class") + && !entry.getName().endsWith("module-info.class")) { + names.add(entry.getName()); + } + } + Collections.sort(names); + for (String name : names) { + InputStream in = zip.getInputStream(zip.getEntry(name)); + try { + NativeSignatureVerifier.collectFromClassBytes(readAll(in), into); + } finally { + in.close(); + } + } + } finally { + zip.close(); + } + } + + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = in.read(buffer)) > 0) { + out.write(buffer, 0, read); + } + return out.toByteArray(); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java index 81cc62e0acf..638b0de938a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeClass.java @@ -656,14 +656,30 @@ public static void addArrayType(String type, int dimenstions) { + // One reusable emit buffer for the whole output pass, reset per class rather + // than reallocated. Parser.writeOutput -> writeFile -> generateCCode is a + // single sequential loop with no executor and one call site, so there is no + // concurrent or re-entrant use to guard against. + // + // This is not micro-tuning. A fresh StringBuilder starts at capacity 16 and + // JavaAPI grows by 1.5x ((len>>1)+len+2), so building N chars allocates about + // 3N chars = 6N bytes in abandoned intermediate arrays. Across 5897 emitted + // files totalling 245MB that is roughly 1.4GB of pure churn, and MEASURED on + // ParparVM the emit phase allocated 2518MB in a single GC cycle against a + // 24MB trigger. Keeping the capacity across classes means the growth series + // runs only until the buffer reaches the largest class, then never again. + private static final StringBuilder EMIT_BUFFER = new StringBuilder(1 << 20); + public String generateCCode(List allClasses) { - StringBuilder b = new StringBuilder(); + StringBuilder b = EMIT_BUFFER; + b.setLength(0); b.append("#include \""); b.append(clsName); b.append(".h\"\n"); + for(String s : dependsClassesInterfaces) { if (exportsClassesInterfaces.contains(s)) { continue; @@ -952,6 +968,12 @@ public String generateCCode(List allClasses) { b.append(clsName); b.append("_"); b.append(bf.getFieldName().replace('$', '_')); + // Inline-guard rather than call: the initialiser's own first + // line already returns when the flag is set, so the call was a + // no-op after the first time -- but a CALL, on a path that runs + // per static-field access. MEASURED: __STATIC_INITIALIZER_* was + // 7.2% of mutator self-time, java.util.Iterator's alone 6.26%. + // Safe as an ACQUIRE load now that the flag is release-stored. b.append("() {\n __STATIC_INITIALIZER_"); b.append(bf.getClsName()); if (bf.isVolatile()) { @@ -1027,7 +1049,7 @@ public String generateCCode(List allClasses) { buildInstanceFieldList(fullFieldList); String nullCheck = ""; - if (System.getProperty("fieldNullChecks", "false").equals("true")) { + if (Util.getProperty("fieldNullChecks", "false").equals("true")) { nullCheck = "if(__cn1T == JAVA_NULL){throwException(getThreadLocalData(), __NEW_INSTANCE_java_lang_NullPointerException(getThreadLocalData()));}\n"; } for(ByteCodeField fld : fullFieldList) { @@ -1205,6 +1227,36 @@ public String generateCCode(List allClasses) { b.append(", objInstance->").append(REFERENCE_CLASS).append("_cn1Strength);\n"); continue; } + // TYPE-IDENTITY CHECK, verifier builds only. + // + // CN1_GC_VERIFY already proves every traced reference RESOLVES, which + // is why a reclaimed-and-recycled slot slips past it: the slot holds a + // perfectly valid object, just not the one the field was pointing at. + // A Linux suite core caught the consequence -- ArrayList.add running on + // an object whose class word said charts.compat.Canvas, reading the + // list's backing-array slot out of two of Canvas's int fields. + // + // The field's DECLARED type is known here and thrown away, so the + // collector has no way to notice. Passing it lets the verifier ask + // whether what the field holds is assignable to what it was declared + // as, which is exactly the question a recycled slot answers wrongly -- + // and it names the field, instead of leaving a SIGSEGV in an unrelated + // method a whole cycle later. + // + // Arrays are skipped for now: their id mapping is dimensional and the + // failure this was written for was a plain object field. + // getRuntimeDescriptor() is the mangled type for a plain object field + // and carries "[]" for an array, which is how arrays are excluded. + String fldType = fld.getRuntimeDescriptor(); + if (fldType != null && fldType.indexOf('[') < 0 + && Parser.getClassObject(fldType) != null) { + b.append("#ifdef CN1_GC_VERIFY\n"); + b.append(" cn1GcVerifyFieldType(threadStateData, objToMark, objInstance->"); + b.append(fld.getClsName()).append("_").append(fld.getFieldName()); + b.append(", cn1_class_id_").append(fldType); + b.append(", \"").append(clsName).append(".").append(fld.getFieldName()).append("\");\n"); + b.append("#endif\n"); + } b.append(" gcMarkObject(threadStateData, "); if (fld.isVolatile()) { b.append("atomic_load_explicit(&objInstance->"); @@ -1527,10 +1579,33 @@ public String generateCCode(List allClasses) { } // insert static initializer + // NOT static: the inline guards emitted at allocation and static-access + // sites live in OTHER translation units and have to test COMPLETION. They + // used to test class__X.initialized instead, which is the wrong flag -- + // that one is the JLS recursion guard and is deliberately set BEFORE + // __CLINIT__ runs, so a thread observing it could skip the initialiser + // while another thread was still inside the class initialiser, and then + // read statics that had not been written yet. Releasing on "started" + // cannot publish writes that happen after it. b.append("static int __").append(clsName).append("_LOADED__=0;\n"); b.append("void __STATIC_INITIALIZER_"); b.append(clsName); - b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__").append(clsName).append("_LOADED__) return;\n\n "); + // ACQUIRE, not a plain load. This is the fast path of a double-checked + // initialisation: the completing store below is a RELEASE, and the two + // together are what make the writes this function performed -- the + // vtable, and every classToInterfaceMap_[classId] row -- visible + // to a thread that observes the flag set. + // + // With plain accesses on arm64 a second thread could see LOADED==1 while + // those table stores were still invisible, then index a row that read as + // NULL. OBSERVED: three identical SIGSEGVs at + // classToInterfaceMap_java_util_NavigableMap[classId] + 0x8, reached from + // TreeSet.clear -> the interface dispatch for NavigableMap.clear, in a + // translator that is single-threaded in its own code but shares the + // process with the GC thread, which also runs Java and so also runs + // class initialisers. + b.append("(CODENAME_ONE_THREAD_STATE) {\n if(__atomic_load_n(&__") + .append(clsName).append("_LOADED__, __ATOMIC_ACQUIRE)) return;\n\n "); // Block-registered enter/exit (the synchronized-method pattern): if the @@ -1579,7 +1654,10 @@ public String generateCCode(List allClasses) { b.append(".vtable = initVtableForInterface();\n"); b.append(" classToInterfaceMap_"); b.append(clsName); - b.append(" = malloc(sizeof(int*) * cn1_array_start_offset);\n"); + // calloc, not malloc: rows are filled only for classes that implement + // this interface, so an id that does not read as a registered row must + // read as NULL rather than as whatever the allocator last left there. + b.append(" = calloc(cn1_array_start_offset, sizeof(int*));\n"); for(ByteCodeClass cls : allClasses) { if(!cls.isInterface) { if(cls.doesImplement(this)) { @@ -1616,9 +1694,21 @@ public String generateCCode(List allClasses) { b.append(".vtable);\n"); } - b.append(" class__"); + b.append(" __atomic_store_n(&class__"); b.append(clsName); - b.append(".initialized = JAVA_TRUE;\n"); + // This flag means STARTED, not completed: the JLS requires a class whose + // initialiser re-enters itself to proceed rather than deadlock, so it has + // to be set before __CLINIT__ runs, and the check above the monitor is + // that recursion guard. Nothing outside this function may treat it as + // "safe to use the class" in the JLS sense -- a class under initialization + // is not finished. The release is what the INLINE GUARDS acquire against: + // they test this flag, and it is what publishes the vtable and the + // classToInterfaceMap rows written just above. Guarding them on + // __X_LOADED__ instead would also be correct about the vtable and would + // additionally hold other threads until __CLINIT__ returned -- a strictly + // later gate than master opens, which moved layout on four native ports + // and is not what the visibility defect required. + b.append(".initialized, JAVA_TRUE, __ATOMIC_RELEASE);\n"); // init static fields and invoke the static initializer code block if(clInitMethod != null) { b.append(" "); @@ -1629,7 +1719,10 @@ public String generateCCode(List allClasses) { b.append(clsName); b.append(");\n"); - b.append("__").append(clsName).append("_LOADED__=1;\n"); + // RELEASE: pairs with the acquire on the fast path above, so everything + // this initialiser wrote happens-before another thread's early return. + b.append("__atomic_store_n(&__").append(clsName) + .append("_LOADED__, 1, __ATOMIC_RELEASE);\n"); b.append("}\n\n"); @@ -2000,7 +2093,6 @@ public String generateCHeader() { b.append("extern void __STATIC_INITIALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE);\n"); - b.append("extern void __FINALIZER_"); b.append(clsName); b.append("(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT objToDelete);\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java index 8eac3b6d495..3b31238b03c 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeField.java @@ -41,7 +41,7 @@ public class ByteCodeField { private int arrayDimensions; private String type; - private Class primitiveType; + private PrimitiveType primitiveType; private boolean finalField; private Object value; private boolean privateField; @@ -81,28 +81,28 @@ public ByteCodeField(String clsName, int access, String name, String desc, Strin type = objectType; break; case 'I': - primitiveType = Integer.TYPE; + primitiveType = PrimitiveType.INT; break; case 'J': - primitiveType = Long.TYPE; + primitiveType = PrimitiveType.LONG; break; case 'B': - primitiveType = Byte.TYPE; + primitiveType = PrimitiveType.BYTE; break; case 'S': - primitiveType = Short.TYPE; + primitiveType = PrimitiveType.SHORT; break; case 'F': - primitiveType = Float.TYPE; + primitiveType = PrimitiveType.FLOAT; break; case 'D': - primitiveType = Double.TYPE; + primitiveType = PrimitiveType.DOUBLE; break; case 'Z': - primitiveType = Boolean.TYPE; + primitiveType = PrimitiveType.BOOLEAN; break; case 'C': - primitiveType = Character.TYPE; + primitiveType = PrimitiveType.CHAR; break; } } @@ -211,31 +211,10 @@ public String getRuntimeDescriptor() { if (primitiveType == null) { return type; } - if (primitiveType == Integer.TYPE) { - return "I"; - } - if (primitiveType == Long.TYPE) { - return "J"; - } - if (primitiveType == Byte.TYPE) { - return "B"; - } - if (primitiveType == Short.TYPE) { - return "S"; - } - if (primitiveType == Float.TYPE) { - return "F"; - } - if (primitiveType == Double.TYPE) { - return "D"; - } - if (primitiveType == Boolean.TYPE) { - return "Z"; - } - if (primitiveType == Character.TYPE) { - return "C"; - } - return null; + // A field is never void, so PrimitiveType.VOID's "V" is unreachable here; + // the chain this replaces returned null for it, which no caller handled + // either. + return primitiveType.getDescriptor(); } public boolean isPrivate() { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java index e956745d0cb..b2b961f107e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeMethodArg.java @@ -33,14 +33,14 @@ public class ByteCodeMethodArg { private final int arrayDimensions; private String type; - private Class primitiveType; + private PrimitiveType primitiveType; public ByteCodeMethodArg(String type, int dim) { this.type = type.replace('/', '_').replace('$', '_'); arrayDimensions = dim; } - public ByteCodeMethodArg(Class type, int dim) { + public ByteCodeMethodArg(PrimitiveType type, int dim) { this.primitiveType = type; arrayDimensions = dim; } @@ -49,13 +49,13 @@ public char getQualifier() { if(type != null || arrayDimensions > 0) { return 'o'; } - if(primitiveType == Long.TYPE) { + if(primitiveType == PrimitiveType.LONG) { return 'l'; } - if(primitiveType == Double.TYPE) { + if(primitiveType == PrimitiveType.DOUBLE) { return 'd'; } - if(primitiveType == Float.TYPE) { + if(primitiveType == PrimitiveType.FLOAT) { return 'f'; } return 'i'; @@ -93,7 +93,11 @@ public int hashCode() { if(type != null) { return type.hashCode(); } - return primitiveType.hashCode(); + // ordinal(), not hashCode(): Enum.hashCode is an identity hash on OpenJDK + // and the ordinal in ParparVM's java.lang.Enum, so hashing on it would make + // a hash container of these args iterate in a different order under the + // self-hosted translator than under the JVM-hosted one. + return primitiveType.ordinal(); } @Override @@ -121,11 +125,11 @@ public boolean equals(Object obj) { } public boolean isVoid() { - return primitiveType == Void.TYPE; + return primitiveType == PrimitiveType.VOID; } public boolean isDoubleOrLong() { - return (primitiveType == Double.TYPE || primitiveType == Long.TYPE) && arrayDimensions == 0; + return (primitiveType == PrimitiveType.DOUBLE || primitiveType == PrimitiveType.LONG) && arrayDimensions == 0; } /** @@ -139,7 +143,7 @@ public String getTypeName() { return type; } - public Class getPrimitiveType() { + public PrimitiveType getPrimitiveType() { return primitiveType; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java index 6dc2226a082..5b6e22b1648 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/ByteCodeTranslator.java @@ -25,13 +25,14 @@ import java.io.DataInputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -155,9 +156,9 @@ private static void sortByName(File[] files) { } void execute(File sourceDir, File outputDir) throws Exception { - File[] directoryList = sourceDir.listFiles(pathname -> + File[] directoryList = Util.listFiles(sourceDir, pathname -> !pathname.isHidden() && !pathname.getName().startsWith(".") && pathname.isDirectory()); - File[] fileList = sourceDir.listFiles(pathname -> + File[] fileList = Util.listFiles(sourceDir, pathname -> !pathname.isHidden() && !pathname.getName().startsWith(".") && !pathname.isDirectory()); // listFiles() returns whatever order the filesystem hands back, which can // differ between two builds of the same input (the app classes are @@ -179,7 +180,7 @@ void execute(File sourceDir, File outputDir) throws Exception { } else { if(!f.isDirectory() && !isBuildMetadata(f)) { // copy the file to the dest dir - copy(Files.newInputStream(f.toPath()), Files.newOutputStream(new File(outputDir, f.getName()).toPath())); + copy(new FileInputStream(f), new FileOutputStream(new File(outputDir, f.getName()))); // Everything that reaches here is hand-written: a port native, a // cn1lib's native, or an application resource. This is the only // point at which its ORIGIN is still known -- one line further on @@ -212,7 +213,7 @@ private void copyDir(File source, File destDir) throws IOException { if(f.isDirectory()) { copyDir(f, destFile); } else { - copy(Files.newInputStream(f.toPath()), Files.newOutputStream(new File(destFile, f.getName()).toPath())); + copy(new FileInputStream(f), new FileOutputStream(new File(destFile, f.getName()))); } } } @@ -226,7 +227,7 @@ private void copyDir(File source, File destDir) throws IOException { * engine compiled in. Set by the platform builders from their class scan. */ static boolean isBundledSqliteEnabled() { - return "true".equals(System.getProperty("cn1.sqlite", "false")); + return "true".equals(Util.getProperty("cn1.sqlite", "false")); } /** @@ -234,7 +235,7 @@ static boolean isBundledSqliteEnabled() { * system libsqlite3, which has no cipher support, with the bundled engine. */ static boolean isBundledSqliteCipherEnabled() { - return "true".equals(System.getProperty("cn1.sqlcipher", "false")); + return "true".equals(Util.getProperty("cn1.sqlcipher", "false")); } /** @@ -267,7 +268,7 @@ static boolean isBundledSqliteCipherEnabled() { * shipping target. */ public static boolean isCheckedCastsEnabled() { - return "true".equalsIgnoreCase(System.getProperty("cn1.checkedCasts", "false")); + return "true".equalsIgnoreCase(Util.getProperty("cn1.checkedCasts", "false")); } /// Writes the bundled SQLite engine into a source root, or takes it back out. @@ -314,7 +315,7 @@ private static File copyRuntimeResource(File srcRoot, String name) throws IOExce */ private static File copyRuntimeResource(File srcRoot, String name, String destName) throws IOException { File dest = new File(srcRoot, destName); - copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), Files.newOutputStream(dest.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), new FileOutputStream(dest)); sourceManifest.recordRuntime(destName, "/" + name); return dest; } @@ -331,7 +332,7 @@ private static File copyRuntimeResource(File srcRoot, String name, String destNa */ private static File copyVendoredResource(File srcRoot, String name) throws IOException { File dest = new File(srcRoot, name); - copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), Files.newOutputStream(dest.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/" + name), new FileOutputStream(dest)); sourceManifest.recordVendored(name, "/" + name); return dest; } @@ -413,7 +414,7 @@ public static void main(String[] args) throws Exception { final String appType = args[7]; final String addFrameworks = args[8]; // we accept 3 argument output types, input directory and output directory - if (System.getProperty("saveUnitTests", "false").equals("true")) { + if (Util.getProperty("saveUnitTests", "false").equals("true")) { System.out.println("Generating Unit Tests"); ByteCodeClass.setSaveUnitTests(true); } @@ -430,7 +431,7 @@ public static void main(String[] args) throws Exception { // Unrecognized output type falls back to the plain copy-through default handler recognizedOutputType = false; } - String[] sourceDirectories = args[1].split(";"); + String[] sourceDirectories = Util.splitLiteral(args[1], ';'); File[] sources = new File[sourceDirectories.length]; for(int iter = 0 ; iter < sourceDirectories.length ; iter++) { sources[iter] = new File(sourceDirectories[iter]); @@ -514,15 +515,15 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File // generated. A project that gets the C and not the .S links against a // missing symbol, which is at least loud. emitVirtualThreadRuntime(srcRoot); - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } - if ("true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false"))) { + if ("true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false"))) { replaceInFile(cn1Globals, "//#define CN1_ON_DEVICE_DEBUG", "#define CN1_ON_DEVICE_DEBUG"); } copyRuntimeResource(srcRoot, "cn1_globals.m", "cn1_globals.c"); copyRuntimeResource(srcRoot, "nativeMethods.m", "nativeMethods.c"); - if (System.getProperty("USE_RPMALLOC", "false").equals("true")) { + if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { copyRuntimeResource(srcRoot, "malloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.h"); @@ -554,7 +555,7 @@ private static void handleCleanOutput(ByteCodeTranslator b, File[] sources, File File classMethodIndexM = new File(srcRoot, "cn1_class_method_index.m"); if (classMethodIndexM.exists()) { File classMethodIndexC = new File(srcRoot, "cn1_class_method_index.c"); - copy(Files.newInputStream(classMethodIndexM.toPath()), Files.newOutputStream(classMethodIndexC.toPath())); + copy(new FileInputStream(classMethodIndexM), new FileOutputStream(classMethodIndexC)); if(!classMethodIndexM.delete()) { System.err.println("Deletion of " + classMethodIndexM.getAbsolutePath() + " failed"); } @@ -622,7 +623,14 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I StringBuilder table = new StringBuilder(); table.append("/* Auto-generated by the ParparVM windows target: maps a classpath\n"); table.append(" * resource path to the RCDATA id embedded in the executable. */\n"); - table.append("#include \n\n"); + table.append("#include \n"); + // Behind _WIN32: the windows APP TYPE is compiled on a Linux host by + // CleanTargetIntegrationTest#generatesRunnableExecutableForWindowsAppType, + // where windows.h does not exist. The id table below is plain C and stays + // unguarded; only the resource-resolving override needs the platform. + table.append("#if defined(_WIN32)\n"); + table.append("#include \n"); + table.append("#endif\n\n"); table.append("typedef struct { const char* name; int id; } CN1ResourceEntry;\n\n"); table.append("static const CN1ResourceEntry cn1ResourceTable[] = {\n"); @@ -634,7 +642,7 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I int id = 1; for (java.util.Map.Entry e : resources.entrySet()) { File staged = new File(resDir, "res" + id); - copy(Files.newInputStream(e.getValue().toPath()), Files.newOutputStream(staged.toPath())); + copy(new FileInputStream(e.getValue()), new FileOutputStream(staged)); // RC filenames are resolved relative to the .rc (srcRoot); llvm-rc and // rc.exe both accept forward slashes. rc.append(id).append(" RCDATA \"cn1_resources/res").append(id).append("\"\n"); @@ -642,7 +650,7 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I id++; } sourceManifest.recordGenerated("cn1_resources.rc"); - Files.write(new File(srcRoot, "cn1_resources.rc").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources.rc"), rc.toString().getBytes(StandardCharsets.UTF_8)); } @@ -654,9 +662,14 @@ private static void embedWindowsResources(File[] sources, File srcRoot) throws I table.append(" if (strcmp(cn1ResourceTable[i].name, name) == 0) { return cn1ResourceTable[i].id; }\n"); table.append(" }\n"); table.append(" return 0;\n"); - table.append("}\n"); + table.append("}\n\n"); + // No cn1FindResource override is emitted. The id table above is built and + // linked, and nothing reads it: Class.getResourceAsStream deliberately does + // not consult embedded resources, because doing so changed how shipping + // applications render (see the comment there). Wiring these together is the + // whole of that future change. sourceManifest.recordGenerated("cn1_resources_table.c"); - Files.write(new File(srcRoot, "cn1_resources_table.c").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); } @@ -705,7 +718,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE int id = 1; for (java.util.Map.Entry e : resources.entrySet()) { File staged = new File(resDir, "res" + id); - copy(Files.newInputStream(e.getValue().toPath()), Files.newOutputStream(staged.toPath())); + copy(new FileInputStream(e.getValue()), new FileOutputStream(staged)); // Absolute path so .incbin resolves regardless of the assembler's // working directory (the build runs out of a separate build dir). String incPath = escapeCString(staged.getAbsolutePath().replace('\\', '/')); @@ -720,7 +733,7 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE id++; } sourceManifest.recordGenerated("cn1_resources_data.S"); - Files.write(new File(srcRoot, "cn1_resources_data.S").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_data.S"), asm.toString().getBytes(StandardCharsets.UTF_8)); } @@ -744,9 +757,14 @@ private static void embedLinuxResources(File[] sources, File srcRoot) throws IOE table.append(" }\n"); table.append(" if (lenOut) { *lenOut = 0; }\n"); table.append(" return 0;\n"); - table.append("}\n"); + table.append("}\n\n"); + // No cn1FindResource override is emitted. The id table above is built and + // linked, and nothing reads it: Class.getResourceAsStream deliberately does + // not consult embedded resources, because doing so changed how shipping + // applications render (see the comment there). Wiring these together is the + // whole of that future change. sourceManifest.recordGenerated("cn1_resources_table.c"); - Files.write(new File(srcRoot, "cn1_resources_table.c").toPath(), + Util.writeBytes(new File(srcRoot, "cn1_resources_table.c"), table.toString().getBytes(StandardCharsets.UTF_8)); } @@ -775,7 +793,19 @@ private static void collectResources(File root, File dir, java.util.LinkedHashMa || ext.equals("mm") || ext.equals("rc")) { continue; } - String rel = root.toPath().relativize(f.toPath()).toString().replace('\\', '/'); + // Relative path by absolute-prefix strip rather than Path.relativize: + // JavaAPI has no java.nio.file, and the translator compiles against it + // when it translates itself. f is always under root here -- it came from + // a walk of root -- so the prefix always matches. + String rootAbs = root.getAbsolutePath(); + String fileAbs = f.getAbsolutePath(); + String rel = fileAbs.startsWith(rootAbs) + ? fileAbs.substring(rootAbs.length()) + : fileAbs; + while (rel.startsWith(File.separator) || rel.startsWith("/")) { + rel = rel.substring(1); + } + rel = rel.replace('\\', '/'); String key = "/" + rel; if (!out.containsKey(key)) { out.put(key, f); @@ -836,7 +866,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File launchImageLaunchimage.mkdirs(); //cleanDir(launchImageLaunchimage); - copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), Files.newOutputStream(new File(launchImageLaunchimage, "Contents.json").toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream("/LaunchImages.json"), new FileOutputStream(new File(launchImageLaunchimage, "Contents.json"))); } File appIconAppiconset = new File(imagesXcassets, "AppIcon.appiconset"); @@ -847,7 +877,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File // wants the 16..512 @1x/@2x "mac" idiom ladder. copy(ByteCodeTranslator.class.getResourceAsStream( platform.hasIosDeviceIdioms() ? "/Icons.json" : "/Icons-macos.json"), - Files.newOutputStream(new File(appIconAppiconset, "Contents.json").toPath())); + new FileOutputStream(new File(appIconAppiconset, "Contents.json"))); File xcproj = new File(root, appName + ".xcodeproj"); @@ -866,17 +896,30 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File // generated. A project that gets the C and not the .S links against a // missing symbol, which is at least loud. emitVirtualThreadRuntime(srcRoot); - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { replaceInFile(cn1Globals, "//#define CN1_INCLUDE_NPE_CHECKS", "#define CN1_INCLUDE_NPE_CHECKS"); } - if ("true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false"))) { + if ("true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false"))) { replaceInFile(cn1Globals, "//#define CN1_ON_DEVICE_DEBUG", "#define CN1_ON_DEVICE_DEBUG"); } copyRuntimeResource(srcRoot, "cn1_globals.m"); copyRuntimeResource(srcRoot, "nativeMethods.m"); - copyRuntimeResource(srcRoot, "java_io_File.m"); - - if (System.getProperty("USE_RPMALLOC", "false").equals("true")) { + // java_io_File_RUNTIME.m, not java_io_File.m. When the application retains + // java.io.File -- which the filesystem fallback makes ordinary -- Parser + // .writeOutput emits the translated class to java_io_File.m and overwrites + // the port's hand-written native that was copied here first. The generated + // File.exists() then has no existsImpl to link against. + // + // OBSERVED as `Undefined symbols: _java_io_File_existsImpl ... referenced + // from _java_io_File_exists___R_boolean in java_io_File.o` on the iOS legs. + // The clean target already avoids the same collision by emitting + // java_io_File_runtime.c; this is that fix for the Apple path. The name only + // has to differ from the generated one -- the compiler globs the directory, + // and NativeSignatureVerifier reads the RESOURCE "/java_io_File.m" rather + // than the emitted filename. + copyRuntimeResource(srcRoot, "java_io_File.m", "java_io_File_runtime.m"); + + if (Util.getProperty("USE_RPMALLOC", "false").equals("true")) { copyRuntimeResource(srcRoot, "malloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.c"); copyRuntimeResource(srcRoot, "rpmalloc.h"); @@ -894,23 +937,35 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File Parser.writeOutput(srcRoot); File templateInfoPlist = new File(srcRoot, appName + "-Info.plist"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Info.plist"), Files.newOutputStream(templateInfoPlist.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Info.plist"), new FileOutputStream(templateInfoPlist)); File templatePch = new File(srcRoot, appName + "-Prefix.pch"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Prefix.pch"), Files.newOutputStream(templatePch.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template/template-Prefix.pch"), new FileOutputStream(templatePch)); copyRuntimeResource(srcRoot, "xmlvm.h"); File projectWorkspaceData = new File(projectXCworkspace, "contents.xcworkspacedata"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), Files.newOutputStream(projectWorkspaceData.toPath())); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.xcworkspace/contents.xcworkspacedata"), new FileOutputStream(projectWorkspaceData)); replaceInFile(projectWorkspaceData, "KitchenSink", appName); File projectPbx = new File(xcproj, "project.pbxproj"); - copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), Files.newOutputStream(projectPbx.toPath())); - - String[] sourceFiles = srcRoot.list((pathname, string) -> - string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") || !pathname.isHidden() && !string.startsWith(".") && !"Images.xcassets".equals(string)); + copy(ByteCodeTranslator.class.getResourceAsStream(templateRoot + "/template.xcodeproj/project.pbxproj"), new FileOutputStream(projectPbx)); + + // File.list(FilenameFilter) is not in JavaAPI; filter the plain listing. + String[] allNames = srcRoot.list(); + java.util.List keptNames = new java.util.ArrayList(); + if (allNames != null) { + for (String string : allNames) { + File pathname = new File(srcRoot, string); + if (string.endsWith(".bundle") || string.endsWith(".xcdatamodeld") + || !pathname.isHidden() && !string.startsWith(".") + && !"Images.xcassets".equals(string)) { + keptNames.add(string); + } + } + } + String[] sourceFiles = keptNames.toArray(new String[keptNames.size()]); StringBuilder fileOneEntry = new StringBuilder(); StringBuilder fileTwoEntry = new StringBuilder(); @@ -926,7 +981,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File List includeFrameworks = new ArrayList<>(); Set optionalFrameworks = new HashSet<>(); - for (String optionalFramework : System.getProperty("optional.frameworks", "").split(";")) { + for (String optionalFramework : Util.splitLiteral(Util.getProperty("optional.frameworks", ""), ';')) { optionalFramework = optionalFramework.trim(); if (!optionalFramework.isEmpty()) { optionalFrameworks.add(optionalFramework); @@ -996,7 +1051,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File includeFrameworks.add("libz.dylib"); includeFrameworks.add("AVKit.framework"); if(!addFrameworks.equalsIgnoreCase("none")) { - includeFrameworks.addAll(Arrays.asList(addFrameworks.split(";"))); + includeFrameworks.addAll(Arrays.asList(Util.splitLiteral(addFrameworks, ';'))); } int currentValue = 0xF63EAAA; @@ -1142,7 +1197,7 @@ private static void handleAppleOutput(ByteCodeTranslator b, File[] sources, File "***FRAMEWORKS2***", frameworks2.toString(), "***RESOURCES***", resources.toString()); } - String bundleVersion = System.getProperty("bundleVersionNumber", appVersion); + String bundleVersion = Util.getProperty("bundleVersionNumber", appVersion); replaceInFile(templateInfoPlist, "com.codename1pkg", appPackageName, "${PRODUCT_NAME}", appDisplayName, "VERSION_VALUE", appVersion, "VERSION_BUNDLE_VALUE", bundleVersion); // Written to the project root, NOT to srcRoot. srcRoot.list() above feeds the @@ -1166,7 +1221,7 @@ private static void writeCmakeProject(File projectRoot, File srcRoot, String app boolean windows = "windows".equalsIgnoreCase(appType); boolean linux = "linux".equalsIgnoreCase(appType); boolean executable = windows || linux; - try (Writer writer = new OutputStreamWriter(Files.newOutputStream(cmakeLists.toPath()), StandardCharsets.UTF_8)) { + try (Writer writer = new OutputStreamWriter(new FileOutputStream(cmakeLists), "UTF-8")) { writer.append("cmake_minimum_required(VERSION 3.10)\n"); // The native Windows port mixes the translated C runtime with a C++ // layer for the COM APIs that have no C binding (DirectWrite), so the @@ -1467,6 +1522,18 @@ private static void writeLinuxLinkSet(Writer writer) throws IOException { // binary, where the companion exists to turn an address back into a Java method. // It is the wrong one for a CI build whose whole job is to be autopsied, so the // level is a cache variable: unset it and nothing changes for anybody. + // A diagnostic-only define hook, empty by default so nothing changes for + // anybody who does not ask. The reason it exists: the collector's own + // heap-integrity verifier (-DCN1_GC_VERIFY) is the designed detector for + // "the sweep reclaimed something a retained object still references", and + // there was no way to turn it on for a generated project without editing + // the emitted CMakeLists by hand. Chasing a dangling field reference + // through core dumps is what made that gap expensive. + writer.append("set(CN1_EXTRA_DEFINES \"\" CACHE STRING\n"); + writer.append(" \"Extra preprocessor defines for diagnostic builds, semicolon separated (e.g. CN1_GC_VERIFY)\")\n"); + writer.append("if(CN1_EXTRA_DEFINES)\n"); + writer.append(" target_compile_definitions(${PROJECT_NAME} PRIVATE ${CN1_EXTRA_DEFINES})\n"); + writer.append("endif()\n"); writer.append("set(CN1_DEBUG_INFO_LEVEL \"1\" CACHE STRING\n"); writer.append(" \"DWARF level for the .debug companion: 1 = lines + function names (lean, the default), 3 = full variable and type information (autopsyable)\")\n"); writer.append("target_compile_options(${PROJECT_NAME} PRIVATE -g${CN1_DEBUG_INFO_LEVEL} -fno-asynchronous-unwind-tables -fno-unwind-tables)\n"); @@ -1568,12 +1635,12 @@ private static String getFileType(String s) { // to be mutated. Also, expire the temporary byte[] buffer so it can // be collected. // - private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOException + private static String readFileAsString(File sourceFile) throws IOException { - try(DataInputStream dis = new DataInputStream(Files.newInputStream(sourceFile.toPath()))) { + try(DataInputStream dis = new DataInputStream(new FileInputStream(sourceFile))) { byte[] data = new byte[(int) sourceFile.length()]; dis.readFully(data); - return new StringBuilder(new String(data, StandardCharsets.UTF_8)); + return new String(data, StandardCharsets.UTF_8); } } // @@ -1584,21 +1651,38 @@ private static StringBuilder readFileAsStringBuilder(File sourceFile) throws IOE // process for large projects. // private static void replaceInFile(File sourceFile, String... values) throws IOException { - StringBuilder str = readFileAsStringBuilder(sourceFile); + // A String rather than a StringBuilder because the translator has to compile + // against ParparVM's own JavaAPI in order to translate itself, and + // StringBuilder there has neither indexOf nor replace. + // + // One pass per target, appending into a fresh builder. The obvious + // translation of the old in-place edit -- indexOf on str.toString(), then + // substring/concat the whole buffer back together per match -- copies the + // ENTIRE file twice for every occurrence, which is the opposite of this + // method's purpose: it exists to avoid the memory spike that made large + // Xcode project.pbxproj rewrites fail with OutOfMemoryError. Each target + // now costs one traversal and one output buffer regardless of how many + // times it matches. + String str = readFileAsString(sourceFile); int totchanges = 0; - // perform the mutations on stringbuilder, which ought to implement - // these operations efficiently. for (int iter = 0; iter < values.length; iter += 2) { String target = values[iter]; String replacement = values[iter + 1]; - int index = 0; - while ((index = str.indexOf(target, index)) >= 0) { - int targetSize = target.length(); - str.replace(index, index + targetSize, replacement); - index += replacement.length(); + int index = str.indexOf(target); + if (index < 0) { + continue; + } + StringBuilder out = new StringBuilder(str.length() + 64); + int from = 0; + while (index >= 0) { + out.append(str, from, index).append(replacement); + from = index + target.length(); totchanges++; + index = str.indexOf(target, from); } + out.append(str, from, str.length()); + str = out.toString(); } // @@ -1607,8 +1691,8 @@ private static void replaceInFile(File sourceFile, String... values) throws IOEx if(verbose) { System.out.println("Rewrite " + sourceFile + " with " + totchanges + " changes"); } - try(Writer fios = new OutputStreamWriter(Files.newOutputStream(sourceFile.toPath()), StandardCharsets.UTF_8)) { - fios.write(str.toString()); + try(Writer fios = new OutputStreamWriter(new FileOutputStream(sourceFile), "UTF-8")) { + fios.write(str); } } @@ -1665,7 +1749,7 @@ private static void emitVirtualThreadRuntime(File srcRoot) throws IOException { // cause, where the link error later names only a symbol. throw new IOException("virtual-thread runtime resource missing: " + name); } - copy(in, Files.newOutputStream(new File(srcRoot, name).toPath())); + copy(in, new FileOutputStream(new File(srcRoot, name))); sourceManifest.recordRuntime(name, "/" + name); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java index 35609a140c8..e9e533d0174 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/BytecodeMethod.java @@ -104,12 +104,12 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { private int maxLocals; private static boolean acceptStaticOnEquals; private static final boolean FORCE_VOLATILE_LOCALS = - "true".equalsIgnoreCase(System.getProperty("CN1_FORCE_VOLATILE_LOCALS", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_FORCE_VOLATILE_LOCALS", "false")); // Frameless codegen gate (-Dcn1.frameless, default on). When off the // eligibility predicate always returns false, so every method emits the // legacy frame code byte-for-byte identical to before. private static final boolean FRAMELESS_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless", "true")); // PHASE 3b: extend frameless codegen to OBJECT-BEARING methods (-Dcn1.frameless.objects, // default off). Such a method keeps its object operand stack + object locals in a // method-local C array on the native stack; the C runtime (built with @@ -117,14 +117,14 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { // stopped thread's native stack. With this OFF, only primitive-only methods are // frameless (identical to the prior phase). Requires the conservative-GC runtime. private static final boolean FRAMELESS_OBJECTS_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless.objects", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless.objects", "true")); // PHASE 3b: extend object-frameless to INSTANCE methods (receiver `this` becomes a // conservatively-scanned C parameter). Now DEFAULT ON: the intermittent multi-threaded // failure that previously gated this off was a pre-existing Thread.start/join visibility // race (alive set on the worker thread async after start() returned), fixed in // java_lang_Thread_start__ (993331107); with it fixed, MtStress is 50/50 deterministic. private static final boolean FRAMELESS_INSTANCE_ENABLED = - "true".equalsIgnoreCase(System.getProperty("cn1.frameless.instance", "true")); + "true".equalsIgnoreCase(Util.getProperty("cn1.frameless.instance", "true")); private int methodOffset; private boolean forceVirtual; private boolean virtualOverriden; @@ -162,7 +162,7 @@ public static void setDependencyGraph(MethodDependencyGraph dependencyGraph) { optimizerOn = op == null || op.equalsIgnoreCase("on"); //optimizerOn = false; - onDeviceDebug = "true".equalsIgnoreCase(System.getProperty("cn1.onDeviceDebug", "false")); + onDeviceDebug = "true".equalsIgnoreCase(Util.getProperty("cn1.onDeviceDebug", "false")); } public static boolean isOnDeviceDebug() { @@ -786,16 +786,16 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri if(methodName.equals("")) { methodName = "__INIT__"; constructor = true; - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); } else { if(methodName.equals("")) { methodName = "__CLINIT__"; - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); staticMethod = true; } else { String retType = desc.substring(pos + 1); if(retType.equals("V")) { - returnType = new ByteCodeMethodArg(Void.TYPE, 0); + returnType = new ByteCodeMethodArg(PrimitiveType.VOID, 0); } else { int dim = 0; while(retType.startsWith("[")) { @@ -818,28 +818,28 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri returnType = new ByteCodeMethodArg(objectType, dim); break; case 'I': - returnType = new ByteCodeMethodArg(Integer.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.INT, dim); break; case 'J': - returnType = new ByteCodeMethodArg(Long.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.LONG, dim); break; case 'B': - returnType = new ByteCodeMethodArg(Byte.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.BYTE, dim); break; case 'S': - returnType = new ByteCodeMethodArg(Short.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.SHORT, dim); break; case 'F': - returnType = new ByteCodeMethodArg(Float.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.FLOAT, dim); break; case 'D': - returnType = new ByteCodeMethodArg(Double.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.DOUBLE, dim); break; case 'Z': - returnType = new ByteCodeMethodArg(Boolean.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.BOOLEAN, dim); break; case 'C': - returnType = new ByteCodeMethodArg(Character.TYPE, dim); + returnType = new ByteCodeMethodArg(PrimitiveType.CHAR, dim); break; } } @@ -869,28 +869,28 @@ public BytecodeMethod(String clsName, int access, String name, String desc, Stri arguments.add(new ByteCodeMethodArg(objectType, currentArrayDim)); break; case 'I': - arguments.add(new ByteCodeMethodArg(Integer.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.INT, currentArrayDim)); break; case 'J': - arguments.add(new ByteCodeMethodArg(Long.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.LONG, currentArrayDim)); break; case 'B': - arguments.add(new ByteCodeMethodArg(Byte.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BYTE, currentArrayDim)); break; case 'S': - arguments.add(new ByteCodeMethodArg(Short.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.SHORT, currentArrayDim)); break; case 'F': - arguments.add(new ByteCodeMethodArg(Float.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.FLOAT, currentArrayDim)); break; case 'D': - arguments.add(new ByteCodeMethodArg(Double.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.DOUBLE, currentArrayDim)); break; case 'Z': - arguments.add(new ByteCodeMethodArg(Boolean.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BOOLEAN, currentArrayDim)); break; case 'C': - arguments.add(new ByteCodeMethodArg(Character.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.CHAR, currentArrayDim)); break; } currentArrayDim = 0; @@ -1395,6 +1395,18 @@ public List debugVarEntries() { return rows; } + /** + * Every local, in a deterministic order, for emitting the C declarations. + * + * Unlike {@link #debugVarEntries} this drops nothing: a local whose slot lies + * outside the frame still needs its declaration, it just has no debug row. + */ + private List declarationOrderedLocals() { + List ordered = new ArrayList(localVariables); + Collections.sort(ordered, DEBUG_VAR_ORDER); + return ordered; + } + /** Slot first, then storage qualifier, so a reused slot's rows stay adjacent. */ private static final Comparator DEBUG_VAR_ORDER = new Comparator() { @Override @@ -1629,29 +1641,29 @@ private void fixUpBarebone() { CustomJump cj = (CustomJump)i; String cmp = cj.getCustomCompareCode(); if (cmp != null) { - cj.setCustomCompareCode(cmp.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + cj.setCustomCompareCode(Util.rewriteLocalObjectRefs(cmp)); } } else if (i instanceof CustomIntruction) { CustomIntruction ci = (CustomIntruction)i; String code = ci.getCode(); if (code != null) { - ci.setCode(code.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setCode(Util.rewriteLocalObjectRefs(code)); } String complexCode = ci.getComplexCode(); if (complexCode != null) { - ci.setComplexCode(complexCode.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setComplexCode(Util.rewriteLocalObjectRefs(complexCode)); } } else if (i instanceof CustomInvoke) { CustomInvoke ci = (CustomInvoke)i; String target = ci.getTargetObjectLiteral(); if (target != null) { - ci.setTargetObjectLiteral(target.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")); + ci.setTargetObjectLiteral(Util.rewriteLocalObjectRefs(target)); } String[] args = ci.getLiteralArgs(); if (args != null) { for (int j=0; j added = new HashSet(); - for (LocalVariable lv : localVariables) { + // Sorted, not in localVariables iteration order: that is a HashSet, so the + // order of these declarations varied between builds of the same input. + // debugVarEntries already had to learn this for the debug side-table; the + // C declarations had the same defect and it stayed invisible because + // HotSpot's identity hash is stable within a run. Translating the + // translator with itself is what surfaced it -- a different runtime, a + // different order, and the same input produced different C. + for (LocalVariable lv : declarationOrderedLocals()) { String variableName = lv.getQualifier() + "locals_"+lv.getIndex()+"_"; if (!added.contains(variableName) && (barebone || lv.getQualifier() != 'o')) { added.add(variableName); @@ -2174,7 +2193,7 @@ public void appendVirtualMethodC(String cls, StringBuilder b, String offset, boo b.append(cls); b.append("(threadStateData);\n "); } - if (System.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { + if (Util.getProperty("INCLUDE_NPE_CHECKS", "false").equals("true")) { b.append("\n if(__cn1ThisObject == JAVA_NULL) THROW_NULL_POINTER_EXCEPTION();\n "); } if(!returnType.isVoid()) { @@ -2359,7 +2378,7 @@ public NativeSignatureVerifier.Signature getNativeSignature() { } return new NativeSignatureVerifier.Signature(symbol.toString(), clsName, methodName, overloadPrefix, cReturnType.toString().trim(), params, - prototype.toString().trim().replaceAll("\\s+", " ")); + Util.collapseWhitespace(prototype.toString().trim())); } public boolean isAbstract() { @@ -2436,6 +2455,228 @@ public String getDesc() { return desc; } + /** + * The type this method allocates and hands straight back -- NEW T, DUP, the + * constructor arguments, T.<init>, ARETURN -- or null for any other shape. + * The point of being this strict is that the caller uses the answer as a + * certainty about the returned object's concrete class, so a body that could + * return something it did not just allocate has to be rejected rather than + * guessed at. + */ + public String allocatedReturnType() { + List real = new ArrayList(); + for (Instruction i : instructions) { + if (i instanceof LabelInstruction || i instanceof LineNumber || i instanceof TryCatch) { + continue; + } + real.add(i); + } + if (real.size() < 4) { + return null; + } + Instruction first = real.get(0); + if (!(first instanceof TypeInstruction) || first.getOpcode() != Opcodes.NEW) { + return null; + } + String type = ((TypeInstruction) first).getTypeName(); + if (type == null || real.get(1).getOpcode() != Opcodes.DUP) { + return null; + } + if (real.get(real.size() - 1).getOpcode() != Opcodes.ARETURN) { + return null; + } + Instruction ctor = real.get(real.size() - 2); + if (!(ctor instanceof Invoke) || ctor.getOpcode() != Opcodes.INVOKESPECIAL) { + return null; + } + Invoke ci = (Invoke) ctor; + if (!"".equals(ci.getName()) || !type.equals(ci.getOwner())) { + return null; + } + // Everything between the DUP and the constructor has to be a plain local + // read. Anything with a side effect could leave a different object under + // the ARETURN, and then the type above would be a lie. + for (int i = 2; i < real.size() - 2; i++) { + Instruction a = real.get(i); + if (!(a instanceof VarOp) || !isLoadOpcode(a.getOpcode())) { + return null; + } + } + return type; + } + + private static boolean isLoadOpcode(int op) { + return op == Opcodes.ALOAD || op == Opcodes.ILOAD || op == Opcodes.LLOAD + || op == Opcodes.FLOAD || op == Opcodes.DLOAD; + } + + private int nextExecutable(int from) { + for (int i = from; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (ins instanceof LabelInstruction || ins instanceof LineNumber || ins instanceof TryCatch) { + continue; + } + return i; + } + return -1; + } + + private int prevExecutable(int from) { + for (int i = from; i >= 0; i--) { + Instruction ins = instructions.get(i); + if (ins instanceof LabelInstruction || ins instanceof LineNumber || ins instanceof TryCatch) { + continue; + } + return i; + } + return -1; + } + + /// The first local slot that cannot hold an incoming argument. + /// + /// Parameters occupy locals WITHOUT an ASTORE, so a slot-write count of one + /// does not mean the slot holds one value over the method's lifetime -- an + /// Iterator parameter in that slot is a second, earlier value. Long and double + /// take two slots each, per the JVM numbering the instruction stream uses. + /// + /// @return the lowest slot index that is definitely not a parameter + private int firstNonParameterSlot() { + int slots = isStatic() ? 0 : 1; + for (ByteCodeMethodArg arg : arguments) { + char q = arg.getQualifier(); + slots += (q == 'l' || q == 'd') ? 2 : 1; + } + return slots; + } + + private int countStoresTo(int slot) { + int n = 0; + for (Instruction ins : instructions) { + if (ins instanceof VarOp && ins.getOpcode() == Opcodes.ASTORE + && ((VarOp) ins).getIndex() == slot) { + n++; + } + } + return n; + } + + /** + * ITERATOR LOWERING: give a for-each loop the concrete Iterator type its + * collection really returns, so the calls stop going through the interface. + * + * A for-each compiles to Iterator.hasNext()/next() through INVOKEINTERFACE, + * which is the most expensive dispatch the VM has -- a lookup in the owning + * class's interface map before the vtable read -- and it runs twice per + * element. Neither the emitter's closed-world devirtualization nor ThinLTO + * can touch it, because both start from a concrete owner and an interface + * call does not have one: java.util.Iterator has 27 implementors here. + * + * The concrete type is recoverable locally even though the translator has no + * general stack-type inference. If the collection's iterator() has exactly + * one reachable implementation, and that implementation's whole body is + * `return new T(...)`, then the object stored by the ASTORE that follows the + * call is a T -- no inference needed. Retyping the calls to INVOKEVIRTUAL on + * T is then enough on its own: the existing devirtualization in + * Invoke.appendInstruction takes any virtual call with no reachable override + * the rest of the way to a direct one, which ThinLTO can inline. + * + * The single-assignment requirement on the local is what makes this sound + * without dataflow. If a slot were written twice, a second iterator of some + * other class could reach the same ALOAD, and a virtual call on the wrong + * class reads its fields out of an object that does not have them -- silent + * on this VM, since ParparVM's CHECKCAST is unchecked. + * + * Like the concat fusion this must run BEFORE the unused-method cull, so the + * newly created edges exist while reachability is computed. + */ + public void lowerIteratorCalls() { + for (int i = 0; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (!(ins instanceof Invoke)) { + continue; + } + Invoke inv = (Invoke) ins; + int op = inv.getOpcode(); + if (op != Opcodes.INVOKEINTERFACE && op != Opcodes.INVOKEVIRTUAL) { + continue; + } + if (!"iterator".equals(inv.getName()) || !"()Ljava/util/Iterator;".equals(inv.getDesc())) { + continue; + } + ByteCodeClass coll = Parser.getClassObject(Util.mangle(inv.getOwner())); + String itType = Parser.resolveConcreteIteratorType(coll); + if (itType == null) { + continue; + } + int st = nextExecutable(i + 1); + if (st < 0) { + continue; + } + Instruction store = instructions.get(st); + if (!(store instanceof VarOp) || store.getOpcode() != Opcodes.ASTORE) { + continue; + } + int slot = ((VarOp) store).getIndex(); + // Exactly one ASTORE is not enough on its own: a parameter reaches its + // slot without one, so a method that takes an Iterator and later reuses + // that slot for this loop's iterator has TWO values in it. Rewriting the + // parameter's calls to the concrete type would dispatch methods that + // read the wrong object layout -- unchecked, on this VM. + if (slot < firstNonParameterSlot() || countStoresTo(slot) != 1) { + continue; + } + retypeIteratorUses(slot, itType, st); + } + } + + /// @param storeIdx index of the ASTORE that put the concrete iterator in the + /// slot; only uses AFTER it are rewritten, since anything + /// earlier cannot be reading the value this store wrote + private void retypeIteratorUses(int slot, String itType, int storeIdx) { + ByteCodeClass itClass = Parser.getClassObject(Util.mangle(itType)); + if (itClass == null) { + return; + } + for (int i = storeIdx + 1; i < instructions.size(); i++) { + Instruction ins = instructions.get(i); + if (!(ins instanceof Invoke) || ins.getOpcode() != Opcodes.INVOKEINTERFACE) { + continue; + } + Invoke inv = (Invoke) ins; + if (!"java/util/Iterator".equals(inv.getOwner())) { + continue; + } + int r = prevExecutable(i - 1); + if (r < 0) { + continue; + } + Instruction recv = instructions.get(r); + if (!(recv instanceof VarOp) || recv.getOpcode() != Opcodes.ALOAD + || ((VarOp) recv).getIndex() != slot) { + continue; + } + // The concrete class has to actually resolve the method, and resolve it + // monomorphically -- otherwise the retyped call has nothing to bind to. + if (Parser.resolveDevirtualizedOwner(itClass, inv.getName(), inv.getDesc()) == null) { + continue; + } + Invoke direct = new Invoke(Opcodes.INVOKEVIRTUAL, itType, inv.getName(), inv.getDesc(), false); + instructions.set(i, direct); + // Register it exactly as addInstruction() would: the list entry alone + // leaves the call with no owning method, no class dependency and no + // edge in the dependency graph, so the cull would not see the concrete + // iterator's methods being called. + direct.setMethod(this); + direct.addDependencies(dependentClasses); + if (dependencyGraph != null) { + String uses = direct.getMethodUsed(); + if (uses != null) { + dependencyGraph.recordMethodCall(this, uses); + } + } + } + } + public Set getLocalVariables() { return localVariables; } @@ -2448,6 +2689,10 @@ public void addDebugInfo(int line) { } public void addLabel(Label l) { + // Named here, in bytecode order, so the generated C label is a function of the + // method alone. See LabelInstruction.assignLabelName. + com.codename1.tools.translator.bytecodes.LabelInstruction.assignLabelName(l, nextLabelIndex); + nextLabelIndex++; addInstruction(new com.codename1.tools.translator.bytecodes.LabelInstruction(l)); } @@ -2455,6 +2700,9 @@ public void addInvoke(int opcode, String owner, String name, String desc, boolea addInstruction(new Invoke(opcode, owner, name, desc, itf)); } + /** Per-method label counter; see addLabel. */ + private int nextLabelIndex; + public void setMaxes(int maxStack, int maxLocals) { this.maxLocals = maxLocals; this.maxStack = maxStack; @@ -2667,10 +2915,10 @@ public void appendOnDeviceDebugInvokeThunk(String declaringClsName, StringBuilde * is more specific (e.g. boolean / byte / short / char). */ private String returnTypeChar() { - if (returnType.getPrimitiveType() == Boolean.TYPE) return "Z"; - if (returnType.getPrimitiveType() == Byte.TYPE) return "B"; - if (returnType.getPrimitiveType() == Short.TYPE) return "S"; - if (returnType.getPrimitiveType() == Character.TYPE) return "C"; + if (returnType.getPrimitiveType() == PrimitiveType.BOOLEAN) return "Z"; + if (returnType.getPrimitiveType() == PrimitiveType.BYTE) return "B"; + if (returnType.getPrimitiveType() == PrimitiveType.SHORT) return "S"; + if (returnType.getPrimitiveType() == PrimitiveType.CHAR) return "C"; return "I"; } @@ -2755,7 +3003,7 @@ public void setEliminated(boolean eliminated) { private int varCounter = 0; // Master off-switch: -DCN1_DISABLE_BCE=true reverts to fully-checked array access. private static final boolean DISABLE_BCE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_BCE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_BCE", "false")); /** * Prove-safe array-bounds-check elimination. Conservative and fail-closed: @@ -2931,7 +3179,7 @@ private static boolean bceForeignEntry(java.util.List r, java.util. // the whole struct to registers. // ------------------------------------------------------------------ private static final boolean DISABLE_SCALAR_REPLACE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_SCALAR_REPLACE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_SCALAR_REPLACE", "false")); private static String srMangle(String s) { return s.replace('.', '_').replace('/', '_').replace('$', '_'); @@ -3223,7 +3471,7 @@ private void scalarReplaceStackAllocations() { // can't dispatch to an escaping override. // ------------------------------------------------------------------ private static final boolean DISABLE_SB_STACK_ALLOC = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_SB_STACK_ALLOC", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_SB_STACK_ALLOC", "false")); private static final String SB_OWNER = "java/lang/StringBuilder"; /** Slots consumed by the argument list of a method descriptor (no receiver). */ @@ -4220,6 +4468,7 @@ private void removeRepeatedCheckcasts() { } } + boolean optimize() { // FUSED OBJECTS, constructor side: rewrite each planned // `ALOAD 0; ; NEWARRAY T; PUTFIELD f` quadruple into the diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java new file mode 100644 index 00000000000..1f9b06dbab7 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/DebugSymbolCompressor.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.zip.GZIPOutputStream; + +/** + * Compresses the on-device-debug symbol table. + * + * This is the translator's only use of {@code java.util.zip}, and it exists as its + * own class so that it is the only thing that has to be replaced when the + * translator is compiled against ParparVM's JavaAPI in order to translate itself. + * JavaAPI has no java.util.zip and cannot gain one: it is mirrored by + * Ports/CLDC11, where the package does not belong. + * + * Nothing else needs the package. The translator reads directories of class files, + * never archives -- every caller extracts a jar before invoking it -- and + * {@code NativeSignatureVerifier}'s archive scan lives behind its own command-line + * entry point. + * + * Symbol tables are large and highly repetitive, so compressing keeps a debug + * binary's footprint modest. + */ +final class DebugSymbolCompressor { + private DebugSymbolCompressor() { + } + + static byte[] gzip(ByteArrayOutputStream raw) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(raw.size() / 3 + 64); + GZIPOutputStream gz = new GZIPOutputStream(out); + try { + raw.writeTo(gz); + } finally { + gz.close(); + } + return out.toByteArray(); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java index 6577408bd6b..11c00d9f869 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptMethodGenerator.java @@ -6037,22 +6037,22 @@ private static void appendJsBodyMethod(StringBuilder out, ByteCodeClass cls, Byt if (returnTypeName != null) { jsReturnType = JavascriptNameUtil.sanitizeClassName(returnTypeName); } else { - Class primitiveType = returnType.getPrimitiveType(); - if (primitiveType == Integer.TYPE) { + PrimitiveType primitiveType = returnType.getPrimitiveType(); + if (primitiveType == PrimitiveType.INT) { jsReturnType = "int"; - } else if (primitiveType == Long.TYPE) { + } else if (primitiveType == PrimitiveType.LONG) { jsReturnType = "long"; - } else if (primitiveType == Double.TYPE) { + } else if (primitiveType == PrimitiveType.DOUBLE) { jsReturnType = "double"; - } else if (primitiveType == Float.TYPE) { + } else if (primitiveType == PrimitiveType.FLOAT) { jsReturnType = "float"; - } else if (primitiveType == Boolean.TYPE) { + } else if (primitiveType == PrimitiveType.BOOLEAN) { jsReturnType = "boolean"; - } else if (primitiveType == Byte.TYPE) { + } else if (primitiveType == PrimitiveType.BYTE) { jsReturnType = "byte"; - } else if (primitiveType == Short.TYPE) { + } else if (primitiveType == PrimitiveType.SHORT) { jsReturnType = "short"; - } else if (primitiveType == Character.TYPE) { + } else if (primitiveType == PrimitiveType.CHAR) { jsReturnType = "char"; } else { jsReturnType = "java_lang_Object"; diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java index 04be9ad4685..2c822e46757 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/JavascriptNativeRegistry.java @@ -128,6 +128,7 @@ enum NativeCategory { "cn1_java_lang_System_currentTimeMillis_R_long", "cn1_java_lang_System_exit_int", "cn1_java_lang_System_gcLight", + "cn1_java_lang_System_getenvImpl_java_lang_String_R_java_lang_String", "cn1_java_lang_System_gcMarkSweep", "cn1_java_lang_System_identityHashCode_java_lang_Object_R_int", "cn1_java_lang_Integer_cn1Value_R_int", diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java index 3db5a800842..4ae3e021028 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifier.java @@ -27,7 +27,6 @@ import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; -import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -48,8 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; /** * Checks that every {@code native} method in a translated project has a C @@ -400,24 +397,35 @@ public int size() { } } - /** Reads {@link #IGNORE_FILE}: one symbol or {@code prefix*} per line. */ + /** + * Reads {@link #IGNORE_FILE}: one symbol or {@code prefix*} per line. + * + * Splits the file itself rather than using a BufferedReader, which ParparVM's + * JavaAPI does not declare -- this runs during translation, so it has to compile + * when the translator is built against that JavaAPI to translate itself. + */ static void readIgnoreFile(File file, Set into) throws IOException { - BufferedReader reader = new BufferedReader( - new InputStreamReader(new FileInputStream(file), UTF8)); - try { - String line; - while ((line = reader.readLine()) != null) { - int hash = line.indexOf('#'); - if (hash >= 0) { - line = line.substring(0, hash); - } - line = line.trim(); - if (line.length() > 0) { - into.add(line); - } + String text = new String(readAll(file), UTF8); + int start = 0; + while (start <= text.length()) { + int end = text.indexOf('\n', start); + String line = end < 0 ? text.substring(start) : text.substring(start, end); + // Accept CRLF as readLine did. + if (line.endsWith("\r")) { + line = line.substring(0, line.length() - 1); + } + int hash = line.indexOf('#'); + if (hash >= 0) { + line = line.substring(0, hash); + } + line = line.trim(); + if (line.length() > 0) { + into.add(line); + } + if (end < 0) { + break; } - } finally { - reader.close(); + start = end + 1; } } @@ -817,7 +825,7 @@ private static List splitTopLevel(String text) { private static String normalizeParameter(String declaration) { String text = declaration.replace("*", " * ").trim(); List tokens = new ArrayList( - Arrays.asList(text.split("\\s+"))); + Arrays.asList(Util.splitWhitespace(text))); // "CODENAME_ONE_THREAD_STATE" is a macro that expands to a full declaration // and carries no separate name to strip. if (tokens.size() > 1 && !"CODENAME_ONE_THREAD_STATE".equals(tokens.get(0))) { @@ -1172,7 +1180,7 @@ public static List collectFromClasses(File root) throws IOException { if (root.isDirectory()) { collectClassesFromDirectory(root, found); } else if (root.getName().endsWith(".jar") || root.getName().endsWith(".zip")) { - collectClassesFromArchive(root, found); + ArchiveClassScanner.collect(root, found); } else if (root.getName().endsWith(".class")) { collectFromClassBytes(readAll(root), found); } @@ -1195,32 +1203,7 @@ private static void collectClassesFromDirectory(File dir, List into) } } - private static void collectClassesFromArchive(File archive, List into) throws IOException { - ZipFile zip = new ZipFile(archive); - try { - List names = new ArrayList(); - for (Enumeration e = zip.entries(); e.hasMoreElements();) { - ZipEntry entry = e.nextElement(); - if (!entry.isDirectory() && entry.getName().endsWith(".class") - && !entry.getName().endsWith("module-info.class")) { - names.add(entry.getName()); - } - } - Collections.sort(names); - for (String name : names) { - InputStream in = zip.getInputStream(zip.getEntry(name)); - try { - collectFromClassBytes(readAll(in), into); - } finally { - in.close(); - } - } - } finally { - zip.close(); - } - } - - private static void collectFromClassBytes(byte[] bytes, final List into) { + static void collectFromClassBytes(byte[] bytes, final List into) { final String[] owner = new String[1]; new ClassReader(bytes).accept(new ClassVisitor(Opcodes.ASM9) { @Override @@ -1293,74 +1276,6 @@ private static boolean isIdentifier(String s) { return true; } - public static void main(String[] args) throws IOException { - List classRoots = new ArrayList(); - List nativeRoots = new ArrayList(); - boolean orphans = true; - for (int iter = 0; iter < args.length; iter++) { - if ("--classes".equals(args[iter]) && iter + 1 < args.length) { - classRoots.add(new File(args[++iter])); - } else if ("--natives".equals(args[iter]) && iter + 1 < args.length) { - nativeRoots.add(new File(args[++iter])); - } else if ("--no-orphans".equals(args[iter])) { - orphans = false; - } else { - System.err.println("unrecognised argument: " + args[iter]); - usage(); - System.exit(2); - } - } - if (classRoots.isEmpty() || nativeRoots.isEmpty()) { - usage(); - System.exit(2); - } - - List required = new ArrayList(); - for (File root : classRoots) { - if (!root.exists()) { - System.err.println("NativeSignatureVerifier: no such path: " + root); - System.exit(2); - } - required.addAll(collectFromClasses(root)); - } - List sources = new ArrayList(); - for (File root : nativeRoots) { - if (!root.exists()) { - System.err.println("NativeSignatureVerifier: no such path: " + root); - System.exit(2); - } - sources.addAll(root.isDirectory() - ? listNativeSourcesRecursive(root) : Collections.singletonList(root)); - } - - SourceIndex index = new SourceIndex(sources); - List problems = verify(required, index); - if (!orphans) { - List filtered = new ArrayList(); - for (Problem problem : problems) { - if (problem.kind != Kind.ORPHAN) { - filtered.add(problem); - } - } - problems = filtered; - } - - if (problems.isEmpty()) { - System.out.println("NativeSignatureVerifier: " + required.size() - + " native method(s) all resolve against " + index.size() - + " C definition(s) in " + sources.size() + " file(s)."); - return; - } - int fatal = report(problems, Mode.STRICT, - required.size() + " native methods, " + sources.size() + " native sources", true); - System.exit(fatal > 0 ? 1 : 0); - } - - private static void usage() { - System.err.println("usage: NativeSignatureVerifier --classes DIR_OR_JAR [--classes ...]" - + " --natives DIR [--natives ...] [--no-orphans]"); - } - private NativeSignatureVerifier() { } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java new file mode 100644 index 00000000000..d5ef2921d96 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/NativeSignatureVerifierCli.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Command-line entry point for {@link NativeSignatureVerifier}, driven by + * scripts/check-native-signatures.sh. + * + * Split out of the verifier for two reasons, both about the self-hosted translator + * build. It is the half that scans jars, so it is the half that needs + * java.util.zip -- which JavaAPI cannot gain, being mirrored by Ports/CLDC11. And a + * second class carrying a {@code main} makes ByteCodeClass.addMethod refuse the + * translation outright with "Multiple main classes", since the clean target does + * not set a preferred main class the way the JavaScript target does. + * + * A translation never comes through here: the verifier's in-process entry points + * are what Parser calls. + * + * NativeSignatureVerifier deliberately keeps NO delegating {@code main}. Nothing + * names it as one -- scripts/check-native-signatures.sh invokes this class, and no + * document spells the old command -- and adding one would recreate the very edge + * the split removes: the verifier would reference the CLI, and the CLI reaches + * java.util.zip. A second {@code main} also brings back the "Multiple main + * classes" refusal above. Backward compatibility for an invocation nobody has is + * not worth either. + */ +public final class NativeSignatureVerifierCli { + private NativeSignatureVerifierCli() { + } + + public static void main(String[] args) throws IOException { + List classRoots = new ArrayList(); + List nativeRoots = new ArrayList(); + boolean orphans = true; + for (int iter = 0; iter < args.length; iter++) { + if ("--classes".equals(args[iter]) && iter + 1 < args.length) { + classRoots.add(new File(args[++iter])); + } else if ("--natives".equals(args[iter]) && iter + 1 < args.length) { + nativeRoots.add(new File(args[++iter])); + } else if ("--no-orphans".equals(args[iter])) { + orphans = false; + } else { + System.err.println("unrecognised argument: " + args[iter]); + usage(); + System.exit(2); + } + } + if (classRoots.isEmpty() || nativeRoots.isEmpty()) { + usage(); + System.exit(2); + } + + List required = new ArrayList(); + for (File root : classRoots) { + if (!root.exists()) { + System.err.println("NativeSignatureVerifier: no such path: " + root); + System.exit(2); + } + required.addAll(NativeSignatureVerifier.collectFromClasses(root)); + } + List sources = new ArrayList(); + for (File root : nativeRoots) { + if (!root.exists()) { + System.err.println("NativeSignatureVerifier: no such path: " + root); + System.exit(2); + } + sources.addAll(root.isDirectory() + ? NativeSignatureVerifier.listNativeSourcesRecursive(root) : Collections.singletonList(root)); + } + + NativeSignatureVerifier.SourceIndex index = new NativeSignatureVerifier.SourceIndex(sources); + List problems = NativeSignatureVerifier.verify(required, index); + if (!orphans) { + List filtered = new ArrayList(); + for (NativeSignatureVerifier.Problem problem : problems) { + if (problem.kind != NativeSignatureVerifier.Kind.ORPHAN) { + filtered.add(problem); + } + } + problems = filtered; + } + + if (problems.isEmpty()) { + System.out.println("NativeSignatureVerifier: " + required.size() + + " native method(s) all resolve against " + index.size() + + " C definition(s) in " + sources.size() + " file(s)."); + return; + } + int fatal = NativeSignatureVerifier.report(problems, NativeSignatureVerifier.Mode.STRICT, + required.size() + " native methods, " + sources.size() + " native sources", true); + System.exit(fatal > 0 ? 1 : 0); + } + + private static void usage() { + System.err.println("usage: NativeSignatureVerifier --classes DIR_OR_JAR [--classes ...]" + + " --natives DIR [--natives ...] [--no-orphans]"); + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java index 6c9ae454f65..7dae4f551ca 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Parser.java @@ -25,7 +25,6 @@ import java.io.*; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.*; import org.objectweb.asm.AnnotationVisitor; @@ -139,6 +138,33 @@ public static synchronized String resolveDevirtualizedOwner(ByteCodeClass owner, } return null; } + /** + * The concrete Iterator class a for-each over this collection type will + * really get, or null when that cannot be established with certainty. + * + * Two things have to hold. The collection's iterator() must have exactly one + * reachable implementation -- resolveDevirtualizedOwner answers that -- and + * that implementation must do nothing but allocate and return, so the class + * it allocates is the class the caller receives. Anything else answers null + * and the call site is left as the interface call it was. + */ + public static synchronized String resolveConcreteIteratorType(ByteCodeClass owner) { + String decl = resolveDevirtualizedOwner(owner, "iterator", "()Ljava/util/Iterator;"); + if (decl == null) { + return null; + } + ByteCodeClass dc = getClassObject(Util.mangle(decl)); + if (dc == null) { + return null; + } + for (BytecodeMethod m : dc.getMethods()) { + if ("iterator".equals(m.getMethodName()) && "()Ljava/util/Iterator;".equals(m.getDesc())) { + return m.allocatedReturnType(); + } + } + return null; + } + private static final MethodDependencyGraph dependencyGraph = new MethodDependencyGraph(); private int lambdaCounter; private int stringConcatCounter; @@ -162,7 +188,7 @@ public static void parse(File sourceFile) throws Exception { } BytecodeMethod.setDependencyGraph(dependencyGraph); ClassReader r; - try (InputStream in = Files.newInputStream(sourceFile.toPath())) { + try (InputStream in = new FileInputStream(sourceFile)) { r = new ClassReader(in); } Parser p = new Parser(); @@ -273,7 +299,7 @@ public static int jdwpAccessFlagsOf(ByteCodeField bf) { */ private static void writeSymbolSidecar(File outputDirectory) throws IOException { java.io.ByteArrayOutputStream raw = new java.io.ByteArrayOutputStream(1 << 20); - try (Writer w = new OutputStreamWriter(raw, StandardCharsets.UTF_8)) { + try (Writer w = new OutputStreamWriter(raw, "UTF-8")) { w.write("version\t1\n"); for (ByteCodeClass bc : classes) { String src = bc.getSourceFile(); @@ -355,17 +381,13 @@ private static void writeSymbolSidecar(File outputDirectory) throws IOException // gzip the payload — symbol tables are large and highly repetitive, // so this keeps the debug binary's footprint modest. - java.io.ByteArrayOutputStream gzOut = new java.io.ByteArrayOutputStream(raw.size() / 3 + 64); - try (java.util.zip.GZIPOutputStream gz = new java.util.zip.GZIPOutputStream(gzOut)) { - raw.writeTo(gz); - } - byte[] gz = gzOut.toByteArray(); + byte[] gz = DebugSymbolCompressor.gzip(raw); // Compiled into the project like any other generated unit when // cn1.onDeviceDebug is on, so it needs provenance for the same reason they do. ByteCodeTranslator.sourceManifest.recordGenerated("cn1_debug_symbols.c"); File f = new File(outputDirectory, "cn1_debug_symbols.c"); - try (Writer w = new OutputStreamWriter(Files.newOutputStream(f.toPath()), StandardCharsets.UTF_8)) { + try (Writer w = new OutputStreamWriter(new FileOutputStream(f), "UTF-8")) { w.write("/* Auto-generated by the Codename One iOS translator. Do not edit.\n"); w.write(" * On-device-debug symbol table (gzip-compressed), streamed to the\n"); w.write(" * desktop debug proxy over CMD_GET_SYMBOLS. */\n"); @@ -381,8 +403,8 @@ private static void writeSymbolSidecar(File outputDirectory) throws IOException } w.write("0x"); int b = gz[i] & 0xff; - w.write(Character.forDigit(b >> 4, 16)); - w.write(Character.forDigit(b & 0xf, 16)); + w.write(Util.hexDigit(b >> 4)); + w.write(Util.hexDigit(b & 0xf)); w.write(','); w.write((i & 15) == 15 ? '\n' : ' '); } @@ -454,6 +476,16 @@ public static NativeSymbolIndex getNativeSymbolIndex(String[] nativeSources) { } private static final ArrayList constantPool = new ArrayList<>(); + // Index of constantPool, so addToConstantPool does not have to scan it. + // + // The list stays the source of truth -- writeOutput emits it in order and the + // emitted indices are positions in it -- and this only answers "where is s", the + // question ArrayList.indexOf was answering with a String.equals against every + // entry already interned. On a self-hosting translation the pool holds ~200k + // strings and that scan was the single largest cost on the mutator thread: + // String.equals 11.2%, the iterator 10.3%, indexOf 6.2% and ArrayList.get 5.1% + // of samples, all of it here. + private static final Map constantPoolIndex = new HashMap(); // Name -> class index, replacing the O(N) linear scans that getClassObject / // getClassByName / ByteCodeClass.findClass used to do. Those run per dependency @@ -489,12 +521,14 @@ public static ByteCodeClass getClassObject(String name) { * Adds the given string to the hardcoded constant pool strings returns the offset in the pool */ public static int addToConstantPool(String s) { - int i = constantPool.indexOf(s); - if(i < 0) { - constantPool.add(s); - return constantPool.size() - 1; - } - return i; + Integer existing = constantPoolIndex.get(s); + if(existing != null) { + return existing.intValue(); + } + int index = constantPool.size(); + constantPool.add(s); + constantPoolIndex.put(s, Integer.valueOf(index)); + return index; } @@ -802,6 +836,26 @@ public static void writeOutput(File outputDirectory) throws Exception { neliminated++; } + // Fuse all-String StringBuilder concat chains into String.cn1ConcatN + // BEFORE the cull, not during code generation. + // + // The cull decides what to keep from the dependency graph, and the graph + // is only told about a call when the instruction is added. A rewrite that + // runs later -- inside BytecodeMethod.optimize(), which happens during + // generateCCode -- inserts calls to methods the cull has already deleted, + // and a deleted method is emitted as `return 0;`. That is not a build + // error: the rewritten call silently answered null, java.io.File got a + // null path, and the translator died in File.getParentFile with a SIGSEGV + // nowhere near the rewrite. Running here, the references exist before + // anything is eliminated. See BytecodeMethod.lowerIteratorCalls. + if (BytecodeMethod.optimizerOn) { + for (ByteCodeClass fuseCls : classes) { + for (BytecodeMethod fuseMtd : fuseCls.getMethods()) { + fuseMtd.lowerIteratorCalls(); + } + } + } + // loop over methods and start eliminating the body of unused methods if (BytecodeMethod.optimizerOn) { if(ByteCodeTranslator.verbose) { @@ -875,7 +929,7 @@ public static void writeOutput(File outputDirectory) throws Exception { generateClassAndMethodIndexHeader(outputDirectory); - boolean concatenate = "true".equals(System.getProperty("concatenateFiles", "false")); + boolean concatenate = "true".equals(Util.getProperty("concatenateFiles", "false")); ConcatenatingFileOutputStream cos = concatenate ? new ConcatenatingFileOutputStream(outputDirectory) : null; for(ByteCodeClass bc : classes) { @@ -914,7 +968,7 @@ public static void writeOutput(File outputDirectory) throws Exception { } private static void readNativeFiles(File outputDirectory) throws IOException { - File[] mFiles = outputDirectory.listFiles(file -> + File[] mFiles = Util.listFiles(outputDirectory, file -> file.getName().endsWith(".m") || file.getName().endsWith("." + ByteCodeTranslator.output.extension())); if(mFiles == null) { return; @@ -1122,7 +1176,14 @@ private static int cullClasses(boolean found, int depth) { // 2nd pass to mark classes as eliminated so that we can propagate down to each // method of the class to mark it eliminated so that virtual methods // aren't included later on when writing virtual methods - Set removedClasses = new HashSet<>(classes); + // LinkedHashSet, not HashSet: ByteCodeClass overrides neither equals nor + // hashCode, so a HashSet here iterates in identity-hash order. Elimination + // is greedy and monotone -- isMethodUsed treats an already-eliminated + // caller as no caller -- so with a cycle in the call graph the ORDER + // decides which member of the cycle survives. Two runtimes hash + // identities differently and culled different methods from the same + // input; translating the translator with itself is what exposed it. + Set removedClasses = new LinkedHashSet<>(classes); tmp.forEach(removedClasses::remove); int nfound = 0; for (ByteCodeClass cls : removedClasses) { @@ -1165,7 +1226,7 @@ private static void writeFile(ByteCodeClass cls, File outputDir, ConcatenatingFi // it back to one file per class. writeBufferInstead != null && ByteCodeTranslator.output.isApple() ? writeBufferInstead : - Files.newOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension()).toPath()); + new FileOutputStream(new File(outputDir, cls.getClsName() + "." + ByteCodeTranslator.output.extension())); if (outMain instanceof ConcatenatingFileOutputStream) { ((ConcatenatingFileOutputStream)outMain).beginNextFile(cls.getClsName()); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java new file mode 100644 index 00000000000..1b06b7e0d04 --- /dev/null +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/PrimitiveType.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +/** + * The nine primitive types, as a token the translator can compare and hash. + * + *

This used to be {@code java.lang.Class}, holding {@code Integer.TYPE} and its + * eight siblings. Nothing ever reflected on those objects: every use was an + * identity comparison against one of the nine constants, or a lookup in a + * {@code HashMap} keyed on them. {@code Class} was standing in for + * an enum, and it carried two problems that an enum does not. + * + *

The first is that {@code X.TYPE} does not exist on a ParparVM target. javac + * lowers the primitive class literal in {@code Integer.TYPE = int.class} to a read + * of the field being initialized, so the wrapper's own {@code } stores + * null into it; three of the nine wrappers do not declare the field at all. Keyed + * on those, both maps collapsed to a single entry and {@code getCType} answered the + * same C type for every primitive -- valid C, every type wrong, nothing thrown. + * That made the maps unusable in a self-hosted translator, which is what forced + * this change. + * + *

The second is ordering. {@code Class} has no {@code hashCode} of its own, so + * {@code ByteCodeMethodArg.hashCode} was returning an identity hash, which varies + * between runs of one JVM. Anything that iterated a hash container of those keys + * and wrote the result would emit a different file each time. + * + *

Note for the same reason that {@link #ordinal()} is used explicitly wherever a + * hash is needed rather than calling {@code hashCode()} on a constant here: + * {@code Enum.hashCode} is an identity hash on OpenJDK and the ordinal in + * ParparVM's {@code java.lang.Enum}, so relying on it would make the JVM-hosted and + * self-hosted translators disagree on hash order -- a difference the self-hosting + * gate would report as a VM divergence. + */ +public enum PrimitiveType { + INT("JAVA_INT", "int", "I"), + LONG("JAVA_LONG", "long", "J"), + SHORT("JAVA_SHORT", "short", "S"), + BYTE("JAVA_BYTE", "byte", "B"), + DOUBLE("JAVA_DOUBLE", "double", "D"), + FLOAT("JAVA_FLOAT", "float", "F"), + BOOLEAN("JAVA_BOOLEAN", "boolean", "Z"), + CHAR("JAVA_CHAR", "char", "C"), + VOID("JAVA_VOID", "void", "V"); + + private final String cType; + private final String sigType; + private final String descriptor; + + private PrimitiveType(String cType, String sigType, String descriptor) { + this.cType = cType; + this.sigType = sigType; + this.descriptor = descriptor; + } + + /** The C type the generated code uses for this primitive, e.g. JAVA_INT. */ + public String getCType() { + return cType; + } + + /** The Java keyword, as it appears in a mangled C method name. */ + public String getSigType() { + return sigType; + } + + /** The JVM field descriptor character, e.g. I for int. */ + public String getDescriptor() { + return descriptor; + } +} diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java index ce0228d484a..b4fb7cba6bd 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/SourceManifest.java @@ -23,11 +23,11 @@ package com.codename1.tools.translator; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.nio.charset.Charset; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -259,7 +259,10 @@ public int size() { */ public void write(File projectRoot) throws IOException { File out = new File(projectRoot, FILE_NAME); - try (Writer w = new OutputStreamWriter(Files.newOutputStream(out.toPath()), UTF8)) { + // java.io rather than java.nio.file: the translator compiles against JavaAPI + // when it translates itself, and JavaAPI has no java.nio.file. See + // vm/selfhost. + try (Writer w = new OutputStreamWriter(new FileOutputStream(out), "UTF-8")) { w.write("# Provenance of every file in the generated project's source directory.\n"); w.write("# Written by the ParparVM translator; consumed by\n"); w.write("# scripts/check-native-warnings.py to decide who owns a compiler warning.\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java index a02f3520ae8..2644dc9a35b 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/Util.java @@ -24,10 +24,12 @@ import com.codename1.tools.translator.bytecodes.Instruction; import com.codename1.tools.translator.bytecodes.TryCatch; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.objectweb.asm.Opcodes; /** @@ -36,36 +38,12 @@ */ public class Util { - private static final Map ctypeMap = new HashMap(); - private static final Map sigTypeMap = new HashMap(); - - static { - ctypeMap.put(Integer.TYPE, "JAVA_INT"); - ctypeMap.put(Long.TYPE, "JAVA_LONG"); - ctypeMap.put(Short.TYPE, "JAVA_SHORT"); - ctypeMap.put(Byte.TYPE, "JAVA_BYTE"); - ctypeMap.put(Double.TYPE, "JAVA_DOUBLE"); - ctypeMap.put(Float.TYPE, "JAVA_FLOAT"); - ctypeMap.put(Boolean.TYPE, "JAVA_BOOLEAN"); - ctypeMap.put(Character.TYPE, "JAVA_CHAR"); - ctypeMap.put(Void.TYPE, "JAVA_VOID"); - sigTypeMap.put(Integer.TYPE, "int"); - sigTypeMap.put(Long.TYPE, "long"); - sigTypeMap.put(Short.TYPE, "short"); - sigTypeMap.put(Byte.TYPE, "byte"); - sigTypeMap.put(Double.TYPE, "double"); - sigTypeMap.put(Float.TYPE, "float"); - sigTypeMap.put(Boolean.TYPE, "boolean"); - sigTypeMap.put(Character.TYPE, "char"); - sigTypeMap.put(Void.TYPE, "void"); + public static String getCType(PrimitiveType type) { + return type == null ? null : type.getCType(); } - public static String getCType(Class cls) { - return ctypeMap.get(cls); - } - - public static String getSigType(Class cls) { - return sigTypeMap.get(cls); + public static String getSigType(PrimitiveType type) { + return type == null ? null : type.getSigType(); } public static List getMethodArgs(String methodDesc) { @@ -94,28 +72,28 @@ public static List getMethodArgs(String methodDesc) { arguments.add(new ByteCodeMethodArg(objectType, currentArrayDim)); break; case 'I': - arguments.add(new ByteCodeMethodArg(Integer.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.INT, currentArrayDim)); break; case 'J': - arguments.add(new ByteCodeMethodArg(Long.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.LONG, currentArrayDim)); break; case 'B': - arguments.add(new ByteCodeMethodArg(Byte.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BYTE, currentArrayDim)); break; case 'S': - arguments.add(new ByteCodeMethodArg(Short.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.SHORT, currentArrayDim)); break; case 'F': - arguments.add(new ByteCodeMethodArg(Float.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.FLOAT, currentArrayDim)); break; case 'D': - arguments.add(new ByteCodeMethodArg(Double.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.DOUBLE, currentArrayDim)); break; case 'Z': - arguments.add(new ByteCodeMethodArg(Boolean.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.BOOLEAN, currentArrayDim)); break; case 'C': - arguments.add(new ByteCodeMethodArg(Character.TYPE, currentArrayDim)); + arguments.add(new ByteCodeMethodArg(PrimitiveType.CHAR, currentArrayDim)); break; } currentArrayDim = 0; @@ -422,4 +400,324 @@ public static char[] getStackOutputTypes(Instruction instr) { } } + + /** + * Writes {@code data} to {@code target}, replacing it. + * + * Stands in for {@code Files.write(Path, byte[])}. ParparVM's JavaAPI has no + * java.nio.file, and the translator has to compile against it to be able to + * translate itself, so the whole translator stays on java.io. + */ + public static void writeBytes(File target, byte[] data) throws IOException { + OutputStream out = new FileOutputStream(target); + try { + out.write(data); + } finally { + out.close(); + } + } + + /** + * The path of {@code f} relative to {@code root}, with '/' separators. + * + * Stands in for {@code root.toPath().relativize(f.toPath())} for the one case + * that needs it: {@code f} is always found by walking {@code root}, so it is + * always underneath it and no ".." segment can arise. + */ + public static String relativePath(File root, File f) { + String rootPath = root.getAbsolutePath(); + String filePath = f.getAbsolutePath(); + if (filePath.startsWith(rootPath)) { + filePath = filePath.substring(rootPath.length()); + } + filePath = filePath.replace('\\', '/'); + while (filePath.startsWith("/")) { + filePath = filePath.substring(1); + } + return filePath; + } + + /** + * Java's {@code \s}: the six characters the regex engine treats as whitespace. + * Deliberately not Character.isWhitespace, which differs -- it excludes the + * vertical tab and accepts many Unicode separators. + * + * 0x0B rather than an escape because a raw control byte in a source file is + * what check-control-characters.py exists to reject. + */ + private static boolean isRegexWhitespace(char c) { + return c == ' ' || c == '\t' || c == '\n' || c == 0x0B || c == '\f' || c == '\r'; + } + + /** + * Equivalent of {@code s.split(String.valueOf(separator))} for a separator that + * is not a regex metacharacter, including the trailing-empty-string removal + * String.split does at the default limit of zero. + * + * The translator has to compile against ParparVM's JavaAPI in order to translate + * itself, and String.split is not declared there. It is one of the methods + * BytecodeComplianceMojo rewrites onto com.codename1.util.regex precisely + * because JavaAPI lacks it, so adding it there would leave two regex engines and + * a rewrite rule whose premise had become false. The few call sites here lose + * the regex instead. + */ + public static String[] splitLiteral(String s, char separator) { + // String.split returns { s } when the pattern never matches, WITHOUT the + // trailing-empty removal below -- so "".split(";") is { "" }, not { }. Missing + // this is the one way a hand-written splitter and the regex part company on + // an input a caller can actually produce (an unset build hint). + if (s.indexOf(separator) < 0) { + return new String[] { s }; + } + List parts = new ArrayList(); + int start = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == separator) { + parts.add(s.substring(start, i)); + start = i + 1; + } + } + parts.add(s.substring(start)); + int end = parts.size(); + while (end > 0 && parts.get(end - 1).isEmpty()) { + end--; + } + return parts.subList(0, end).toArray(new String[end]); + } + + /** + * Equivalent of {@code s.split("\\s+")}, including the leading empty string + * String.split produces when the input starts with whitespace, and the removal + * of trailing empty strings. See {@link #splitLiteral} for why this is not a + * regex. + */ + public static String[] splitWhitespace(String s) { + // See splitLiteral: no match means { s }, trailing-empty removal skipped. + boolean matched = false; + for (int j = 0; j < s.length(); j++) { + if (isRegexWhitespace(s.charAt(j))) { + matched = true; + break; + } + } + if (!matched) { + return new String[] { s }; + } + List parts = new ArrayList(); + int i = 0; + int start = 0; + while (i < s.length()) { + if (isRegexWhitespace(s.charAt(i))) { + parts.add(s.substring(start, i)); + while (i < s.length() && isRegexWhitespace(s.charAt(i))) { + i++; + } + start = i; + } else { + i++; + } + } + parts.add(s.substring(start)); + int end = parts.size(); + while (end > 0 && parts.get(end - 1).isEmpty()) { + end--; + } + return parts.subList(0, end).toArray(new String[end]); + } + + /** + * Equivalent of {@code s.replaceAll("\\s+", " ")}. See {@link #splitLiteral} + * for why this is not a regex. + */ + public static String collapseWhitespace(String s) { + StringBuilder b = new StringBuilder(s.length()); + int i = 0; + while (i < s.length()) { + char c = s.charAt(i); + if (isRegexWhitespace(c)) { + b.append(' '); + while (i < s.length() && isRegexWhitespace(s.charAt(i))) { + i++; + } + } else { + b.append(c); + i++; + } + } + return b.toString(); + } + + /** + * Equivalent of + * {@code s.replaceAll("locals\\[(\\d+)\\]\\.data\\.o", "olocals_$1_")}: rewrites + * an indexed object local into the scalar-replaced name the barebone path emits. + * + * Besides removing the regex (see {@link #splitLiteral}), this drops a Pattern + * compile that used to happen once per barebone method in every build. + */ + public static String rewriteLocalObjectRefs(String s) { + final String prefix = "locals["; + final String suffix = "].data.o"; + int at = s.indexOf(prefix); + if (at < 0) { + return s; + } + StringBuilder b = new StringBuilder(s.length()); + int from = 0; + while (at >= 0) { + int digits = at + prefix.length(); + int end = digits; + while (end < s.length() && s.charAt(end) >= '0' && s.charAt(end) <= '9') { + end++; + } + if (end > digits && s.startsWith(suffix, end)) { + b.append(s, from, at); + b.append("olocals_").append(s, digits, end).append('_'); + from = end + suffix.length(); + } else { + // \d+ needs at least one digit and "].data.o" must follow it, so this + // occurrence is not a match; copy it through and keep scanning after it. + b.append(s, from, digits); + from = digits; + } + at = s.indexOf(prefix, from); + } + b.append(s, from, s.length()); + return b.toString(); + } + + /** + * {@code System.getProperty(key, defaultValue)}, falling back to the + * environment. + * + * ParparVM's JavaAPI declares only the one-argument form, and it returns null + * unconditionally -- a native binary has no -D to read. The translator has to + * compile against that JavaAPI in order to translate itself, so the two-argument + * form is provided here instead of being added to JavaAPI, and every knob gains + * an environment spelling that works in a translated build. cn1.sqlite is read + * from CN1_SQLITE, INCLUDE_NPE_CHECKS from INCLUDE_NPE_CHECKS. + * + * NativeSignatureVerifier.mode() already reached for getenv for exactly this + * reason; this generalizes it rather than adding a second convention. + */ + /** + * Memoized ParparVM name mangling: '/' and '$' both become '_'. + * + * The tree contains 95 hand-written copies of + * {@code x.replace('/', '_').replace('$', '_')}, 54 of them in the + * per-instruction emit classes (Invoke, Field, CustomInvoke, Ldc), so the + * SAME owner string is re-mangled once per emitted instruction. The distinct + * inputs are bounded by the class count (5782 on the hellocodenameone + * corpus) while the calls run into the millions. + * + * String.replace already returns {@code this} when the character is absent, + * so the '$' pass is usually free; the '/' pass is the one that allocates a + * char[] and a String every time. Caching turns that into one lookup. + * + * Not synchronized: the translator parses and emits on a single thread -- + * Parser.writeOutput is one sequential loop with no executor and a single + * writeFile call site. + */ + private static final java.util.Map MANGLE_CACHE = + new java.util.HashMap(); + + public static String mangle(String name) { + if (name == null) { + return null; + } + String m = MANGLE_CACHE.get(name); + if (m == null) { + m = name.replace('/', '_').replace('$', '_'); + MANGLE_CACHE.put(name, m); + } + return m; + } + + public static String getProperty(String key, String defaultValue) { + String value = System.getProperty(key); + if (value == null) { + value = System.getenv(environmentName(key)); + } + return value == null ? defaultValue : value; + } + + /** + * "cn1.sqlite" -> "CN1_SQLITE". Folded by hand: String.toUpperCase is locale + * sensitive and CN1 has no java.util.Locale to ask for the root locale, so on a + * Turkish device the 'i' of "cn1.sqlite" would not fold to 'I' and the variable + * would never be found. + */ + private static String environmentName(String key) { + StringBuilder b = new StringBuilder(key.length()); + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i); + if (c >= 'a' && c <= 'z') { + b.append((char) (c - 'a' + 'A')); + } else if ((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) { + b.append(c); + } else { + b.append('_'); + } + } + return b.toString(); + } + + /** + * Stands in for {@code java.io.FileFilter}, which ParparVM's JavaAPI does not + * declare. Kept as a functional interface so the call sites keep their lambdas. + */ + public interface FileMatcher { + boolean accept(File file); + } + + /** + * Stands in for {@code java.io.FilenameFilter}. + */ + public interface FileNameMatcher { + boolean accept(File dir, String name); + } + + /** + * {@code dir.listFiles(filter)}, including its null return when {@code dir} is + * not a directory -- callers test for it. + */ + public static File[] listFiles(File dir, FileMatcher matcher) { + File[] all = dir.listFiles(); + if (all == null) { + return null; + } + List kept = new ArrayList(all.length); + for (int i = 0; i < all.length; i++) { + if (matcher.accept(all[i])) { + kept.add(all[i]); + } + } + return kept.toArray(new File[kept.size()]); + } + + /** + * {@code dir.list(filter)}, including its null return when {@code dir} is not a + * directory. + */ + public static String[] list(File dir, FileNameMatcher matcher) { + String[] all = dir.list(); + if (all == null) { + return null; + } + List kept = new ArrayList(all.length); + for (int i = 0; i < all.length; i++) { + if (matcher.accept(dir, all[i])) { + kept.add(all[i]); + } + } + return kept.toArray(new String[kept.size()]); + } + + /** + * {@code Character.forDigit(digit, 16)} for a digit already known to be in + * range. JavaAPI has no forDigit. + */ + public static char hexDigit(int digit) { + return (char) (digit < 10 ? '0' + digit : 'a' - 10 + digit); + } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java index 2ec078764bd..b753a9357bc 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomInvoke.java @@ -102,7 +102,7 @@ public String getMethodUsed() { public void addDependencies(List dependencyList) { String dependencyOwner = owner; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); String resolvedConcreteOwner = resolveConcreteInvokeOwner(bc, true); if (resolvedConcreteOwner != null) { dependencyOwner = resolvedConcreteOwner; @@ -131,7 +131,7 @@ public void addDependencies(List dependencyList) { if(origOpcode != Opcodes.INVOKEINTERFACE && origOpcode != Opcodes.INVOKEVIRTUAL) { return; } - bld.append(owner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(owner)); bld.append("_"); if(name.equals("")) { bld.append("__INIT__"); @@ -177,7 +177,7 @@ private String resolveConcreteInvokeOwner(ByteCodeClass ownerClass, boolean allo if (currentClass != null && (ownerName.equals(currentClass) || currentClass.startsWith(ownerName + "_"))) { return null; } - ByteCodeClass concreteClass = Parser.getClassObject(ownerClass.getConcreteClass().replace('/', '_').replace('$', '_')); + ByteCodeClass concreteClass = Parser.getClassObject(Util.mangle(ownerClass.getConcreteClass())); // The nearest class in the concrete type's own hierarchy that actually // declares the method -- which is what the runtime would dispatch to for // an instance of it. Resolving against concreteClass's declarations alone @@ -288,7 +288,7 @@ public boolean appendExpression(StringBuilder b) { // so we need to check boolean isVirtual = true; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -321,13 +321,13 @@ public boolean appendExpression(StringBuilder b) { if(origOpcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -343,7 +343,7 @@ public boolean appendExpression(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // keep in sync with Invoke: direct/devirtualized calls of the mapped // String/StringBuilder natives get the inlined fast path @@ -442,7 +442,7 @@ private boolean tryAppendInlinedConstructor(StringBuilder b) { // Memset elimination: allocate into a temp, build fully, THEN publish. // Literal-arg ctor with the receiver on-stack (from NEW;DUP): the // survivor sits one slot below the receiver (SP[-2]); pop the receiver. - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); inlineCtorPlan.appendInitBeforePublish(b, cType, argExprs, argCats, 2, 1); return true; } @@ -489,7 +489,7 @@ private void appendFusedAllocBlock(StringBuilder b) { for (int i = 0; i < kids.size(); i++) { lenExprs[i] = kids.get(i).siteLengthExpr(temps); } - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); fusedPlan.appendFusedAlloc(b, cType, lenExprs, 1, 2); // NOTE: the enclosing brace is closed AFTER the ordinary call emission by // appendInstruction (the temps must stay in scope for the call). @@ -540,7 +540,7 @@ public void appendInstruction(StringBuilder b) { // so we need to check boolean isVirtual = true; if (origOpcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -573,13 +573,13 @@ public void appendInstruction(StringBuilder b) { if(origOpcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -595,7 +595,7 @@ public void appendInstruction(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // keep in sync with Invoke: direct/devirtualized calls of the mapped // String/StringBuilder natives get the inlined fast path diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java index b93718c8429..07ab36aa1e9 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/CustomJump.java @@ -68,13 +68,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(TryCatch.isTryCatchInMethod()) { b.append("JUMP_TO(label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(label, instructions)); b.append(");\n"); } else { b.append("goto label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(";\n"); } if(customSuffix != null) { diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java index f535414024a..95147ccf367 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Field.java @@ -23,6 +23,8 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.Util; + import java.util.List; import org.objectweb.asm.Opcodes; @@ -79,7 +81,7 @@ public void addDependencies(List dependencyList) { } public String getFieldFromThis() { - return "get_field_" + owner.replace('/', '_').replace('$', '_') + + return "get_field_" + Util.mangle(owner) + "_" + name + "(__cn1ThisObject)"; } @@ -88,14 +90,14 @@ public String setFieldFromThis(int arg) { // Instance field setters only need value/target operands. // special case for this if(arg == 0) { - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1ThisObject, __cn1ThisObject);\n"; } if(isObject()) { - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1Arg" + arg + ", __cn1ThisObject);\n"; } - return " set_field_" + owner.replace('/', '_').replace('$', '_') + + return " set_field_" + Util.mangle(owner) + "_" + name + "(__cn1Arg" + arg + ", __cn1ThisObject);\n"; } @@ -124,7 +126,7 @@ public String pushFieldFromThis() { break; } b.append("(get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); b.append("(__cn1ThisObject));\n"); @@ -143,14 +145,14 @@ public boolean assignTo(String varName, StringBuilder sb) { } if (opcode == Opcodes.GETSTATIC) { b.append("get_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); b.append("()"); } else { b.append("get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); StringBuilder sb3 = new StringBuilder(); @@ -224,17 +226,17 @@ public void appendInstruction(StringBuilder sbOut) { break; } b.append("(get_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); b.append("());\n"); break; case Opcodes.PUTSTATIC: { //b.append("SAFE_RETAIN(1);\n "); b.append("set_static_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); - b.append(name.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(name)); if (isObject()) { b.append("(threadStateData, "); } else { @@ -300,7 +302,7 @@ public void appendInstruction(StringBuilder sbOut) { } b.append("(get_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); @@ -317,7 +319,7 @@ public void appendInstruction(StringBuilder sbOut) { case Opcodes.PUTFIELD: { //b.append("SAFE_RETAIN(1);\n "); b.append("set_field_"); - b.append(owner.replace('/', '_').replace('$', '_')); + b.append(Util.mangle(owner)); b.append("_"); b.append(name); b.append("("); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java index f722f2fcab6..ab71a78e33f 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedConstructor.java @@ -554,8 +554,10 @@ private static boolean descMatchesArrayType(String fieldDesc, int arrayType) { public void appendFusedAlloc(StringBuilder b, String cType, String[] lenExprs, int recvSlot, int survSlot) { b.append(" { /* FUSED construction of ").append(cType).append(" */\n"); - b.append(" if(__builtin_expect(!class__").append(cType) - .append(".initialized, 0)) __STATIC_INITIALIZER_").append(cType).append("(threadStateData);\n"); + // ACQUIRE; see the note in TypeInstruction. + b.append(" if(__builtin_expect(!__atomic_load_n(&class__").append(cType) + .append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_").append(cType) + .append("(threadStateData);\n"); for (int i = 0; i < children.size(); i++) { b.append(" int __fLen").append(i).append(" = ").append(lenExprs[i]).append(";\n"); } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedFieldInit.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedFieldInit.java index 5814aca5ed3..a717d485b9e 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedFieldInit.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/FusedFieldInit.java @@ -56,9 +56,27 @@ public void appendInstruction(StringBuilder b) { String field = owner + "_" + child.getFieldName(); String lhs = "((struct obj__" + owner + "*)__cn1ThisObject)->" + field; b.append(" if(").append(lhs).append(" == JAVA_NULL) { /* fused field: not pre-installed */\n"); - b.append(" ").append(lhs).append(" = allocArray(threadStateData, ") + // Through a TEMPORARY, so the barrier can see the value before the field does. + // + // This path publishes an INDEPENDENT array into a field of an already-existing + // object, and a raw C assignment takes neither half of the store barrier that the + // ordinary PUTFIELD path emits. That is not cosmetic. CN1_WRITE_BARRIER is the + // SATB insertion half, so without it an array allocated and installed during a + // concurrent mark is recorded nowhere and can be swept while the field still + // points at it -- the same hazard cloneArray and System.arraycopy each carry an + // explicit bulk barrier for. Under a generational build it is also the promotion + // hook: the array is freshly allocated and therefore young, the owner is not, and + // a verifier over the self-hosting corpus reported exactly this shape 1552 times + // (BiBOP owner -> young Object[]/Label[]/Frame[]) before this line existed. + // + // FusedConstructor's own children need no barrier and get none: those arrays are + // carved out of the OWNER'S block by cn1FusedInstallPrimArray and have no + // independent GC identity. This one calls allocArray, so it does. + b.append(" JAVA_OBJECT __cn1ffi = allocArray(threadStateData, ") .append(child.ctorLengthExpr()).append(", ").append(child.arrayClassRef()) .append(", sizeof(").append(child.elemCType()).append("), 1);\n"); + b.append(" CN1_WRITE_BARRIER(__cn1ThisObject, __cn1ffi);\n"); + b.append(" ").append(lhs).append(" = __cn1ffi;\n"); b.append(" }\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java index 4f7adfa789c..7d590fbdac3 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Invoke.java @@ -91,7 +91,7 @@ private String getCMethodName() { public void addDependencies(List dependencyList) { String dependencyOwner = owner; if (opcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); String resolvedConcreteOwner = resolveConcreteInvokeOwner(bc, true); if (resolvedConcreteOwner != null) { dependencyOwner = resolvedConcreteOwner; @@ -121,7 +121,7 @@ public void addDependencies(List dependencyList) { if(opcode != Opcodes.INVOKEINTERFACE && opcode != Opcodes.INVOKEVIRTUAL) { return; } - bld.append(owner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(owner)); bld.append("_"); if(name.equals("")) { bld.append("__INIT__"); @@ -167,7 +167,7 @@ private String resolveConcreteInvokeOwner(ByteCodeClass ownerClass, boolean allo if (currentClass != null && (ownerName.equals(currentClass) || currentClass.startsWith(ownerName + "_"))) { return null; } - ByteCodeClass concreteClass = Parser.getClassObject(ownerClass.getConcreteClass().replace('/', '_').replace('$', '_')); + ByteCodeClass concreteClass = Parser.getClassObject(Util.mangle(ownerClass.getConcreteClass())); // The nearest class in the concrete type's own hierarchy that actually // declares the method -- which is what the runtime would dispatch to for // an instance of it. Resolving against concreteClass's declarations alone @@ -227,7 +227,7 @@ private void appendFusedAllocBlock(StringBuilder b) { for (int i = 0; i < kids.size(); i++) { lenExprs[i] = kids.get(i).siteLengthExpr(argExprByParam); } - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); fusedPlan.appendFusedAlloc(b, cType, lenExprs, n + 1, n + 2); } @@ -261,7 +261,7 @@ private boolean tryAppendInlinedConstructor(StringBuilder b) { // argCats == null: every argExpr here is a pure SP[-k].data.x read // (the args were evaluated onto the operand stack BEFORE this ), // so no temp hoisting is needed. - String cType = owner.replace('/', '_').replace('$', '_'); + String cType = Util.mangle(owner); inlineCtorPlan.appendInitBeforePublish(b, cType, argExprs, null, n + 2, n + 1); return true; } @@ -307,7 +307,7 @@ public void appendInstruction(StringBuilder b) { // if it is. boolean isVirtual = true; if (opcode == Opcodes.INVOKEVIRTUAL) { - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { System.err.println("WARNING: Failed to find class object for owner "+owner+" when rendering virtual method "+name); } else { @@ -340,7 +340,7 @@ public void appendInstruction(StringBuilder b) { if(opcode == Opcodes.INVOKESTATIC) { // find the actual class of the static method to work around javac not defining it correctly - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); invokeOwner = findActualOwner(bc); } if (invokeOwner.startsWith("[")) { @@ -348,7 +348,7 @@ public void appendInstruction(StringBuilder b) { // as an owner. We'll just change this to java_lang_Object instead. bld.append("java_lang_Object"); } else{ - bld.append(invokeOwner.replace('/', '_').replace('$', '_')); + bld.append(Util.mangle(invokeOwner)); } bld.append("_"); if(name.equals("")) { @@ -364,7 +364,7 @@ public void appendInstruction(StringBuilder b) { ArrayList args = new ArrayList<>(); String returnVal = BytecodeMethod.appendMethodSignatureSuffixFromDesc(desc, bld, args); if (isVirtualCall) { - BytecodeMethod.addVirtualMethodsInvoked(bld.substring("virtual_".length())); + BytecodeMethod.addVirtualMethodsInvoked(bld.toString().substring("virtual_".length())); } else { // direct/devirtualized calls of the hottest String/StringBuilder // natives get the call-site-inlined fast path (cn1_intrinsics.h) @@ -501,7 +501,7 @@ public void appendInstruction(StringBuilder b) { // Master off-switch: -DCN1_DISABLE_INLINE=true disables trivial-method inlining. private static final boolean DISABLE_INLINE = - "true".equalsIgnoreCase(System.getProperty("CN1_DISABLE_INLINE", "false")); + "true".equalsIgnoreCase(Util.getProperty("CN1_DISABLE_INLINE", "false")); /** * If this invoke is a direct (provably monomorphic) instance call to a trivial @@ -523,7 +523,7 @@ public Field asInlinableFieldAccess() { if (desc.length() < 3 || desc.charAt(0) != '(' || desc.charAt(1) != ')' || desc.charAt(2) == 'V') { return null; } - BytecodeMethod target = findMethodUp(Parser.getClassObject(owner.replace('/', '_').replace('$', '_'))); + BytecodeMethod target = findMethodUp(Parser.getClassObject(Util.mangle(owner))); if (target == null || !target.isStatic()) { return null; } @@ -586,10 +586,10 @@ public Field asInlinableFieldAccess() { */ private BytecodeMethod resolveDirectTarget() { if (opcode == Opcodes.INVOKESPECIAL) { - return findMethodUp(Parser.getClassObject(owner.replace('/', '_').replace('$', '_'))); + return findMethodUp(Parser.getClassObject(Util.mangle(owner))); } // INVOKEVIRTUAL - ByteCodeClass bc = Parser.getClassObject(owner.replace('/', '_').replace('$', '_')); + ByteCodeClass bc = Parser.getClassObject(Util.mangle(owner)); if (bc == null) { return null; } @@ -601,7 +601,7 @@ private BytecodeMethod resolveDirectTarget() { if (rc == null) { return null; // genuinely virtual -> target not fixed -> unsafe to inline } - return findMethodUp(Parser.getClassObject(rc.replace('/', '_').replace('$', '_'))); + return findMethodUp(Parser.getClassObject(Util.mangle(rc))); } /** @@ -629,7 +629,7 @@ private static BytecodeMethod trivialStaticForwarderTarget(BytecodeMethod m) { if (rc != Opcodes.IRETURN && rc != Opcodes.LRETURN && rc != Opcodes.FRETURN && rc != Opcodes.DRETURN && rc != Opcodes.ARETURN) return null; BytecodeMethod t = inner.findMethodUp(Parser.getClassObject( - inner.owner.replace('/', '_').replace('$', '_'))); + Util.mangle(inner.owner))); return (t != null && t.isStatic()) ? t : null; } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java index cc0594d406d..718b0c21c24 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Jump.java @@ -112,13 +112,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(TryCatch.isTryCatchInMethod()) { b.append("JUMP_TO(label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(label, instructions)); b.append(");\n"); } else { b.append("goto label_"); - b.append(label.toString()); + b.append(LabelInstruction.labelName(label)); b.append(";\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java index 0c3d85e1ceb..f7c228030c7 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/LabelInstruction.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.Hashtable; import java.util.List; import java.util.Map; @@ -56,6 +57,55 @@ static class Pair { // a lot of strings. private static Map usedLabels = new Hashtable(); + /** + * Stable names for the C labels generated from ASM labels. + * + * These used to be {@code Label.toString()}, which ASM defines as + * {@code "L" + System.identityHashCode(this)}. That made the emitted C depend on + * identity hash codes, with two consequences. It is not reproducible -- nothing + * promises an identity hash is stable. And it is not even VALID on a runtime + * whose identity hash can be negative: ParparVM's is the object pointer narrowed + * to int, so a self-hosted translator emitted {@code label_L-180306432001}, which + * C reads as a subtraction, and every method with a try/catch failed to compile. + * + * Numbering is per method and assigned in bytecode order as the labels are + * visited, so a method's C depends only on that method. A global counter would + * work too, but it would make every method downstream of any change renumber, + * which turns one real difference into thousands when two outputs are compared. + * + * C labels are function-scoped, so the same name in two methods is not a clash. + * + * An IdentityHashMap because Label overrides neither equals nor hashCode, and two + * distinct labels must never share a name. + */ + private static final Map labelNames = new IdentityHashMap(); + + /** + * Names {@code l} as the {@code index}th label of its method. Called from + * BytecodeMethod.addLabel while the method is being parsed. + */ + public static void assignLabelName(Label l, int index) { + if (!labelNames.containsKey(l)) { + labelNames.put(l, "L" + index); + } + } + + /** + * The C label name for {@code l}. + * + * Every label reaching emission has been through addLabel, so the fallback is + * unreachable; it is spelled with a distinct prefix so that if it ever does fire + * it cannot collide with a real per-method name. + */ + public static String labelName(Label l) { + String name = labelNames.get(l); + if (name == null) { + name = "Lx" + labelNames.size(); + labelNames.put(l, name); + } + return name; + } + // cleanup between passes, free the garbage! public static void cleanup() { @@ -63,6 +113,7 @@ public static void cleanup() tryEndLabels.clear(); labelCatchDepth.clear(); usedLabels.clear(); + labelNames.clear(); } public LabelInstruction(org.objectweb.asm.Label parent) { super(-1); @@ -160,7 +211,7 @@ public void appendInstruction(StringBuilder b) { return; } b.append("\nlabel_"); - b.append(parent); + b.append(labelName(parent)); b.append(":\n"); Integer tryCount = tryEndLabels.get(parent); if(tryCount != null) { @@ -181,19 +232,19 @@ public void appendInstruction(StringBuilder b) { for(int iter = strs.size() - 1; iter >= 0 ; iter--) { Pair s = strs.get(iter); b.append(" tryBlockOffset"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); b.append(" = threadStateData->tryBlockOffset;\n"); b.append(" BEGIN_TRY("); b.append(s.cls); b.append(", catch_"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); //b.append("); NSLog(@\"Begin try on: %s %d off: %i\\n\", __FILE__, __LINE__, getThreadLocalData()->tryBlockOffset);"); b.append(");\n restoreTo"); - b.append(parent); + b.append(labelName(parent)); b.append(s.cls); b.append(s.counter); b.append(" = threadStateData->threadObjectStackOffset;\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java index 9f9e297d37b..551dc3629e9 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/Ldc.java @@ -23,6 +23,8 @@ package com.codename1.tools.translator.bytecodes; +import com.codename1.tools.translator.Util; + import com.codename1.tools.translator.ByteCodeClass; import com.codename1.tools.translator.Parser; import java.util.List; @@ -57,7 +59,7 @@ public void addDependencies(List dependencyList) { int sort = ((Type) cst).getSort(); Type tp = (Type) cst; if (sort == Type.OBJECT) { - String t = tp.getInternalName().replace('/', '_').replace('$', '_'); + String t = Util.mangle(tp.getInternalName()); if(!dependencyList.contains(t)) { dependencyList.add(t); } @@ -75,7 +77,7 @@ public void addDependencies(List dependencyList) { case Type.SHORT: return; } - String t = ttt.getInternalName().replace('/', '_').replace('$', '_'); + String t = Util.mangle(ttt.getInternalName()); ByteCodeClass.addArrayType(t, tp.getDimensions()); if(!dependencyList.contains(t)) { dependencyList.add(t); @@ -159,15 +161,15 @@ public String getValueAsString() { Type tp = (Type) cst; if (sort == Type.OBJECT) { //b.append("/* LDC: '"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class__"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append(");\n"); b.append("(JAVA_OBJECT)&class__"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); } else if (sort == Type.ARRAY) { //b.append("/* LDC Array: '"); - //b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + //b.append(Util.mangle(tp.getInternalName())); //b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class_array"); b.append("(JAVA_OBJECT)&class_array"); b.append(tp.getDimensions()); @@ -199,7 +201,7 @@ public String getValueAsString() { b.append("JAVA_SHORT"); break; default: - b.append(ttt.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(ttt.getInternalName())); break; } //b.append(");\n"); @@ -283,13 +285,13 @@ public void appendInstruction(StringBuilder b) { Type tp = (Type) cst; if (sort == Type.OBJECT) { b.append("/* LDC: '"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class__"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append(");\n"); } else if (sort == Type.ARRAY) { b.append("/* LDC Array: '"); - b.append(tp.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(tp.getInternalName())); b.append("'*/\n PUSH_POINTER((JAVA_OBJECT)&class_array"); b.append(tp.getDimensions()); b.append("__"); @@ -320,7 +322,7 @@ public void appendInstruction(StringBuilder b) { b.append("JAVA_SHORT"); break; default: - b.append(ttt.getInternalName().replace('/', '_').replace('$', '_')); + b.append(Util.mangle(ttt.getInternalName())); break; } b.append(");\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java index 711b8d20867..7858d24321d 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/SwitchInstruction.java @@ -55,13 +55,13 @@ public void appendInstruction(StringBuilder b, List instructions) { b.append(keys[iter]); if(TryCatch.isTryCatchInMethod()) { b.append(": JUMP_TO(label_"); - b.append(labels[iter].toString()); + b.append(LabelInstruction.labelName(labels[iter])); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(labels[iter], instructions)); b.append(");\n"); } else { b.append(": goto label_"); - b.append(labels[iter].toString()); + b.append(LabelInstruction.labelName(labels[iter])); b.append(";\n"); } } @@ -69,13 +69,13 @@ public void appendInstruction(StringBuilder b, List instructions) { if(dflt != null) { if(TryCatch.isTryCatchInMethod()) { b.append(" default: JUMP_TO(label_"); - b.append(dflt.toString()); + b.append(LabelInstruction.labelName(dflt)); b.append(", "); b.append(LabelInstruction.getLabelCatchDepth(dflt, instructions)); b.append(");\n"); } else { b.append(" default: goto label_"); - b.append(dflt.toString()); + b.append(LabelInstruction.labelName(dflt)); b.append(";\n"); } } diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java index a097c80b5b1..d36cf7ad31a 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TryCatch.java @@ -109,21 +109,21 @@ public void appendInstruction(StringBuilder b, List instructions) { // threadObjectStackOffset from trash and later callee frames were // allocated on top of this frame's locals. clang happened to spill. b.append(" volatile int restoreTo"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(";\n volatile int tryBlockOffset"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(";\n DEFINE_CATCH_BLOCK(catch_"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(", label_"); - b.append(handler); + b.append(LabelInstruction.labelName(handler)); b.append(", restoreTo"); - b.append(start); + b.append(LabelInstruction.labelName(start)); b.append(cid); b.append(counter); b.append(");\n"); diff --git a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java index 8eb39cf8a35..feb8f2095aa 100644 --- a/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java +++ b/vm/ByteCodeTranslator/src/com/codename1/tools/translator/bytecodes/TypeInstruction.java @@ -254,9 +254,25 @@ public void appendInstruction(StringBuilder b, List l) { // reaches it as a root (its pointer rides the operand stack) and // scans its fields, so any heap objects it references stay live. // It is never freed; it simply dies when the frame unwinds. - b.append("if(__builtin_expect(!class__"); + // NOTE: this guard necessarily tests class__X.initialized rather + // than __X_LOADED__, because X is a DIFFERENT class from the one + // being emitted and __X_LOADED__ is file-local to X's own + // translation unit. initialized is set before __CLINIT__ runs, so + // this can still enter the allocation while X's is in + // flight -- pre-existing, and the reason the guards emitted from + // ByteCodeClass (same translation unit) use the completion flag + // instead. Closing it here needs a globally visible completion + // flag on struct clazz, which is a larger change than this. + // + // ACQUIRE: this guard SKIPS the initialiser when the flag is + // set, so it never takes the class monitor and cannot rely on + // the monitor's release. Pairs with the __ATOMIC_RELEASE store + // in ByteCodeClass. A plain load here let a thread see the flag + // set while the vtable / classToInterfaceMap rows it describes + // were still invisible. + b.append("if(__builtin_expect(!__atomic_load_n(&class__"); b.append(type); - b.append(".initialized, 0)) __STATIC_INITIALIZER_"); + b.append(".initialized, __ATOMIC_ACQUIRE), 0)) __STATIC_INITIALIZER_"); b.append(type); b.append("(threadStateData); memset(&__cn1stk_"); b.append(stackAllocId); diff --git a/vm/ByteCodeTranslator/src/java_io_File.m b/vm/ByteCodeTranslator/src/java_io_File.m index 2cd51c822c4..729d2e6c5ad 100644 --- a/vm/ByteCodeTranslator/src/java_io_File.m +++ b/vm/ByteCodeTranslator/src/java_io_File.m @@ -133,9 +133,17 @@ JAVA_OBJECT java_io_File_listImpl___java_lang_String_R_java_lang_String_1ARRAY(C type check, and it hands the collector String metadata for an array payload. cn1MainArgs has always used the array class; these three did not. Fixed on all of them, including the two that predate the Windows arm. */ - JAVA_OBJECT arr = allocArray(threadStateData, [files count], &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); - - for (int i=0; i<[files count]; i++) { + /* [files count] is NSUInteger -- 64-bit -- while allocArray's length and the + element setter's index are JAVA_INT. Narrow ONCE and explicitly, and loop on + the narrowed value so the bound and the index have the same type. The + implicit conversion this replaces is what the native warning census caught, + and it only became visible when this file started being compiled at all: the + translated java_io_File.c used to overwrite it on the Apple targets, which is + the collision fixed earlier on this branch. */ + JAVA_INT fileCount = (JAVA_INT)[files count]; + JAVA_OBJECT arr = allocArray(threadStateData, fileCount, &class_array1__java_lang_String, sizeof(JAVA_OBJECT), 1); + + for (JAVA_INT i = 0; i < fileCount; i++) { NSString* f = [files objectAtIndex:i]; JAVA_OBJECT s = fromNSString(CN1_THREAD_STATE_PASS_ARG f); CN1_SET_ARRAY_ELEMENT_OBJECT(arr, i, s); diff --git a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js index 9d89c5ce3c3..5f471960bee 100644 --- a/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js +++ b/vm/ByteCodeTranslator/src/javascript/parparvm_runtime.js @@ -5885,6 +5885,16 @@ bindNative(["cn1_java_lang_Class_getComponentType_R_java_lang_Class"], function( } return classObjectForName(def.componentClass); }); +// A browser has no process environment, so getenv ANSWERS null rather than +// failing. Without this the symbol falls through to the unsupported-native path, +// which emits `throw new Error("environment variables are not available...")` -- +// and Class.getResourceAsStream consults CN1_RESOURCE_PATH, so a JavaScript +// application asking for a resource got an exception where it previously got +// null. Returning null is both the safe answer and the correct one: the variable +// genuinely is not set. +bindNative(["cn1_java_lang_System_getenvImpl_java_lang_String_R_java_lang_String"], function(name) { + return null; +}); bindNative(["cn1_java_lang_Class_isPrimitive_R_boolean"], function(__cn1ThisObject) { return __cn1ThisObject.__classDef && __cn1ThisObject.__classDef.isPrimitive ? 1 : 0; }); bindNative(["cn1_java_lang_reflect_Array_newInstanceImpl_java_lang_Class_int_R_java_lang_Object"], function(componentClass, length) { if (!componentClass || !componentClass.__classDef) { diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4e88a1d8ebc..ee8f4f0f199 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -1030,6 +1030,33 @@ JAVA_VOID java_lang_System_arraycopy___java_lang_Object_int_java_lang_Object_int * heap corruption on the arm64 clean target). memmove is the correct, * overlap-safe primitive. */ memmove( (*dstArr).data + (dstOffset * byteSize), (*srcArr).data + (srcOffset * byteSize), length * byteSize); +#ifdef CN1_NURSERY + // THE NURSERY BARRIER, which the bulk copy above bypasses exactly as it bypasses the + // two SATB halves -- and for the same reason: no per-element setter runs, so + // CN1_WRITE_BARRIER never fires. The nursery's whole safety argument is that a heap + // object can never reference a nursery object, because any store that would create + // such a reference promotes the value first. A bulk copy of references into a heap + // array breaks that invariant silently. + // + // It is not a theoretical hole: ArrayList.grow copies its backing array through here, + // and with it unpatched a self-hosted translation faults inside ArrayList's iterator + // after ~8 minor collections -- i.e. as soon as the arena has wrapped once and the + // block holding the unpromoted element has been handed out again. Before the first + // wrap the dangling reference still points at intact memory and nothing is observed, + // which is why this survives any short run. + // + // Only when the DESTINATION is outside the nursery: a nursery-to-nursery copy keeps + // both ends in the young generation, which is the case promotion exists to avoid. + if(!cls->primitiveType && !cn1IsYoungObject(dst)) { + JAVA_ARRAY_OBJECT* cn1__d = ((JAVA_ARRAY_OBJECT*)(*dstArr).data) + dstOffset; + int cn1__i; + for(cn1__i = 0 ; cn1__i < length ; cn1__i++) { + if(cn1__d[cn1__i] != JAVA_NULL) { + cn1NurseryWriteBarrier(dst, (JAVA_OBJECT)cn1__d[cn1__i]); + } + } + } +#endif if(cn1__satbReg) { cn1SatbBulkEnd(); } @@ -1463,16 +1490,6 @@ JAVA_LONG java_lang_Double_doubleToLongBits___double_R_long(CODENAME_ONE_THREAD_ return u.l; } -JAVA_LONG java_lang_Double_doubleToRawLongBits___double_R_long(CODENAME_ONE_THREAD_STATE, JAVA_DOUBLE n1) { - union { - JAVA_DOUBLE d; - JAVA_LONG l; - } u; - - u.d = n1; - return u.l; -} - JAVA_FLOAT java_lang_Float_intBitsToFloat___int_R_float(CODENAME_ONE_THREAD_STATE, JAVA_INT n1) { union { @@ -1973,6 +1990,7 @@ JAVA_OBJECT java_lang_Class_getName___R_java_lang_String(CODENAME_ONE_THREAD_STA return newStringFromCString(threadStateData, clz->clsName); } + JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls) { struct clazz* clz = (struct clazz*)cls; return clz->isArray; @@ -1988,6 +2006,12 @@ JAVA_BOOLEAN java_lang_Class_isArray___R_boolean(CODENAME_ONE_THREAD_STATE, JAVA JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT cls2) { struct clazz* clz1 = (struct clazz*)cls; struct clazz* clz2 = (struct clazz*)cls2; + // A primitive class carries CN1_PRIMITIVE_CLASS_ID, which indexes no row of + // the instanceof tables, so it must never reach instanceofFunction. The JDK + // rule is also simply identity: int is assignable only from int. + if(clz1->primitiveType || clz2->primitiveType) { + return clz1 == clz2 ? JAVA_TRUE : JAVA_FALSE; + } // A.isAssignableFrom(B): target is A, the class under test is B. return instanceofFunction(clz1->classId, clz2->classId); } @@ -1995,6 +2019,9 @@ JAVA_BOOLEAN java_lang_Class_isAssignableFrom___java_lang_Class_R_boolean(CODENA JAVA_BOOLEAN java_lang_Class_isInstance___java_lang_Object_R_boolean(CODENAME_ONE_THREAD_STATE, JAVA_OBJECT cls, JAVA_OBJECT obj) { if(obj == JAVA_NULL) { return JAVA_FALSE; } struct clazz* clz1 = (struct clazz*)cls; + // No object is ever an instance of a primitive class, and its sentinel + // classId indexes no instanceof table row -- see isAssignableFrom above. + if(((struct clazz*)cls)->primitiveType) { return JAVA_FALSE; } struct clazz* clz2 = (struct clazz*)CN1_CLASS_OF(obj); // tag-aware: a tagged Integer has no header // A.isInstance(o): target is A, the class under test is o's class. These were // reversed, so isInstance searched the TARGET's supertype table for the @@ -2537,6 +2564,31 @@ JAVA_VOID java_lang_System_gcLight__(CODENAME_ONE_THREAD_STATE) { int cn1GcProbeThrew = 0; #endif JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { + // FREEZE: refuse to start a cycle at all once the exit census has claimed the + // heap. Clearing System.gcShouldLoop is not sufficient on its own -- the GC + // thread may already have evaluated `while(gcShouldLoop)` and be on its way + // here, and System's start-up path re-raises that flag after its initial wait. + // Either way the census would see gcCurrentlyRunning false, start walking, and + // have the pending cycle resume and sweep underneath it. Checked here because + // this is the one door every cycle comes through. + // CLAIM the cycle, do not merely check a flag. Loading a freeze flag and then + // setting gcCurrentlyRunning is two steps, and the collector can be preempted + // between them: the census would raise the freeze, see gcCurrentlyRunning still + // false, and start walking a heap this thread is about to sweep. The claim below + // is a single compare-exchange, so a cycle is either started or refused with + // nothing observable in between. + { + int cn1Expected = CN1_GC_CYCLE_IDLE; + if(!atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &cn1Expected, + CN1_GC_CYCLE_RUNNING, memory_order_acq_rel, memory_order_acquire)) { + // In practice only the FROZEN case can be taken: System's GC thread is + // the sole caller (System.java's `while(gcShouldLoop)` loop), so no second + // entrant can observe RUNNING. Refusing on RUNNING too is defence rather + // than policy -- two concurrent cycles would be worse than a skipped one -- + // and it means this is not a behaviour change for any existing caller. + return; + } + } gcCurrentlyRunning = JAVA_TRUE; if(firstTimeGcThread) { firstTimeGcThread = JAVA_FALSE; @@ -2662,6 +2714,14 @@ JAVA_VOID java_lang_System_gcMarkSweep__(CODENAME_ONE_THREAD_STATE) { // of malloc entirely for exactly this reason. lowMemoryMode = JAVA_FALSE; gcCurrentlyRunning = JAVA_FALSE; + // Release the claim. Only ever RUNNING -> IDLE: a census that froze while this + // cycle ran holds the state at FROZEN and this must not clobber it, which is why + // the transition is a compare-exchange rather than a store. + { + int cn1Running = CN1_GC_CYCLE_RUNNING; + atomic_compare_exchange_strong_explicit(&cn1GcCycleState, &cn1Running, + CN1_GC_CYCLE_IDLE, memory_order_acq_rel, memory_order_relaxed); + } } JAVA_VOID java_lang_System_exit___int(CODENAME_ONE_THREAD_STATE, JAVA_INT i) { diff --git a/vm/JavaAPI/src/java/lang/Boolean.java b/vm/JavaAPI/src/java/lang/Boolean.java index 043fee9956d..62c36722b01 100644 --- a/vm/JavaAPI/src/java/lang/Boolean.java +++ b/vm/JavaAPI/src/java/lang/Boolean.java @@ -27,6 +27,28 @@ * Since: JDK1.0, CLDC 1.0 */ public final class Boolean implements Comparable { + + /** + * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. + */ + public static final Class TYPE = null; + /** * The Boolean object corresponding to the primitive value false. */ diff --git a/vm/JavaAPI/src/java/lang/Class.java b/vm/JavaAPI/src/java/lang/Class.java index 00b5f6466ec..79da8c77e3b 100644 --- a/vm/JavaAPI/src/java/lang/Class.java +++ b/vm/JavaAPI/src/java/lang/Class.java @@ -40,6 +40,11 @@ public final class Class implements java.lang.reflect.Type { public ClassLoader getClassLoader() { + if (isPrimitive()) { + // A primitive class is bootstrap-defined and must report null, which is + // what reflection code tests to tell such a type from a loaded one. + return null; + } return ClassLoader.getSystemClassLoader(); } @@ -50,6 +55,40 @@ public ClassLoader getClassLoader() { * following code fragment returns the runtime Class descriptor for the * class named java.lang.Thread: Classt= Class.forName("java.lang.Thread") */ + /** + * Returns the Class object for {@code className}. + * + * ParparVM links the whole program ahead of time, so there is no second class + * loader to consult: both extra arguments are accepted and ignored, and the + * class is resolved exactly as the one-argument form resolves it. The overload + * exists because library bytecode calls it -- ASM's + * ClassWriter.getCommonSuperClass does -- and an absent overload is a link + * error in translated code, not a compile error here. + * + * <p>What {@code initialize == true} does NOT do here: it does not run the + * named class's static initializer. ParparVM runs one on first use -- the + * generated code calls the class's static initializer at every NEW, GETSTATIC + * and INVOKESTATIC -- so any code that goes on to TOUCH the class sees its + * statics initialized as normal. What does not work is using forName purely for + * a registration side effect and never referencing the class again, the + * JDBC-driver idiom. That pattern cannot work on this platform for a second + * reason anyway: obfuscation rewrites class names, so a name looked up as a + * string does not survive a release build. + * + * <p>WHY IT IS NOT IMPLEMENTED, rather than left as an oversight: forcing the + * initializer needs a way to reach it from a Class object, and {@code struct + * clazz} carries no static-initializer function pointer -- only newInstanceFp + * and enumValueOfFp. Adding one is a field on EVERY class in EVERY application, + * to serve a flag whose only in-tree caller is ASM, which passes + * {@code initialize = false}. The cost is paid by every app and the benefit is + * claimed by none, so this stays documented rather than built. If a real caller + * ever needs it, emit the pointer then. + */ + public static java.lang.Class forName(java.lang.String className, boolean initialize, + ClassLoader loader) throws java.lang.ClassNotFoundException { + return forName(className); + } + public static java.lang.Class forName(java.lang.String className) throws java.lang.ClassNotFoundException { className = className.replace('$', '.'); Class c = forNameImpl(className); @@ -136,7 +175,180 @@ public static java.lang.Class forName(java.lang.String className) throws java.la * class upon which the getResourceAsStream method was called. */ public java.io.InputStream getResourceAsStream(java.lang.String name){ - return null; + if (name == null) { + return null; + } + String absolute = name; + if (!absolute.startsWith("/")) { + // Relative names resolve against this class's package, as the javadoc + // above describes. + // + // KNOWN LIMITATION, for a NESTED class only. getName() cannot be told + // apart from a package here, because ParparVM builds the runtime class + // name as clsName.replace('_', '.') in ByteCodeClass -- it starts from + // the MANGLED name, so the '$' that separates a nested class from its + // outer one arrives as a '.', and so does any '_' in a class's own + // name. Outer$Inner therefore reports "a.b.Outer.Inner" where the JDK + // reports "a.b.Outer$Inner", and the package derived below is + // "a.b.Outer" rather than "a.b". + // + // The consequence is a MISS, not a wrong file: the derived path is a + // directory named after a class, which a resource tree does not have, + // so the lookup returns null exactly as it did before this method was + // implemented. It is deliberately not patched up by walking shorter + // prefixes -- a package really can be named like a class, and that + // would turn today's miss into a confidently wrong hit. The fix + // belongs in the name the VM reports, which is a change to getName() + // for every translated application and wants its own testing. + // + // PUSHBACK, so the next reader does not re-open this: fixing it HERE + // means guessing where the package ends, and every guess is wrong for + // some real input -- a package may legitimately be named like a class, + // and a class name may legitimately contain '_'. A guess would convert + // today's harmless miss into a confident wrong answer. The defect is + // that getName() is lossy; it is fixed there or not at all. + String className = getName(); + int lastDot = className.lastIndexOf('.'); + absolute = lastDot < 0 ? "/" + name + : "/" + className.substring(0, lastDot).replace('.', '/') + "/" + name; + } + // Resources linked INTO the executable are deliberately not consulted here. + // + // CORRECTION, because the first version of this comment blamed the wrong + // thing: withdrawing this tier did NOT fix the ValidatorLightweightPicker + // screenshot difference, which persists without it. That is still an open + // question about this branch and the cause is elsewhere. + // + // The tier stays withdrawn on its own merits rather than that one. On + // master this method is `return null` on every ParparVM target, so no + // application has ever received anything from it and every caller has + // always taken its not-found path. Handing those callers a resource for the + // first time is a behaviour change for every shipping application, and it + // is a separate feature from self-hosting, which needs only the filesystem + // tier below. The javadoc this replaces claimed "nothing can regress, only + // start working" -- an assumption that every not-found path is strictly + // worse than the resource, which is not something this change established. + // + // The filesystem tier below stays, because it is OPT-IN: it answers only + // when CN1_RESOURCE_PATH names a search root, which no application sets and + // the self-hosted translator does. So an application sees exactly what it + // saw on master -- null -- and the translator can still find the C runtime + // it has to copy into its output. + // + // Letting applications read their own embedded resources is a good feature + // and wants its own change, where the screenshot baselines it moves can be + // reviewed as the point of the change rather than as fallout from one. + return cn1FileResource(absolute); + } + + + /** + * The filesystem half of {@link #getResourceAsStream}: looks the resource up + * under a search path, so a translated command-line program can read files that + * sit beside it rather than being linked into it. + * + * The path comes from CN1_RESOURCE_PATH, else a "cn1runtime" directory next to + * the executable. Entries are separated the way the platform separates path + * entries. + */ + private static java.io.InputStream cn1FileResource(String absolute) { + String path = System.getenv("CN1_RESOURCE_PATH"); + if (path == null || path.length() == 0) { + return null; + } + String relative = absolute.substring(1); + // A resource name is not a path expression. Refusing any ".." segment keeps + // a lookup inside the search root it was found under; without it a name + // like "../../etc/passwd" reads straight out of the filesystem, and the + // caller is usually passing a name that came from data. + if (relative.length() == 0 || cn1EscapesRoot(relative)) { + return null; + } + int from = 0; + while (from <= path.length()) { + int end = cn1PathEntryEnd(path, from); + String root = end < 0 ? path.substring(from) : path.substring(from, end); + if (root.length() > 0) { + java.io.File candidate = new java.io.File(root, relative); + // isFile(), not exists(): a DIRECTORY with the requested name exists + // and cannot be opened, and returning on that would abandon the + // search. Later roots still get their turn, which is the point of + // having a search path at all -- an earlier root holding an + // unusable candidate must not mask a usable one behind it. + if (candidate.isFile()) { + try { + return new java.io.FileInputStream(candidate); + } catch (java.io.IOException err) { + // Unreadable here does not mean absent everywhere: keep going. + err = null; + } + } + } + if (end < 0) { + break; + } + from = end + 1; + } + return null; + } + + /** + * True when any segment of a resource-relative path is "..". + * + * A backslash counts as a separator as well as '/'. Resource names are + * '/'-separated by specification, but nothing stops a caller passing a Windows + * path, and there File("root", "..\\..\\x") escapes exactly as the '/' form + * does -- checking only '/' would leave the traversal open on the one platform + * whose separator it is. + */ + private static boolean cn1EscapesRoot(String relative) { + int from = 0; + for (int i = 0; i <= relative.length(); i++) { + boolean atEnd = i == relative.length(); + if (!atEnd && relative.charAt(i) != '/' && relative.charAt(i) != '\\') { + continue; + } + if (relative.substring(from, i).equals("..")) { + return true; + } + from = i + 1; + } + return false; + } + + /** + * The index that ends the search-path entry starting at {@code from}, or -1 for + * the last one. + * + * This cannot use {@code File.pathSeparatorChar}, which is a hard-coded ':' in + * this class library rather than a platform value -- on a native Windows build + * that splits "C:\\res;D:\\res" after the drive letter and every entry is + * nonsense. Both separators are therefore accepted, and a ':' is not a + * separator when it sits directly after a single-letter entry and is followed + * by a slash, which is exactly a DOS drive prefix and never a POSIX path. + */ + private static int cn1PathEntryEnd(String path, int from) { + for (int i = from; i < path.length(); i++) { + char c = path.charAt(i); + if (c == ';') { + return i; + } + if (c == ':') { + boolean driveLetter = i == from + 1 + && i + 1 < path.length() + && (path.charAt(i + 1) == '\\' || path.charAt(i + 1) == '/') + && cn1IsLetter(path.charAt(from)); + if (!driveLetter) { + return i; + } + } + } + return -1; + } + + /** ASCII letter test; Character.isLetter is locale-aware and not wanted here. */ + private static boolean cn1IsLetter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); } /** @@ -193,6 +405,12 @@ public java.io.InputStream getResourceAsStream(java.lang.String name){ * Creates a new instance of a class. */ public java.lang.Object newInstance() throws java.lang.InstantiationException, java.lang.IllegalAccessException { + if (isPrimitive()) { + // A primitive descriptor has no constructor, and its newInstanceFp is + // zero -- the native calls that pointer unconditionally, so letting one + // through jumps to address zero instead of throwing. + throw new InstantiationException(); + } Object o = newInstanceImpl(); if(o == null) { throw new InstantiationException(); @@ -211,6 +429,11 @@ public java.lang.Object newInstance() throws java.lang.InstantiationException, j * returns "void". */ public java.lang.String toString() { + if (isPrimitive()) { + // "int", not "int class" -- java.lang.Class documents the primitive form + // as the name alone. + return getName(); + } return getName() + " class"; } diff --git a/vm/JavaAPI/src/java/lang/Double.java b/vm/JavaAPI/src/java/lang/Double.java index 22f161feb5d..79a3e648abe 100644 --- a/vm/JavaAPI/src/java/lang/Double.java +++ b/vm/JavaAPI/src/java/lang/Double.java @@ -88,6 +88,14 @@ public byte byteValue(){ * If the argument is NaN, the result is 0x7ff8000000000000L. * In all cases, the result is a long integer that, when given to the longBitsToDouble(long) method, will produce a floating-point value equal to the argument to doubleToLongBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. See {@link Float#floatToRawIntBits} for why this delegates. + */ + public static long doubleToRawLongBits(double value) { + return doubleToLongBits(value); + } + public native static long doubleToLongBits(double value); /** diff --git a/vm/JavaAPI/src/java/lang/Float.java b/vm/JavaAPI/src/java/lang/Float.java index 5b257d00f64..fc940da7ff7 100644 --- a/vm/JavaAPI/src/java/lang/Float.java +++ b/vm/JavaAPI/src/java/lang/Float.java @@ -28,6 +28,28 @@ * Since: JDK1.0, CLDC 1.1 */ public final class Float extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. + */ + public static final Class TYPE = null; + /** * The largest positive value of type float. It is equal to the value returned by Float.intBitsToFloat(0x7f7fffff). * See Also:Constant Field Values @@ -113,6 +135,20 @@ public boolean equals(java.lang.Object obj){ * Returns the bit representation of a single-float value. The result is a representation of the floating-point argument according to the IEEE 754 floating-point "single precision" bit layout. Bit 31 (the bit that is selected by the mask 0x80000000) represents the sign of the floating-point number. Bits 30-23 (the bits that are selected by the mask 0x7f800000) represent the exponent. Bits 22-0 (the bits that are selected by the mask 0x007fffff) represent the significand (sometimes called the mantissa) of the floating-point number. If the argument is positive infinity, the result is 0x7f800000. If the argument is negative infinity, the result is 0xff800000. If the argument is NaN, the result is 0x7fc00000. In all cases, the result is an integer that, when given to the * method, will produce a floating-point value equal to the argument to floatToIntBits. */ + /** + * The raw IEEE 754 bits of {@code value}, without collapsing NaN to the + * canonical NaN. + * + * Delegates rather than declaring a second native. ParparVM's floatToIntBits + * is a bare union punt that does not collapse NaN to the canonical NaN -- so it + * is already the raw operation, and the two differ in the spec but not here. A + * separate native would be one more mangled symbol to get wrong, silently, for + * no behavioural difference. + */ + public static int floatToRawIntBits(float value) { + return floatToIntBits(value); + } + public native static int floatToIntBits(float value); /** diff --git a/vm/JavaAPI/src/java/lang/Integer.java b/vm/JavaAPI/src/java/lang/Integer.java index 0bcf391a733..387648fe877 100644 --- a/vm/JavaAPI/src/java/lang/Integer.java +++ b/vm/JavaAPI/src/java/lang/Integer.java @@ -359,6 +359,18 @@ public static int signum(int i) { return (i >> 31) | (-i >>> 31); // Hacker's delight 2-7 } + /** + * Rotates the two's-complement binary representation of {@code i} left by + * {@code distance} bits. + * + * The shift distance is used modulo 32 by the JLS shift rules, which is what + * makes the negation on the right half correct for every distance, including + * zero and multiples of 32. + */ + public static int rotateLeft(int i, int distance) { + return (i << distance) | (i >>> -distance); + } + public static int compare(int f1, int f2) { if (f1 > f2) return 1; diff --git a/vm/JavaAPI/src/java/lang/Short.java b/vm/JavaAPI/src/java/lang/Short.java index 233a1698268..16610a821a9 100644 --- a/vm/JavaAPI/src/java/lang/Short.java +++ b/vm/JavaAPI/src/java/lang/Short.java @@ -27,6 +27,28 @@ * Since: JDK1.1, CLDC 1.0 */ public final class Short extends Number implements Comparable { + + /** + * The class object for the primitive type this class wraps. + * + * Null on every ParparVM target, and declared only because ASM's compiled + * bytecode reads it: org.objectweb.asm.Type compares against Short.TYPE, + * Float.TYPE and Boolean.TYPE, so the self-hosted translator does not link + * without the three fields existing. ASM is a jar we cannot edit, which is + * the one case where JavaAPI grows to meet a dependency rather than the + * dependency being removed. + * + * It cannot be given a real value here. javac lowers a primitive class + * literal to a read of the boxed type's own TYPE field, so the obvious + * initializer compiles to "getstatic TYPE; putstatic TYPE" -- it reads the + * field it is initializing and stores the null straight back. The six + * wrappers that already declare TYPE are null for exactly that reason. + * Giving all nine real values needs VM-side primitive class objects; that + * work is not part of this change, and nothing in the translator depends on + * it now that the C-type tables are keyed on the PrimitiveType enum. + */ + public static final Class TYPE = null; + /** * The maximum value a Short can have. * See Also:Constant Field Values diff --git a/vm/JavaAPI/src/java/lang/TypeNotPresentException.java b/vm/JavaAPI/src/java/lang/TypeNotPresentException.java new file mode 100644 index 00000000000..5320f04fa67 --- /dev/null +++ b/vm/JavaAPI/src/java/lang/TypeNotPresentException.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +package java.lang; + +/** + * Thrown when an application tries to access a type using a string naming the + * type, but no definition for that type can be found. + */ +public class TypeNotPresentException extends java.lang.RuntimeException { + private final String typeName; + + public TypeNotPresentException(String typeName, Throwable cause) { + super("Type " + typeName + " not present", cause); + this.typeName = typeName; + } + + /** + * The fully qualified name of the unavailable type. + */ + public String typeName() { + return typeName; + } +} diff --git a/vm/JavaAPI/src/java/util/ArrayList.java b/vm/JavaAPI/src/java/util/ArrayList.java index 8a3c4129a15..4f99090a2a5 100644 --- a/vm/JavaAPI/src/java/util/ArrayList.java +++ b/vm/JavaAPI/src/java/util/ArrayList.java @@ -38,6 +38,27 @@ public class ArrayList extends AbstractList implements List, RandomAcce /** * Constructs a new instance of {@code ArrayList} with ten capacity. */ + // ISOLATION (PR #5766): the lazy default-capacity allocation that used to sit + // here is withdrawn. It replaced the eager new Object[10] with a SHARED static + // zero-length array, which also gave java.util.ArrayList a it had + // never had -- master's only static is a compile-time serialVersionUID, so the + // class previously emitted no static initializer at all. + // + // The suite then began stopping after exactly 145 of 166 screenshots on every + // target except glibc-x64, with ArrayList state corrupt at the point of + // failure: AIOOBE 89 inside pendingIdleSerialCalls.add, then AIOOBE -1, then a + // NullPointerException inside ArrayList.get, which only happens when the + // backing array reference itself is null. + // + // The list logic is NOT at fault: a differential fuzz of this exact source + // against java.util.ArrayList ran 3000 seeds x 200 random operations with no + // divergence, and every access to the corrupted list in Display is inside + // synchronized(lock). The corruption is therefore below Java, which makes the + // new and the process-wide shared array the part worth removing + // before anything subtler is blamed. + // + // The iterator below is the change that carried the measured win (iteration + // 25.5% -> 12.4% of mutator self-time) and is kept. public ArrayList() { this(10); } @@ -322,6 +343,100 @@ public void ensureCapacity(int minimumCapacity) { } } + /** + * Direct-array iterator, overriding AbstractList's generic SimpleListIterator. + * + * The inherited one was the single hottest method in a large translation -- + * 16.45% of mutator self-time on the 5782-class hellocodenameone corpus, more + * than twice the next entry. Three costs per element, none inherent: + * + * - a try/catch around the body, to turn IndexOutOfBoundsException into + * NoSuchElementException. ParparVM has no zero-cost exception tables, so a + * try block is a setjmp -- once per element, in the hottest loop in the + * program. An explicit bounds test costs a compare. + * - size() and get() as VIRTUAL calls on the outer list, with no JIT to + * inline them. + * - the index recomputed as size() - numLeft every iteration instead of + * being carried in a cursor. + * + * MEASURED after: the iteration path fell from 25.5% of mutator self-time to + * 12.4%, ArrayList.get from 7.42% to 0.55%, and _setjmp from 1.61% to zero. + * + * Semantics are unchanged: same ConcurrentModificationException on structural + * modification, same NoSuchElementException past the end, remove() still + * works. Reads array[firstIndex + i] exactly as get(int) does. + * + * Applies to every `for (x : list)` in every translated application whatever + * the loop's static type, because dispatch lands on the concrete ArrayList. + */ + // Package-private, not private: a private inner class whose constructor is + // reached from the outer class makes javac synthesise an access bridge and a + // ArrayList$1 marker type, so every iterator() paid an extra class and an + // aconst_null for the bridge argument. Nothing outside java.util can see it + // either way. + class ArrayListIterator implements Iterator { + private int cursor; + private int lastReturned = -1; + private int expectedModCount = modCount; + + public boolean hasNext() { + return cursor < size; + } + + public E next() { + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + int i = cursor; + if (i >= size) { + throw new NoSuchElementException(); + } + // The i < size test is only a bounds check while the list's + // firstIndex + size <= array.length invariant holds, so the array + // itself has to be checked too. The iterator this replaced could not + // read out of range: it went through get(), which bounds-checks, inside + // a try that turned IndexOutOfBoundsException into + // NoSuchElementException. Dropping that -- the try was the point, since + // ParparVM has no zero-cost exception tables -- also dropped the only + // bounds check on the read, and ParparVM does NOT check an array read in + // a release build. The result was an out-of-bounds read of the heap + // rather than a recoverable exception, which is how an unrelated int[] + // ended up with a zeroed header and the screenshot suite died 145 tests + // in. OpenJDK's own ArrayList.Itr carries this identical guard + // (`if (i >= elementData.length) throw new ConcurrentModificationException()`); + // omitting it is the whole defect. One compare, and the measured win + // stays. + E[] a = array; + int idx = firstIndex + i; + if (idx < 0 || idx >= a.length) { + throw new ConcurrentModificationException(); + } + cursor = i + 1; + lastReturned = i; + return a[idx]; + } + + public void remove() { + if (lastReturned < 0) { + throw new IllegalStateException(); + } + if (modCount != expectedModCount) { + throw new ConcurrentModificationException(); + } + ArrayList.this.remove(lastReturned); + if (lastReturned < cursor) { + cursor--; + } + lastReturned = -1; + expectedModCount = modCount; + } + } + + @Override + public Iterator iterator() { + return new ArrayListIterator(); + } + @Override public E get(int location) { if (location < 0 || location >= size) { diff --git a/vm/JavaAPI/src/java/util/IdentityHashMap.java b/vm/JavaAPI/src/java/util/IdentityHashMap.java index 4010c21c92a..188a307ac74 100644 --- a/vm/JavaAPI/src/java/util/IdentityHashMap.java +++ b/vm/JavaAPI/src/java/util/IdentityHashMap.java @@ -125,25 +125,52 @@ static class IdentityHashMapIterator implements Iterator { final MapEntry.Type type; + /** + * Which of the three views this iterator serves. + * + * Keys and values come straight out of the table; only entrySet has to + * materialise an Entry, and only there can the caller observe one. The + * generic {@code type} callback cannot express that, because it takes a + * MapEntry -- so serving a key iterator through it allocated an Entry per + * next() purely to read one field back out and drop it. Measured on a + * self-hosting translation of the ParparVM translator: 1,366,140 such + * entries, 43.7MB, all garbage. java.util.HashMap already had separate + * key/value/entry iterators for exactly this reason; this one was missed. + */ + static final int KIND_ENTRY = 0; + static final int KIND_KEY = 1; + static final int KIND_VALUE = 2; + + final int kind; + boolean canRemove = false; IdentityHashMapIterator(MapEntry.Type value, IdentityHashMap hm) { associatedMap = hm; type = value; + kind = KIND_ENTRY; + expectedModCount = hm.modCount; + } + + IdentityHashMapIterator(int iteratorKind, IdentityHashMap hm) { + associatedMap = hm; + type = null; + kind = iteratorKind; expectedModCount = hm.modCount; } public boolean hasNext() { - while (position < associatedMap.elementData.length) { - // if this is an empty spot, go to the next one - if (associatedMap.elementData[position] == null) { - position += 2; - } else { - return true; - } + // elementData hoisted into a local: it was re-loaded from the outer map + // on every comparison AND on every array access, twice per probe step. + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; } - return false; + position = p; + return p < len; } void checkConcurrentMod() throws ConcurrentModificationException { @@ -152,19 +179,50 @@ void checkConcurrentMod() throws ConcurrentModificationException { } } + @SuppressWarnings("unchecked") public E next() { - checkConcurrentMod(); - if (!hasNext()) { + // The concurrent-modification test and the null-skipping scan are + // INLINED here rather than reached through checkConcurrentMod() and + // hasNext(). + // + // An enhanced-for already pays two interface dispatches per element + // (hasNext then next); routing next() through two more non-inlined + // calls made it four, and ParparVM has no JIT to fold them away. + // MEASURED on the 5782-class hellocodenameone translation: + // IdentityHashMapIterator.next 6.43% of mutator self-time with + // checkConcurrentMod a further 1.84%, second only to the ArrayList + // iterator. + // + // Behaviour is unchanged: same ConcurrentModificationException on a + // structural change, same NoSuchElementException past the end, and + // position still advances past empty slots exactly as hasNext() did. + if (expectedModCount != associatedMap.modCount) { + throw new ConcurrentModificationException(); + } + Object[] data = associatedMap.elementData; + int p = position; + int len = data.length; + while (p < len && data[p] == null) { + p += 2; + } + if (p >= len) { + position = p; throw new NoSuchElementException(); } - IdentityHashMapEntry result = associatedMap - .getEntry(position); - lastPosition = position; - position += 2; - + lastPosition = p; + position = p + 2; canRemove = true; - return type.get(result); + + if (kind == KIND_KEY) { + Object key = associatedMap.elementData[lastPosition]; + return (E) (key == NULL_OBJECT ? null : key); + } + if (kind == KIND_VALUE) { + Object value = associatedMap.elementData[lastPosition + 1]; + return (E) (value == NULL_OBJECT ? null : value); + } + return type.get(associatedMap.getEntry(lastPosition)); } public void remove() { @@ -687,11 +745,7 @@ public boolean remove(Object key) { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public K get(MapEntry entry) { - return entry.key; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_KEY, IdentityHashMap.this); } }; } @@ -739,11 +793,7 @@ public void clear() { @Override public Iterator iterator() { return new IdentityHashMapIterator( - new MapEntry.Type() { - public V get(MapEntry entry) { - return entry.value; - } - }, IdentityHashMap.this); + IdentityHashMapIterator.KIND_VALUE, IdentityHashMap.this); } @Override diff --git a/vm/selfhost/README.md b/vm/selfhost/README.md new file mode 100644 index 00000000000..90852587d9f --- /dev/null +++ b/vm/selfhost/README.md @@ -0,0 +1,370 @@ +# Self-hosting ParparVM + +Builds `ByteCodeTranslator` with ParparVM itself: the translator's own bytecode, +plus ASM's, is translated to C and compiled into a native binary. + +It buys two things: + +1. **Validation.** The translator is a ~37k-line real program that exercises + collections, strings, file I/O, exceptions and the GC at scale. Running the + native build and the JVM build over the same input and diffing the emitted C + is an end-to-end conformance test of the whole VM, and the corpus grows on its + own as the translator does. +2. **Performance and memory.** A translation is a short-lived, allocation-heavy + batch job -- the shape where AOT should beat a cold JVM. + +## `stubs/` + +The self-hosted binary does the `clean`/`ios`/`macos` translation and nothing +else, so a few classes are replaced by no-op stubs when it is built. They are +never selected at run time; they exist so the source set compiles without +dragging in API that ParparVM's JavaAPI deliberately lacks. + +| stub | why | +|---|---| +| `Javascript*` | the JavaScript target, ~12.5k lines. Needs `java.util.regex` and `ConcurrentHashMap`. | +| `ArchiveClassScanner` | `java.util.zip`. Reachable only from `NativeSignatureVerifier`'s command-line entry point; the translator itself never reads an archive. | +| `DebugSymbolCompressor` | `java.util.zip` again, for the on-device-debug symbol sidecar. | + +`java.util.zip` cannot simply be added to JavaAPI: JavaAPI is mirrored by +`Ports/CLDC11`, where the package does not belong. + +Everything else the translator needs was removed from the translator rather than +added to JavaAPI -- see `Util`'s `splitLiteral`, `collapseWhitespace`, +`rewriteLocalObjectRefs`, `getProperty`, `listFiles` and `writeBytes`. Adding +`String.split`/`replaceAll` to JavaAPI in particular would have collided with +`BytecodeComplianceMojo`, which rewrites those calls onto +`com.codename1.util.regex` precisely because JavaAPI does not declare them. + +## Building and verifying + +```bash +export JDK_8_HOME=/path/to/a/working/jdk8 +./build-selfhost.sh # -> target/parpar +./verify-selfhost.sh # gates D and A +``` + +`build-selfhost.sh` compiles the source set against JavaAPI alone, stages ASM as +class directories (the translator walks directories, never archives), translates, +and clangs the result. The `-fwrapv -fno-strict-aliasing -fno-builtin-fmod(f)` +flags are mandatory for generated C -- Java arithmetic wraps and clang -O3 +miscompiles without them. + +The binary finds the C runtime it has to copy into its output through +`Class.getResourceAsStream`, which now consults resources linked into the +executable and then a search path named by `CN1_RESOURCE_PATH`. Before this it +returned a hard-coded null on every ParparVM target. + +## State + +Gate D (the native translator against itself) passes. Gate A (JVM against native) +is at **245 of 247 files byte-identical** on a JavaAPI-sized corpus, and binaries +built from the two trees produce identical output. + +The two files that still differ are `java_util_HashMap.c` and `.h`: the native +translator's dead-code pass culls seven more methods than the JVM's +(`cn1PutSlot`, `cn1MaybeGrow`, `clearImpl`, `containsKeyImpl`, `getImpl`, +`putImpl`, `removeImpl`), and emits them as empty stubs. Both trees compile, link +and run correctly, so the extra culling is safe here, but the two runtimes should +not disagree and the cause is not yet found. What is already ruled out: it is not +nondeterminism -- gate D passes on both sides -- and it is not identity-hash +iteration order, which was tested directly by re-running the JVM under +`-XX:hashCode=2` and getting byte-identical output. + +## What self-hosting has already found + +Three defects that were invisible to every existing test, because each was +self-consistent on HotSpot: + +- **`Integer.TYPE` and the other wrapper `TYPE` fields were null.** `TYPE = + int.class` compiles to `getstatic TYPE; putstatic TYPE`. A `Map` keyed on them + collapsed onto the single null key. `Util`'s primitive-to-C-type maps are exactly + that shape. +- **C label names came from identity hash codes.** ASM's `Label.toString()` is + `"L" + System.identityHashCode(this)`. That made the emitted C irreproducible, + and on ParparVM -- whose identity hash is the object pointer narrowed to int, so + often negative -- it emitted `label_L-180306432001`, which C reads as a + subtraction. Every method with a try/catch failed to compile. +- **C local-variable declarations were emitted in `HashSet` iteration order**, so + the same input produced different C. `debugVarEntries` had already had to learn + this for the debug side-table; the declarations had the same defect. + +Only the first is a runtime bug. The other two are reproducible-build defects in +the translator that a second runtime made visible. + +## Performance + +`bench-selfhost.sh` runs each arm over the same corpus, interleaved, and reports the +minimum wall clock and the peak `phys_footprint`. It refuses to print ratios unless +every arm emitted identical C. The reference JVM is **JDK 25** -- what HotSpot can +actually do; JDK 8 is kept only because it is what the builders currently fork. + +Translating the self-hosting corpus (ASM + the translator's own classes, ~570 +classes) on a 64 GB / 16-core Mac, release shape (`-O3 -flto=thin`): + +| | wall clock | peak footprint | +|---|---:|---:| +| parpar | 1.84 s | 1443 MB | +| jdk25 | 1.56 s | 516 MB | +| jdk8 | 2.27 s | 502 MB | + +**vs JDK 25: 1.18x slower, 2.79x more memory. vs JDK 8: 1.24x faster.** + +Two fixes got it there from 6x slower; both are described below. Wall clock on this +machine is only meaningful when it is quiet -- at load 113 the same benchmark +produced samples from 3.6 s to 24 s for every arm, JVM included. CPU time +(`user+sys`) is far more robust to contention, and by that measure the two are +level or better: parpar 4.35 s against jdk25 4.82 s on a loaded host. + +### Fix 1: the mutator slept instead of allocating + +`sample` on the original build put 64% of the process's samples in one stack, and +the mutator was not marking or sweeping -- it was asleep: + +``` +Ldc.getValueAsString -> cn1BibopAlloc -> cn1BibopMaybeGc + -> cn1PacingPark -> usleep -> nanosleep -> __semwait_signal +``` + +`CN1_LOG_PACING_PARKS` reported only **two** park events for the whole run, so each +was seconds long. `cn1BibopPacingCap` computed a generous cap -- `cn1CachedFreeMem/8`, +4 GB here -- and then clamped it to `trigger * 8` once the footprint passed +`CN1_PACING_GROWTH_FLOOR_BYTES`. That floor was a flat **512 MB**, and early in the +run the trigger is still at its own 24 MB floor, so the ceiling was **192 MB** +(`minCapKb=196608` confirmed it). A program with a ~1.4 GB live set cannot stay +inside a 192 MB allocation window, so it parked against a collector that could +never get under it. + +A fixed 512 MB says "this process has grown"; it does not say the machine is under +pressure, and the bound exists for pressure. The floor now scales: +`max(512MB, availableMemory/4)`. Where `cn1_available_memory` is the flat 100 MB +placeholder (Linux, Windows, the non-Apple fallback) the absolute floor still wins +and behaviour is unchanged; the floor can only ever rise, never fall. This is the +no-per-process-ceiling path only -- where a ceiling exists (iOS's dirty-memory +limit, or an explicit budget) `cn1PacingPark` takes the bounded branch and never +reaches this code. `ProcessBudgetPacingIntegrationTest` confirms both halves: its +control arm reports `minCapKb=4194304` with no parks, and its budget-bounded arm +still holds a 120 MB limit at a 60 MB peak across 427 parks. + +`cn1RefreshFreeMemCache()` also had exactly one caller, inside the mark cycle, so +`cn1CachedFreeMem` was 0 until the first collection and both the cap and this floor +fell to their absolute minimums during the window with the least reason to throttle. +It is primed in `cn1BibopDoInit` now. + +### Fix 2: the constant pool was O(n^2) + +With pacing out of the way, the main thread's own profile was dominated by +`Parser.addToConstantPool`, which did `constantPool.indexOf(s)` -- a `String.equals` +against every string already interned. On a self-hosting translation the pool holds +~200k strings: `String.equals` 11.2%, the list iterator 10.3%, `indexOf` 6.2% and +`ArrayList.get` 5.1% of main-thread samples, all of it there. A `HashMap` side index +answers the same question directly; the list stays the source of truth, so the +emitted indices are unchanged and gate A still passes byte-identical. + +### What is left: memory + +The remaining gap is peak footprint. Sweeping the GC trigger from 8 MB to 256 MB -- +four cycles down to two -- moves peak by less than 15%, so this is retained data +rather than uncollected garbage, and page-pool slack is about 2 MB, so it is not +fragmentation either. `CN1_HEAP_REPORT` on a census build prints the split. + +Two allocation defects came out of the per-class census and are fixed: + +- **`IdentityHashMap` allocated an `Entry` on every `next()`**, even for key and + value iteration, where the entry was built only to read one field back out of it + and drop it. 1,366,140 of them, 43.7 MB, all garbage. `java.util.HashMap` already + had separate key/value/entry iterators for exactly this reason and this map had + been missed; it now has the same split. +- **`ArrayList()` eagerly allocated `Object[10]`**, a 128-byte slot for every list, + including one never added to. It now shares a zero-length array until the first + growth. The first growth allocates exactly ten and not the twelve the general + growth path would pick, because ten keeps a small list in the size class it + already occupied -- growing to twelve would have traded a win on empty lists for + a loss on every list of one to ten elements. + +Measured together on the self-hosting corpus: + +| | before | after | +|---|---:|---:| +| allocations | 10,160,401 objects / 991 MB | 8,706,929 / 940 MB | +| legacy-heap objects | 729,174 | 444,783 | +| Java live heap | 860 MB | 770 MB | +| process peak | 1467 MB | 1324 MB | + +`CollectionSemanticsIntegrationTest` holds both against a real JDK -- empty-list +operations, the three growth paths, identity semantics, null keys and values +through each of the three views, iterator removal, and a rehash. It was confirmed +to fail when the key iterator stops mapping the table's sentinel back to null. + +**`HashMap` was investigated and deliberately left alone.** It eagerly allocates +three arrays (keys, values, meta) at capacity 16, which looks like the same defect, +but the maps in this workload are populated rather than empty. Rebuilding with a +default capacity of 1 -- the cheapest probe for "how much of that table is wasted" +-- made everything worse, because the maps then regrow repeatedly: + +| default capacity | Object[] allocs | int[] allocs | Java live | +|---|---:|---:|---:| +| 16 (current) | 1,324,987 | 213,725 | 770 MB | +| 1 (probe) | 1,802,249 | 452,356 | 882 MB | + +Growth there is also post-insert by design, so the shared-empty-table trick that +works for ArrayList would have the put path writing into the shared table. Not +worth it for an unmeasured win in the hottest class in the runtime. + +### Heap telemetry + +A census build answers "what is actually in the heap": + +```bash +CN1_SELFHOST_CFLAGS="-DCN1_ALLOC_CENSUS" ./build-selfhost.sh -O3 +CN1_HEAP_REPORT=1 ./target/parpar-O3 clean ... 2> report.txt +``` + +Three reports, after every sweep and once at exit: + +- `[JHEAP]` -- BiBOP pages reserved / live / slack, plus the legacy heap. Answers + "is this fragmentation?" (here: no, slack is ~2 MB of 715 MB). +- `[LIVE]` -- **the live heap by class**, occupied bytes, objects, bytes each, and + how many the last mark proved reachable. This is the one that was missing. +- `[ALLOC]` -- allocation volume by class. Churn, which costs CPU, as opposed to + retention, which costs memory. A class can dominate one and not the other. + +`[LIVE]` charges each object what it OCCUPIES -- a whole BiBOP size-class slot, a +whole malloc block -- so the per-class rows add up to the footprint and rounding +waste is charged to the class that causes it. + +**Read the post-sweep report, not the exit one, for reachability.** `reachable` +means "carries the current mark", so at exit -- long after the last cycle -- almost +everything looks unreachable whether it is or not. At exit that column says 6%; at +the last sweep, with fresh marks, it says 75%. + +### What the census says about this workload + +The `[LIVE]` report is printed **pre-sweep**, which is the only point where the four +reasons a slot is still occupied are distinguishable: `traced` (the current mark +reached it), `fresh` (allocated since the mark, kept by the grace rule), `aging` +(known dead, kept one more cycle) and `dead` (this sweep returns it). Post-sweep +the grace stamp makes the first two identical, and the first version of this census +reported one as the other. + +At the last cycle of a self-hosting translation: + +``` +occupied 4,441,347 objects 349MB + traced 47% fresh 30% aging 14% dead 9% +``` + +**Only 47% of the occupied heap is traced live. The rest is held by collector +policy, not by the program.** Per class the split is sharper still -- `char[]` is +**5% traced and 76% fresh**, i.e. almost pure churn caught between cycles: + +``` + 68.84MB 726070 objs 99 B/obj traced 49% fresh 20% aging 19% dead 12% java.lang.Object[] + 51.17MB 483395 objs 110 B/obj traced 5% fresh 76% aging 13% dead 6% char[] + 26.12MB 363289 objs 75 B/obj traced 57% fresh 27% aging 10% dead 5% java.lang.String + 15.09MB 240879 objs 65 B/obj traced 5% fresh 62% aging 21% dead 13% boolean[] +``` + +The mechanism is the sweep's own rule, confirmed directly by +`experiments/PinProbe`: a dead object needs **three cycles** to have its slot +returned -- one of grace while it is fresh, one of aging, then reclamation. A +translation completes three or four cycles in 1.4s, so most of what it allocates is +never eligible to be freed and the heap grows towards total allocation volume +(940MB allocated, 1.3GB peak, ~150-300MB genuinely live). + +Collecting faster helps, but does not change the ratio, because the grace rule +keeps everything allocated since the last mark whatever the rate: + +| | cycles | peak | traced at last cycle | +|---|---:|---:|---:| +| 1 mark thread | 3 | 1320 MB | 47% | +| `-DCN1_GC_MARK_THREADS=4` | 8 | **1172 MB** | 25% | +| 4 threads + `CN1_GC_TRIGGER_MB=24` | 7 | 1259 MB | 39% | + +So the dominant lever is **allocation churn**, and the `[ALLOC]` census names it: +`char[]` 368MB, `Object[]` 196MB, `String` 77MB, `SimpleListIterator` 40MB. Cutting +an allocation removes roughly three cycles of occupancy, not one object. + +Two hypotheses this ruled OUT, both of which looked plausible: + +- **Conservative stack roots pinning dead objects.** `experiments/PinProbe` shows + the marks are precise and depth makes no difference: a dropped batch reads 100% + kept on the cycle after it is allocated (the grace stamp) and 0% on the next, + identically whether it was allocated in a shallow frame, under a 400-deep + recursion, or with the stack scrubbed afterwards. +- **Fragmentation.** `[JHEAP]` puts page-pool slack at ~2MB of 715MB. + +Where the process memory sits, from `vmmap --summary` around peak: + +``` +MALLOC_LARGE 551.5M virtual / 435.8M dirty BiBOP arenas +MALLOC_LARGE (empty) 53.7M / 50.2M dirty freed, not returned +MALLOC_SMALL 232.0M / 111.7M dirty legacy heap +Stack 12.2M / 0.2M +``` + +It is all malloc'd heap; there is no large non-heap component. (An earlier note +here claimed ~600MB was "not the Java heap" -- that compared an exit-time census +against the whole-run peak and was wrong.) + +### The second grace cycle: vestigial in origin, load-bearing today + +A dead object needs three cycles because the sweep keeps it twice -- once as `fresh` +(never marked) and once as `aging` (`mark == V-1`). The first is load-bearing. The +second arrived in November 2014, commit `31528ecfa6`: + +``` +- if(o->__codenameOneGcMark != currentGcMarkValue) { // free what was not marked ++ if(o->__codenameOneGcMark < currentGcMarkValue - 1) { // keep one extra generation +``` + +message: "Delayed GCing of elements to prevent them from being collected due to a +race condition with the GC thread". **That collector had no SATB barrier** -- zero +matches for satb or snapshot at that commit -- so keeping an extra generation made a +lost-object race improbable rather than impossible. + +**It was removed, measured, and put back.** Removing it is verifier-green and +gauntlet-green and gives byte-identical self-hosting output, and it is worth about +**2-3% of peak** (1334 -> 1322 MB, 1349 -> 1302 MB). Not worth it, because four later +mechanisms have since been built on the rule: + +- Two `java.lang.ref` clearing sites that must use **exactly** the sweep's liveness + test. Their comment spells out the failure: "FAILING to clear one the sweep frees + hands get() a dangling pointer", and on ParparVM a dangling read is a native crash + no Java catch can see. +- The fast-sweep page shortcut, whose `gcGraceEpoch < V-1` bound is derived from the + per-slot rule. Its comment records what happened when the two disagreed: "testing + != V let it drop whole pages holding V-1 slots... 26,924 slots in one run. That is + what left kept objects pointing into reclaimed memory" -- issue 5425. +- The legacy and BiBOP sweeps ageing in step, so a matured `Hashtable.Entry` at V-1 + is never kept while its page-resident payload at V-1 has already gone. + +And the verifier does not cover the coupling: the measurement above changed the sweep +without changing the ref-clearing sites, which is precisely the dangling-`get()` bug, +and it still came back green. + +So the rule started as a band-aid and is now structural. Removing it means changing +all four together and re-deriving the fast-sweep bound, for 2-3%. The churn is worth +more and risks nothing. + +### String: the NSString field is free + +`java.lang.String` carries a `long nsString` for the Apple targets' direct NSString +mapping, and the obvious question is what that costs everywhere else. Measured: +nothing. + +``` +sizeof(obj__java_lang_String) = 48 nsString at offset 40 +``` + +The fields before it end at 36 and the struct is 8-aligned, so four of those eight +bytes were padding already. Without the field the struct is 40 bytes -- and BiBOP's +size classes are 32, 48, 64, ..., so 40 and 48 both land in the same 48-byte slot. +Removing it would save zero bytes per String while costing the Apple targets a +side table and a lookup. Keep it. + +The strings themselves are still the largest single consumer (`char[]`, 368 MB +allocated). Note that a compact Latin-1 path already exists for the concat +fast path -- `cn1FusedLatin1Begin` allocates the String and a `byte[]` payload in +one BiBOP slot -- so the remaining `char[]` volume is strings built some other way. +That is the next thing to look at. diff --git a/vm/selfhost/bench-selfhost.sh b/vm/selfhost/bench-selfhost.sh new file mode 100755 index 00000000000..30b13fb1732 --- /dev/null +++ b/vm/selfhost/bench-selfhost.sh @@ -0,0 +1,196 @@ +#!/bin/bash +# Wall clock and peak memory: the native translator against JVM-hosted ones. +# +# bench-selfhost.sh [rounds] +# +# Reference JVMs come from SELFHOST_REF_JAVAS (comma-separated java binaries). +# The default is JDK 25 first, then JDK 8. JDK 25 is the honest headline -- it is +# what HotSpot can actually do -- and JDK 8 is kept only because it is what the +# builders currently fork. +# +# Discipline, following vm/benchmarks/run-benchmark.sh: +# +# - Arms are INTERLEAVED within each round. Sequential A-then-B on this hardware +# carries a thermal bias large enough to invent a result. +# - Time takes the MINIMUM of N: the floor is the machine's best, and noise only +# ever adds. Memory takes the MAXIMUM, because a peak is a max. +# - Raw per-round samples are printed, not just the extremum: a lone minimum +# hides a bimodal distribution. +# - Ratios are refused unless every arm emitted identical C. A speed number from +# a translator that emits different output is meaningless. +# +# Memory is the peak phys_footprint reported by /usr/bin/time -l on macOS, which +# is the same quantity vmmap calls "Physical footprint (peak)". NEVER ps rss: +# vm/CLAUDE.md records 151/207/219 MB measured for one unchanged binary. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +CLASSES="${1:?usage: bench-selfhost.sh [rounds]}" +APP="${2:?}"; PKG="${3:?}"; ROUNDS="${4:-5}" + +T="$REPO/vm/selfhost/target" +# -O3 -flto=thin is the documented release shape (vm/benchmarks/README.md); +# CN1_SELFHOST_BIN overrides it for an A/B against the -O1 diff-gate build. +# The binary the GATES verify, not a separate optimised build nobody refreshes. +# It defaulted to $T/parpar-O3, which build-selfhost.sh does not produce and +# nothing kept current: the bench spent four runs comparing a months-old binary +# against a current JVM translator and correctly refusing to print a ratio, while +# Gate A passed byte-identical on the same corpus. Benchmarking a binary the gates +# have not verified cannot produce a meaningful number. +PARPAR="${CN1_SELFHOST_BIN:-$T/parpar}" +JAPI="$T/javaapi-classes" +TR="$REPO/vm/ByteCodeTranslator/target/classes" +ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" +# Reference JVMs, resolved rather than hard-coded. A developer-specific absolute +# path here meant anyone else running the documented command died under `set -e` +# while BUILDING the arm list, before a single measurement -- the benchmark was +# runnable by one machine. +# SELFHOST_REF_JAVAS explicit comma-separated list, wins outright +# JDK_25_HOME a modern JDK to compare against +# java on PATH whatever this shell would run +cn1_first_java() { + for c in "${JDK_25_HOME:-}/bin/java" "$(command -v java 2>/dev/null || true)"; do + [ -n "$c" ] && [ -x "$c" ] && { echo "$c"; return; } + done +} +DEFAULT_JAVAS="$(cn1_first_java),${JDK_8_HOME:-}/bin/java" +IFS=',' read -r -a REF_JAVAS <<< "${SELFHOST_REF_JAVAS:-$DEFAULT_JAVAS}" + +W="$T/bench"; rm -rf "$W"; mkdir -p "$W" + +# $1 = arm label, $2 = output dir; runs one translation +invoke() { + local arm=$1 out=$2 + if [ "$arm" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$PARPAR" \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none + else + "$arm" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$out" "$APP" "$PKG" "$APP" 1.0 clean none + fi +} + +# "25" -> jdk25, "1.8.0_372" -> jdk8. Taking the leading number alone turns 1.8.0 +# into "jdk1", which is why the second sed exists. +label() { + case "$1" in + parpar) echo parpar;; + *) "$1" -version 2>&1 | head -1 \ + | sed -e 's/.*version "\([0-9][0-9.]*\).*/\1/' \ + -e 's/^1\.\([0-9]*\).*/\1/' -e 's/\..*//' -e 's/^/jdk/';; + esac +} + +ARMS=(parpar "${REF_JAVAS[@]}") +declare -a NAMES +# Names have to be UNIQUE, not merely descriptive: every tree, log and diff is filed +# under one, so two arms sharing a label make the second `mv` land inside the first +# arm's directory and the correctness check then compares a tree against itself +# nested one level down -- a divergence that is an artefact of naming. Two arms +# collide easily now that JDK 25 is discovered rather than hard-coded: with no +# JDK_25_HOME the PATH fallback and JDK_8_HOME can both be Java 8. +for a in "${ARMS[@]}"; do + base="$(label "$a")" + name="$base"; k=2 + for prev in "${NAMES[@]}"; do + if [ "$prev" = "$name" ]; then name="${base}#${k}"; k=$((k+1)); fi + done + NAMES+=("$name") +done +# Say which executable each arm actually is, so a "#2" suffix is never a mystery. +for i in "${!ARMS[@]}"; do echo "arm : ${NAMES[$i]} -> ${ARMS[$i]}"; done +echo "corpus : $CLASSES" +echo "arms : ${NAMES[*]} rounds: $ROUNDS" +echo "memory : peak phys_footprint (/usr/bin/time -l)" +# STALENESS: a benchmark of yesterday's binary is worse than no benchmark, because +# the number looks like a measurement. Refuse rather than warn. +if [ -n "$(find "$REPO/vm/ByteCodeTranslator/src" "$REPO/vm/JavaAPI/src" -type f -newer "$PARPAR" -print -quit 2>/dev/null)" ]; then + echo "REFUSING: $PARPAR is older than the translator/JavaAPI sources it was built from." + echo " run vm/selfhost/build-selfhost.sh first." + exit 1 +fi + +# --- correctness precondition: every arm must emit the same C ------------------- +# Same absolute output path for all arms, sequentially, because the generated +# CMakeLists embeds srcRoot.getAbsolutePath(). +OUT="$W/out" +for i in "${!ARMS[@]}"; do + mkdir -p "$OUT" + invoke "${ARMS[$i]}" "$OUT" > "$W/${NAMES[$i]}.log" 2>&1 || { echo "${NAMES[$i]} FAILED"; tail -5 "$W/${NAMES[$i]}.log"; exit 1; } + # CLEAR the destination first. `mv` into an EXISTING directory nests the new + # tree inside it (tree-parpar/out/...) and leaves the previous run's dist/ in + # place, so the correctness check below compares a stale tree against a fresh + # one and reports a divergence that is pure harness. Observed: a tree-parpar + # left by an earlier session made every later run declare the VM divergent + # while Gate A passed byte-identical on the same corpus. + rm -rf "$W/tree-${NAMES[$i]}" + mv "$OUT" "$W/tree-${NAMES[$i]}" +done +for i in "${!ARMS[@]}"; do + [ "$i" -eq 0 ] && continue + if ! diff -rq "$W/tree-${NAMES[0]}" "$W/tree-${NAMES[$i]}" > "$W/diff-${NAMES[$i]}.txt" 2>&1; then + echo "DIVERGENCE (${NAMES[0]} vs ${NAMES[$i]}) -- ratios would be meaningless:" + sed "s|.*/$APP-src/||;s| and .*||" "$W/diff-${NAMES[$i]}.txt" | head -5 + exit 1 + fi +done +files=$(find "$W/tree-${NAMES[0]}" -type f | wc -l | tr -d ' ') +[ "$files" -gt 10 ] || { echo "VACUOUS: only $files files"; exit 1; } +echo "output : $files files, identical across all arms" +echo + +# --- timing, interleaved -------------------------------------------------------- +declare -a SAMPLES +for r in $(seq 1 "$ROUNDS"); do + for i in "${!ARMS[@]}"; do + rm -rf "$W/run"; mkdir -p "$W/run" + s=$(python3 -c 'import time;print(time.monotonic())') + invoke "${ARMS[$i]}" "$W/run" > /dev/null 2>&1 + e=$(python3 -c 'import time;print(time.monotonic())') + SAMPLES[$i]="${SAMPLES[$i]} $(python3 -c "print(f'{$e-$s:.2f}')")" + done +done +declare -a MINS +for i in "${!ARMS[@]}"; do + MINS[$i]=$(printf '%s\n' ${SAMPLES[$i]} | sort -n | head -1) + printf "time %-8s min %6ss samples:%s\n" "${NAMES[$i]}" "${MINS[$i]}" "${SAMPLES[$i]}" +done + +# --- memory, measured separately so the probe cannot perturb the clock ---------- +# +# Sampled in EVERY round and reduced with max, because the header promises the +# maximum of N samples and a peak is a max. Measuring each arm once let a single +# noisy run decide the reported ratio, which is the same mistake as quoting a +# memory figure from one process: the number looked like a measurement and was a +# sample. +declare -a PEAKS PEAK_MAX +for i in "${!ARMS[@]}"; do PEAK_MAX[$i]=0; done +for round in $(seq 1 "$ROUNDS"); do +for i in "${!ARMS[@]}"; do + rm -rf "$W/run"; mkdir -p "$W/run" + if [ "${ARMS[$i]}" = parpar ]; then + env CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" /usr/bin/time -l "$PARPAR" \ + clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null + else + /usr/bin/time -l "${ARMS[$i]}" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$CLASSES" "$W/run" "$APP" "$PKG" "$APP" 1.0 clean none 2>"$W/mem.txt" >/dev/null + fi + PEAKS[$i]=$(awk '/peak memory footprint/{print $1}' "$W/mem.txt") + printf "mem %-8s round %d peak %8.0f MB\n" "${NAMES[$i]}" "$round" \ + "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" + [ "${PEAKS[$i]}" -gt "${PEAK_MAX[$i]}" ] && PEAK_MAX[$i]="${PEAKS[$i]}" +done +done +for i in "${!ARMS[@]}"; do + PEAKS[$i]="${PEAK_MAX[$i]}" + printf "mem %-8s MAX over %d round(s) %8.0f MB\n" "${NAMES[$i]}" "$ROUNDS" \ + "$(python3 -c "print(${PEAKS[$i]}/1048576.0)")" +done + +echo +for i in "${!ARMS[@]}"; do + [ "$i" -eq 0 ] && continue + python3 -c " +t=${MINS[0]}/${MINS[$i]}; m=${PEAKS[0]}/${PEAKS[$i]} +print(f'vs ${NAMES[$i]}: time {t:.2f}x ({\"parpar faster\" if t<1 else \"parpar slower\"}), memory {m:.2f}x ({\"parpar smaller\" if m<1 else \"parpar larger\"})')" +done diff --git a/vm/selfhost/build-selfhost.sh b/vm/selfhost/build-selfhost.sh new file mode 100755 index 00000000000..a3da57dba35 --- /dev/null +++ b/vm/selfhost/build-selfhost.sh @@ -0,0 +1,190 @@ +#!/bin/bash +# Builds the ParparVM translator with ParparVM: its own bytecode, plus ASM's, is +# translated to C and compiled into a native binary. +# +# build-selfhost.sh [-O1|-O3] default -O1 +# +# Requirements: +# JDK_8_HOME a working JDK 8 (JavaAPI and the translator compile with it) +# clang, and maven on PATH the first time (to resolve ASM) +# +# The mandatory clang flags below are not negotiable for generated C: Java +# arithmetic wraps, and clang -O3 provably miscompiles without -fwrapv +# -fno-strict-aliasing -fno-builtin-fmod(f). See vm/benchmarks/README.md. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +OPT="${1:--O1}" +# -O3 implies ThinLTO: that IS the documented release shape (vm/benchmarks/README.md), +# and measured here it is the only rung that beats -O1 -- 1.45s against 1.61s for -O1, +# 1.70s for -O2 and 1.73s for plain -O3. Benchmarking a bare -O3 binary and calling it +# the release build understates it, so the flag is not left to the caller to remember. +case "$OPT" in -O3) CN1_SELFHOST_CFLAGS="-flto=thin $CN1_SELFHOST_CFLAGS";; esac +CC="${CN1_SELFHOST_CC:-clang}" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +OUT="$REPO/vm/selfhost/target" +mkdir -p "$OUT" + +# 1. translator classes + ASM classpath, built once by maven and then cached. +TRANSLATOR="$REPO/vm/ByteCodeTranslator/target/classes" +# Rebuild when the classes are MISSING or STALE. Testing only for existence meant +# that re-running this after editing a translator source silently self-hosted the +# previous build, and the resulting binary was then compared against a JVM side +# built from the new sources -- which reports the intended change as a VM +# divergence. verify-selfhost.sh carries the same guard for the same reason, and +# maven's own incremental check is not enough on its own: it answered "Nothing to +# compile - all classes are up to date" for a source three hours newer than its +# class. +TR_MANIFEST="$REPO/vm/ByteCodeTranslator/target/selfhost-src.manifest" +# The staging copy lives OUTSIDE target/, because `mvn clean` below deletes that whole +# directory -- staging it inside meant the file was gone by the time it was compared and +# moved into place, so the guard rebuilt on every single run and then failed on the +# missing file. Caught by asking the gate to stay SILENT when nothing changed, which is +# the half of a negative control that is easy to skip. +TR_MANIFEST_NOW="$(mktemp -t cn1selfhostmanifest)" +trap 'rm -f "$TR_MANIFEST_NOW"' EXIT +find "$REPO/vm/ByteCodeTranslator/src" -type f | sort > "$TR_MANIFEST_NOW" +needs_build=0 +if [ ! -f "$TRANSLATOR/com/codename1/tools/translator/ByteCodeTranslator.class" ]; then + needs_build=1 +elif [ ! -f "$TR_MANIFEST" ] || ! cmp -s "$TR_MANIFEST" "$TR_MANIFEST_NOW"; then + # A MANIFEST DIFF, because -newer cannot see a DELETION. Removing or renaming a + # source or a runtime resource makes no remaining file newer, so the timestamp test + # below is satisfied, maven is never re-run, and the deleted file survives in + # target/classes. The JVM translator then keeps embedding a runtime resource that no + # longer exists in the tree -- and because BOTH sides of the self-host comparison + # consume that same stale copy, Gate A still passes. A gate that cannot fail on a + # deleted file is not covering deletions. + # + # Same mechanism the JavaAPI block below already uses, and for the same reason; + # -type f rather than -name '*.java' because the C runtime ships as classpath + # resources (cn1_globals.m, nativeMethods.m, java_io_File.m, cn1_win_compat.c ...). + echo "translator source set changed (file added or removed) -- rebuilding" + needs_build=1 +elif [ -n "$(find "$REPO/vm/ByteCodeTranslator/src" -type f -newer "$TRANSLATOR" -print -quit 2>/dev/null)" ]; then + # -type f, not -name '*.java'. The translator carries its C runtime as CLASSPATH + # RESOURCES -- cn1_globals.m, nativeMethods.m, java_io_File.m, cn1_win_compat.c, + # xmlvm.h and the rest -- and maven copies them into target/classes. Watching + # only Java sources meant editing any of those left the old copy in place, so + # the self-hosted binary embedded an obsolete runtime while the JVM side used + # the new one. That surfaces as a Gate A divergence pointing at the VM, which is + # exactly the misdiagnosis this guard exists to prevent. + echo "translator sources or resources are newer than $TRANSLATOR -- rebuilding" + needs_build=1 +fi +if [ "$needs_build" = 1 ]; then + # `clean` because the incremental check cannot be trusted here; it also removes + # selfhost-asm-classpath.txt, which the next block regenerates. The clean is what + # actually evicts a deleted resource from target/classes, so the manifest test above + # is only useful paired with it. + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator -am clean package -DskipTests) +fi +# Record the manifest only after a build that succeeded -- `set -e` aborts above on +# failure, so reaching here means target/classes matches this file list. Writing it +# earlier would let one failed build convince every later run it was up to date. It has +# to be written AFTER the maven run for the same reason the staging copy is kept out of +# target/: the clean would otherwise remove it. +mkdir -p "$(dirname "$TR_MANIFEST")" +cp -f "$TR_MANIFEST_NOW" "$TR_MANIFEST" +ASM_CP_FILE="$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt" +if [ ! -f "$ASM_CP_FILE" ]; then + (cd "$REPO/vm" && mvn -q -B -pl ByteCodeTranslator dependency:build-classpath \ + -Dmdep.outputFile=target/selfhost-asm-classpath.txt) +fi +ASM_CP="$(cat "$ASM_CP_FILE")" + +# 2. the C runtime the translator emits from its own classpath resources. +# +# Copy EVERY non-Java file maven would have staged, not a hand-listed four. The +# list drifts: java_io_File.m, cn1_win_compat.c and xmlvm.h are all read through +# the same classpath lookup, and a hand-written subset silently ships whichever +# ones nobody remembered. +( cd "$REPO/vm/ByteCodeTranslator/src" && find . -type f ! -name '*.java' -print ) \ + | while read -r rel; do + mkdir -p "$TRANSLATOR/$(dirname "$rel")" + cp "$REPO/vm/ByteCodeTranslator/src/$rel" "$TRANSLATOR/$rel" + done + +# 3. JavaAPI, rebuilt from source whenever the source set changed. +# +# The presence check alone is not enough, and it fails in a way that looks like a VM +# bug rather than a stale cache: a class compiled before a method stopped being +# native still declares it native, so the translator emits a call to a symbol nothing +# defines. Three things invalidate it and it takes all three -- `-newer` catches an +# edited or added source, but a DELETED one moves no remaining file's timestamp, so +# the sorted manifest is what catches removals. Comparing a file list rather than +# hashing timestamps keeps this portable; `stat` takes -f on BSD and -c on Linux. +JAVAAPI="$OUT/javaapi-classes" +STAMP="$OUT/javaapi-classes.stamp" +MANIFEST="$OUT/javaapi-classes.manifest" +find "$REPO/vm/JavaAPI/src" -name '*.java' | sort > "$MANIFEST.now" +if [ ! -f "$JAVAAPI/java/lang/Object.class" ] || [ ! -f "$STAMP" ] || [ ! -f "$MANIFEST" ] || \ + ! cmp -s "$MANIFEST" "$MANIFEST.now" || \ + [ -n "$(find "$REPO/vm/JavaAPI/src" -name '*.java' -newer "$STAMP" -print -quit 2>/dev/null)" ]; then + rm -rf "$JAVAAPI"; mkdir -p "$JAVAAPI" + "$J8/bin/javac" -nowarn -Xmaxerrs 10000 -source 1.8 -target 1.8 -d "$JAVAAPI" $(cat "$MANIFEST.now") + mv "$MANIFEST.now" "$MANIFEST" + touch "$STAMP" +else + rm -f "$MANIFEST.now" +fi + +# 4. The self-host source set: every translator source except the ones a stub +# replaces, plus the two classes that carry their own main(). +# +# Only the sources that CANNOT compile against JavaAPI are stubbed, and the list +# is driven by what is in stubs/ rather than by a name pattern. A blanket +# "Javascript*" exclusion is what stubbed JavascriptNativeRegistry, which +# compiles fine and -- as the comment at its call site in Parser warns -- is +# consulted on EVERY target, not just JavaScript. Answering false there culled +# java.util.HashMap's getImpl/putImpl/removeImpl/containsKeyImpl/clearImpl and +# the two helpers only they call, and the native translator emitted seven +# methods as empty stubs that the JVM one emitted in full. +SRC="$REPO/vm/ByteCodeTranslator/src" +STUBS="$REPO/vm/selfhost/stubs" +STUBBED=$(cd "$STUBS" && find . -name '*.java' | sed 's|.*/||;s|\.java$||' | tr '\n' '|' | sed 's/|$//') +SRCLIST="$OUT/sources.txt" +find "$SRC" -name '*.java' \ + | grep -vE "/($STUBBED)\.java$" \ + | grep -v '/CastSemanticsVerifier\.java$' \ + | grep -v '/NativeSignatureVerifierCli\.java$' > "$SRCLIST" +find "$STUBS" -name '*.java' >> "$SRCLIST" + +# 5. compile it against JavaAPI ALONE. -Xmaxerrs because javac's default cap of 100 +# silently truncates and makes a large gap look small. +rm -rf "$OUT/classes"; mkdir -p "$OUT/classes" +"$J8/bin/javac" -nowarn -Xmaxerrs 100000 -source 1.8 -target 1.8 \ + -bootclasspath "$JAVAAPI" -cp "$ASM_CP" -d "$OUT/classes" "@$SRCLIST" + +# 6. ASM as class files: the translator walks directories, never archives. +rm -rf "$OUT/asm-classes"; mkdir -p "$OUT/asm-classes" +for jar in $(echo "$ASM_CP" | tr ':' '\n' | grep -E 'asm.*\.jar$'); do + (cd "$OUT/asm-classes" && unzip -oq "$jar" -x 'module-info.class' 'META-INF/*') +done + +# 7. translate. The app name has to be the mangled main class: three classes in the +# set declare main, and ByteCodeClass.addMethod refuses to pick one otherwise. +APP=com_codename1_tools_translator_ByteCodeTranslator +rm -rf "$OUT/out"; mkdir -p "$OUT/out" +# CN1_SELFHOST_JAVA_OPTS reaches the TRANSLATOR that emits the C, not the C compiler -- +# it is how a codegen-level ablation is run. The one that matters for GC work is +# -Dcn1.frameless.objects=false -Dcn1.frameless.instance=false, which reverts to pushing +# every object reference onto threadObjectStack; with frameless codegen on (the default) +# a live reference can exist ONLY in a C local, so any collector that scans the precise +# stack alone will miss it. Word-split on purpose: this is a list of options. +"$J8/bin/java" -Xmx4g $CN1_SELFHOST_JAVA_OPTS -cp "$TRANSLATOR:$ASM_CP" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAVAAPI;$OUT/asm-classes;$OUT/classes" "$OUT/out" \ + "$APP" com.codename1.tools.translator "$APP" 1.0 clean none \ + > "$OUT/translate.log" 2>&1 \ + || { echo "TRANSLATE FAILED"; tail -40 "$OUT/translate.log"; exit 1; } + +# 8. compile. The .S as well as the .c: the virtual-thread context switch is emitted +# beside the generated sources and the C half references it, so a *.c-only +# invocation links against a missing cn1VirtualThreadSwitch. +SRCDIR="$OUT/out/dist/$APP-src" +ASMS=$(ls "$SRCDIR"/*.S 2>/dev/null || true) +BIN="$OUT/parpar$( [ "$OPT" = "-O3" ] && echo "-O3" || echo "" )" +$CC $OPT -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ + $CN1_SELFHOST_CFLAGS -I"$SRCDIR" "$SRCDIR"/*.c $ASMS -lm -lpthread -o "$BIN" \ + 2> "$OUT/cc.log" || { echo "COMPILE FAILED"; tail -40 "$OUT/cc.log"; exit 1; } +echo "built $BIN" diff --git a/vm/selfhost/experiments/README.md b/vm/selfhost/experiments/README.md new file mode 100644 index 00000000000..a61efa8d5c5 --- /dev/null +++ b/vm/selfhost/experiments/README.md @@ -0,0 +1,47 @@ +# Heap experiments + +Small programs that answer one question each about the collector, read through the +census (`-DCN1_ALLOC_CENSUS` + `CN1_HEAP_REPORT=1`, see ../README.md). + +## PinProbe + +**Question: does the conservative stack scan pin objects that are provably dead?** + +Three arms, three distinct classes so one run compares them in one census: +`PinShallow` allocated and dropped in a shallow frame, `PinDeep` allocated at the +bottom of a 400-deep recursion, `PinScrub` the same but with the stack overwritten +before collecting. Nothing holds a reference to any of them, so a precise collector +reclaims all three and any survivor is a stale stack word mistaken for a pointer. + +Build and run: + +```bash +javac -bootclasspath -d /tmp/exp/classes src/com/exp/PinProbe.java +java -cp : com.codename1.tools.translator.ByteCodeTranslator \ + clean ";/tmp/exp/classes" /tmp/exp/out PinProbe com.exp PinProbe 1.0 clean none +clang -O3 -flto=thin -w -fwrapv -fno-strict-aliasing -fno-builtin-fmod -fno-builtin-fmodf \ + -DCN1_ALLOC_CENSUS -I /*.c /*.S -lm -lpthread -o /tmp/exp/pinprobe +CN1_HEAP_REPORT=1 /tmp/exp/pinprobe 2>&1 | grep -E 'PROBE|Pin(Shallow|Deep|Scrub)' +``` + +**Answer: no.** All three arms behave identically, and the marks are precise -- a +batch reads 100% live on the cycle after it is allocated and 0% on the next, with +no difference between the shallow, deep and scrubbed arms. What the probe found +instead is the reclamation LATENCY: a dead object needs **three cycles** to have +its slot returned. + +``` +cycle 1: PinShallow 200,000 occupied, 100% kept <- grace: fresh objects are stamped live +cycle 2: PinShallow 200,000 occupied, 0% kept <- known dead, still occupying +cycle 3: PinShallow 196,320 occupied <- reclaimed +``` + +That is the sweep's own rule: `m == -1` (fresh) gets one cycle of grace, `m == V-1` +is kept for another, and only `m < V-1` is reclaimed. It is why a short program +retains nearly everything it allocates -- the ParparVM translator completes three +or four cycles in 1.4s, so most of what it allocates is never eligible. + +This probe is also the reason the census reports its four buckets **pre-sweep**: +read post-sweep, the grace stamp makes "traced live" and "kept because it is fresh" +indistinguishable, and the first version of the census reported the second as the +first. diff --git a/vm/selfhost/experiments/REGISTRY.md b/vm/selfhost/experiments/REGISTRY.md new file mode 100644 index 00000000000..360ec21a9a4 --- /dev/null +++ b/vm/selfhost/experiments/REGISTRY.md @@ -0,0 +1,1119 @@ +# ParparVM selfhost: performance theorems, experiments, red-team + +Corpus: `scripts/hellocodenameone/.../macos-build/{classes,macPort}` (5782 classes). +Reference: JDK 25 = 6.56s / 1319MB / 2916 emitted .c files. + +Every run is gated on `rc==0` AND `cfiles==2916`. This gate exists because a run +that crashed early first read as a *fast* run: without `CN1_RESOURCE_PATH`, +`Class.getResourceAsStream` returns null and `ByteCodeTranslator.copy()` +dereferences it -- a bare SIGSEGV, no message. Under lldb the first stop is +SIGUSR2 (the GC's stop-the-world signal), which hides the real fault until it is +passed through. + +## Measurement hygiene + +- Arms are INTERLEAVED across reps, not grouped: sequential A-then-B on this + hardware carried a 10-15% thermal bias. +- `load1` is recorded per run. Wall clock on this box has been observed to vary + 63s..93s for the SAME binary and config as load moves; PEAK FOOTPRINT is stable + to ~3%. Treat memory results as tight and wall-clock results as needing reps. +- Memory is `phys_footprint` ("peak memory footprint" from `/usr/bin/time -l`), + never `ps rss` -- rss gave 151/207/219MB for one unchanged binary previously. +- The output hash is over file CONTENTS keyed by path RELATIVE to the output dir. + Hashing absolute paths made every arm differ by construction (the dir name IS + the arm), which would have read as "GC config changes emitted C". + +## Known confounds + +- `fseventsd` runs at ~100% CPU during every run, reacting to 2916 files written. + Constant across arms, but it is a real contributor to load. +- Load is largely SELF-inflicted (the benchmark + fseventsd), not an external job. +- `-DCN1_GC_CONFORM` adds instrumentation overhead: use it for ATTRIBUTION AND + RATIOS WITHIN A RUN, never for absolute wall-clock comparison against x_base. + +## Build-flag trap (cost a whole experiment) + +`parpar-O3` was built with `-DCN1_GC_MARK_THREADS=4` inherited from +`CN1_SELFHOST_CFLAGS`, NOT the source default of 1. The first worklist experiment +was built without that flag, so it ran serial and conflated worklist size with +marker count. Every arm is now built with fully explicit flags. + +--- + +## T1 -- wall clock is bound by pacing backpressure, not by GC CPU + +Two independent methods, both done, agreeing: + +- M1 (sampling profiler): main thread 93.3% of samples in + `cn1BibopAlloc -> cn1BibopMaybeGc -> cn1PacingPark -> usleep -> __semwait_signal`. +- M2 (mechanism removal): raising `CN1_GC_PACING_CAP_MB` so the cap never binds + cuts wall from ~90s to 25.4s. +- M3 (in-process, pending): `-DCN1_GC_CONFORM` prints + `[GCSTALL] cause=PACING_VOLUME count= totalMs=`, a non-sampling measure. + +VERDICT: confirmed. The premise "GC runs on another core so it is free in wall +clock" does not hold: GC throughput gates the mutator through admission control. + +## T2 -- ORIGINAL CLAIM, FALSIFIED + +Claimed: the ~10GB footprint is caused by `cn1PacingPastGrowthFloor()` returning +`freeMem/4` (~10GB on this 64GB box), so the cap ceiling never engages. + +Falsified by the cap sweep: at `CN1_GC_PACING_CAP_MB=256` the footprint is still +6586MB, and peak RISES with the cap (6.6 -> 10.3 -> 12.6 -> 12.7GB) and saturates. +Footprint is not set by the pacing cap. Note also that the cap override +SHORT-CIRCUITS `cn1BibopPacingCap` entirely, including the growth-floor gate, so +this sweep never tested the growth-floor mechanism in the first place -- the two +claims were being conflated. + +## T2' -- the footprint floor is pages never returned to the OS + +- M1 (direct trace, done): `CN1_LOG_PAGE_RELEASE=1` shows 7 release events over + the run, FIVE of them clamped at exactly `taken=1024` = `CN1_BIBOP_RELEASE_PER_SWEEP`. + Total 6016 pages x 64KB = 376MB returned against a 10GB peak. `rejected=0`, + `releaseErrno=0`, so `MADV_FREE_REUSABLE` is succeeding where it is attempted. + Cadence: page-returning "major" sweeps run only when a cycle is QUIET + (< trigger/4 = 6MB allocated) or every `CN1_BIBOP_MAJOR_SWEEP_CYCLES`=16 cycles. + This run is 22 cycles and never goes quiet. +- M2 (mechanism perturbation, DONE -- FALSIFIES THE THEOREM): `x_trim` = + `-DCN1_BIBOP_MAJOR_SWEEP_CYCLES=1 -DCN1_BIBOP_RELEASE_PER_SWEEP=131072` + released 23790 pages = 1487MB, FOUR TIMES base's 376MB, and never hit its + budget (largest single sweep 8544 << 131072). Peak footprint did not move: + + | arm | wall (2 reps) | peak MB | + |------|---------------|---------------| + | base | 62.9 / 64.9 | 10047 / 10003 | + | trim | 72.4 / 71.2 | 10169 / 9881 | + + So the budget WAS binding and lifting it DOES free 4x more memory, but the + peak is a high-water mark that is re-dirtied immediately. Trim cadence is not + the floor. `x_trim` is also ~12% SLOWER, from the extra madvise syscalls. + + All four runs emit byte-identical C (sha `d6e7d4e2e350`): GC configuration + does not change translator output. That is a validation result in its own + right, and it is why the hash column is worth carrying. + +RIVAL EXPLANATIONS this pair cannot separate on its own, and which "x_trim did +not help" would equally predict: + (a) FRAGMENTATION -- partially occupied pages cannot be released at any budget. + (b) REPRESENTATION SIZE -- the translator holds all 5782 parsed classes live for + the whole run; JDK 25 holds the same graph in 1319MB total. If parpar's + per-object cost is ~5x, the floor is the live set itself. +Discriminator: `-DCN1_ALLOC_CENSUS` + `CN1_HEAP_REPORT=1` prints +`[LIVE:exit] occupied N objects X.XXMB`, a DIRECT measure of occupied bytes. +If occupied ~= 6GB the floor is (b); if occupied is ~1.5GB with a 10GB footprint +it is trim/float; fragmentation shows as occupied-pages-mostly-empty. + +## T3 -- mark helpers are idle; marking is effectively serial + +- M1 (profiler, done): 3 helper threads 99.2% idle in `__psynch_cvwait` + (33538/33798 samples). GC thread ~100% busy: 33.4% `gcMarkDrain` (the SERIAL + drain), 28.4% `cn1ConservativeResolve`, 20.5% gcMarkObject/gcMarkArrayObject. +- M1b (static, done): only ONE of ~15 `gcMarkDrain*` call sites is + `gcMarkDrainParallel`. The grace pass, full drains and overflow rescan are all + serial. Telemetry for the run: graceDrains=521 fullDrains=219 over cycles=22. +- M2 (pending): wall clock for `-DCN1_GC_MARK_THREADS=` 1 vs 4 vs 8 should be + ~equal if the helpers contribute nothing. Falsified if mt8 is materially faster. + +## T4 -- mutator assist almost never fires + +- M1 (profiler, done): zero samples in `cn1GcMutatorAssist` anywhere in the file. + Under `cn1PacingPark`, ~31,000 samples are `usleep`; exactly one park subtree + shows 15 samples of real marking (`__GC_MARK_java_util_HashMap`). ~0.05%. +- M2 (pending): `CN1_GC_NO_MUTATOR_ASSIST=1` should not change wall clock. + +CORRECTION TO AN EARLIER CLAIM: "assist is inert" is too strong for THIS binary. +`cn1GcMutatorAssist` bails when `gcMarkActiveWorkers <= 0`, which the code +documents is "true only on the parallel path" -- so it is fully inert only in a +true serial build (`gcMarkThreadCount == 1`, the SOURCE default). In the +4-marker binary actually profiled it is armed during the one parallel drain site +and idle the rest of the time. + +## T5 -- mark-worklist overflow is not the bottleneck + +Overflow fires in 9 of 22 cycles (64K-entry fixed worklist), forcing the serial +BiBOP page-rescan. + +- M1 (pending, first attempt INVALID -- built without `-DCN1_GC_MARK_THREADS=4`): + wall clock at worklist 8192 vs 65536 vs 1048576. +- M2 (pending): `-DCN1_GC_CONFORM` prints + `[GCSTALL] rescanPasses= rescanUseful= rescanSlots= rescanPushes=`, a direct + measure of how much rescan work each worklist size causes. + +Working hypothesis to be tested, not yet evidence: the page-rescan may have +better locality than pointer-chasing a 10GB heap, making overflow a MITIGATION +rather than a cost. + +## The efficient frontier (scouting pass, 1 rep, load 17-38) + +| cap MB | wall | peak MB | +|----------|-------|---------| +| 256 | 90.5 | 6586 | +| 1024 | 47.4 | 10265 | +| 4096 | 30.5 | 12598 | +| 16384 | 25.4 | 12667 | +| default | 70.3 | 10246 | + +The DEFAULT policy is on neither end of this frontier. Needs reps before it is +load-bearing, given the observed wall-clock variance. + +--- + +# Round 2: what actually causes the 8x retention and 10x wall clock + +JDK 25 runs the SAME program with the same allocation pattern at 1319MB peak; +parpar peaks at 10486MB. Emitted output is 245MB of C. So the gap is RETENTION +and THROUGHPUT, not allocation volume. + +## Assumptions under test + +- A1 MARK-THROUGHPUT BOUND (with positive feedback). Cycle duration scales with + heap size; the mutator allocates all through a cycle; so the heap grows; so + cycles lengthen. Small changes in mark throughput should produce + DISPROPORTIONATE changes in end-to-end time. +- A2 DEFERRED RECLAMATION. grace+aging holds dead objects ~3 cycles, so float is + ~3x per-cycle allocation regardless of cycle length. Predicts allocated-per-cycle + stays ~constant (pinned to the trigger) and occupancy is a fixed multiple of it. +- A3 SERIAL RESCAN EXPLOSION. Worklist overflow drops the collector into an + O(heap) page-rescan fixpoint (fires in 9 of 22 cycles). + +## EXP-1 (perturbational): mark parallelism dose-response, at TWO SCALES + +Small corpus (395 emitted files), 2 reps, all four arms byte-identical output: + +| arm | wall | peak MB | +|---------|-----------|-----------| +| mt1 | 1.19/1.10 | 1239/1329 | +| mt4 | 1.16/1.13 | 1128/1271 | +| mt1wl1m | 1.15/1.10 | 1332/1351 | +| mt4wl1m | 1.07/1.06 | 881/893 | + +No effect at small scale -- which RULES OUT any fixed per-cycle cost. + +Large corpus, fixed 240s budget (a full mt1 run is ~3.5h, so progress rate +replaces time-to-completion): + +| arm | outcome | .c files | cull phase | +|---------|--------------|----------|------------| +| mt1 | TIMEOUT 240s | 34 | 17s | +| mt1wl1m | TIMEOUT 240s | 11 | -- | +| mt4 | finished 70s | 2916 | 5s | + +A 4x change in mark parallelism produces a ~60x change in end-to-end time, and +ONLY at scale. That superlinearity is the signature of a feedback loop, and it +supports A1. + +A3 IS FALSIFIED: a 16x larger worklist did not rescue mt1, it made it WORSE +(11 files vs 34). Overflow/rescan is not the serial pathology. This also +independently reconfirms the earlier worklist result -- a bigger worklist is +neutral-to-worse everywhere it has been tried, so overflow behaves as a +MITIGATION rather than a cost. + +CONFOUND IN EXP-1, still open: `gcMarkThreadCount == 1` removes the helper +threads AND disables `cn1GcMutatorAssist` (which bails on +`gcMarkActiveWorkers <= 0`, true only on the parallel path). mt1-vs-mt4 cannot +say which of the two matters. The `noassist` arm (mt4 + CN1_GC_NO_MUTATOR_ASSIST=1) +separates them: if it behaves like mt4 the helpers are decisive; if it behaves +like mt1 the assist is. + +## EXP-2 (observational): per-cycle accounting, no perturbation + +`-DCN1_GC_INSTRUMENT` emits per epoch +`[BIBOP-ADAPT] allocatedMB= triggerMB= occupiedMB= liveMB= reclaimedMB=`, +computed by the SWEEP ITSELF rather than by the census walker -- an independent +instrument for the same quantities. + +Discriminator registered before the run: +- A1 predicts allocatedMB PER CYCLE GROWS over the run as cycles lengthen, and + occupancy grows superlinearly. +- A2 predicts allocatedMB per cycle stays ~pinned to triggerMB, with occupancy a + fixed ~3-4x multiple of it. + +## CORRECTION to the earlier T3 claim + +"Helpers are 99.2% idle" was a NORMALIZATION ERROR: 99.2% was measured against +total wall clock, which is dominated by the mutator's pacing sleep, not against +mark time. The mt1/mt4 result shows the parallel path is decisive at scale. + +## EXP-2 RESULT: A1 CONFIRMED, A2 REFUTED + +Per-cycle accounting from `-DCN1_GC_INSTRUMENT` (61.98s run, 2916 files, 10756MB): + +| epoch | allocatedMB | triggerMB | occupiedMB | liveMB | reclaimedMB | +|-------|-------------|-----------|------------|--------|-------------| +| 13 | 35.1 | 96 | 157 | 41 | 39.6 | +| 14 | 84.1 | 48 | 2602 | 68 | 35.6 | +| 15 | 2518.8 | 24 | 5082 | 699 | 35.9 | +| 16 | 2427.4 | 24 | 5238 | 1156 | 1844.1 | +| 17-21 | 192.0 | 24 | 338-677 | 41-96 | 47-226 | + +ONE CYCLE ALLOCATED 2518MB AGAINST A 24MB TRIGGER -- a 105x overshoot. + +- A2 predicted allocatedMB pinned near triggerMB. It is not: the trigger loses + all control once a cycle is in flight, because a new cycle cannot start until + the current one finishes. REFUTED. +- A1 predicted allocated-per-cycle grows as cycles lengthen. Confirmed: + 35 -> 84 -> 2519 -> 2427. +- The LIVE set is 41-1156MB for the whole run. Occupied peaks at 5238MB, ~5x + live, and the 10.3GB footprint is entirely produced by epochs 14-16. +- Epochs 17-21 pin at exactly 192.0MB = `CN1_BIBOP_GC_MAX_TRIGGER_BYTES`. + +## EXP-1 CONFOUND CLOSED: it is the HELPERS, not the assist + +| arm | wall (2 reps) | peak MB | +|----------|---------------|---------------| +| mt4 | 77.1 / 102.1 | 10277 / 10257 | +| noassist | 81.1 / 77.9 | 10170 / 10295 | +| mt8 | 77.0 / 66.6 | 9886 / 9848 | + +`CN1_GC_NO_MUTATOR_ASSIST=1` changes nothing, so the mt1 catastrophe is the loss +of the HELPER THREADS, not the loss of the assist. mt8 is marginally better than +mt4 on both time and peak -- diminishing returns past 4, consistent with A1. + +Every arm in every round emits byte-identical C (`d6e7d4e2e350`). + +## Conclusion + +Mark throughput is the gating resource. Allocation is bounded only by the pacing +park, whose cap is sized off AVAILABLE MACHINE RAM (`fm/8`, or `fm/2` for a +thread flagged high-throughput), so a mutator can run GBs ahead of a collector +that has not finished its cycle. That is what turns a ~100MB-1.2GB live set into +a 10GB footprint, and what makes wall clock 93% sleep. + +The bound should be a function of the LIVE SET and the collector's measured mark +rate, not of machine RAM. `cn1ConservativeResolve` at 28.4% of GC-thread time is +the largest single mark cost and the obvious first target for raising that rate. + +NOT YET TESTED: whether raising mark throughput actually collapses the footprint +(the feedback loop predicts it should improve BOTH time and memory together, +rather than trading them off as the pacing cap does). + +## MECHANISM CORRECTION (verified in code, not assumed) + +An earlier write-up said the trigger "loses control because a new cycle cannot +start until the current one finishes", implying trigger crossings are lost while +a cycle runs. That is WRONG, and the code says so explicitly: the crossing is +LATCHED to one request per cycle window and is "deliberately NOT suppressed while +a cycle is running". The `!gcCurrentlyRunning` suppression that did behave that +way was removed as issue 5537 -- it starved the collector, because +`bibopBytesSinceGc` is zeroed at cycle START so the mutator re-crosses the +trigger all cycle and every crossing was discarded; by the time the cycle ended +every mutator was parked on the run-ahead cap and no crossing was left to make +the request (measured: mark 40ms, mutator park 212ms). + +The real mechanism is plainer. There is ONE collector running ONE cycle at a +time, `bibopBytesSinceGc` resets at cycle start, and the mutator keeps allocating +for the cycle's whole duration, bounded only by the pacing cap. So + + allocated per cycle = cycle duration x allocation rate + +At seconds per cycle and ~GB/s that is gigabytes -- the measured 2518MB. + +WHY THE CORRECTION MATTERS FOR THE FIX: the trigger cannot be made more +responsive, because it already is. Only two levers remain -- shorten cycles +(mark throughput), or throttle allocation to actual collector progress (a pacing +cap derived from live set and measured mark rate rather than machine RAM). +That is exactly what the mark-throughput dose-response is measuring. + +## CORRECTION to the "Conclusion" section above + +That section says mark throughput gates everything and implies bounding the heap +and raising mark throughput are the same fix, via a feedback loop: +faster marking -> shorter cycles -> less allocation per cycle -> smaller heap. + +THE LOOP DOES NOT CLOSE. Its forward direction is directly testable by varying +mark parallelism, and the heap does not respond: + +| arm | peak MB (2 reps) | wall, min of 2 | +|------|------------------|----------------| +| mt2 | 9904 / 10236 | 62.75 | +| mt4 | 10541 / 10328 | 75.48 | +| mt8 | 10264 / 10366 | 60.14 | +| mt16 | 10211 / ... | 62.54 | + +Going 2 -> 16 markers is a large change in marking speed and peak memory is flat +within 4%. Wall clock is non-monotonic and dominated by machine noise (the same +binary and config measured 62.75s and 93.65s), so it cannot rank these arms at +all; see the load discussion above. + +The supported model is a FLOOR, NOT A LOOP: + + - BELOW a mark-throughput floor (mt1, one marker) the system collapses: 60x at + scale, and no effect whatsoever at small scale. + - ABOVE the floor, heap size is set by the PACING CAP -- which is derived from + machine RAM -- and is insensitive to how fast marking runs. + +So the two levers are SEPARATE, and the earlier claim that they are one fix was +wrong. The memory lever is the cap. The throughput floor is a distinct constraint +that only binds if you fall below it. + +This also weakens the prediction behind `resolveshare.sh`: if faster marking does +not shrink the heap, the converse (a smaller heap making marking cheaper) rests +only on the cache-size argument, not on measurement. The experiment is still +worth running because it measures a different quantity -- resolve's SHARE at a +GIVEN heap size, rather than heap size at a given mark rate -- but a large effect +should no longer be expected. + +--- + +# Round 3: the generational (young-generation) collector + +## Starting point, measured before any change + +Corpus `asm-classes;classes`, release shape `-O3 -flto=thin`, 5 interleaved rounds: + +| arm | wall (min of 5) | peak footprint | +|--------|-----------------|----------------| +| parpar | 1.28s | 1261 MB | +| jdk25 | 1.09s | 534 MB | +| jdk8 | 1.65s | 505 MB | + +1.17x slower than JDK 25 and **2.36x its memory**. Memory is the bigger gap AND +the reproducible one (peak footprint is stable to ~3%; wall clock on this box +varies with load and needs min-of-N), so memory is the target and wall clock is +reported only to show it did not regress. + +## THERE WAS ALREADY A YOUNG GENERATION IN THE TREE + +Before writing anything: `CN1_NURSERY` is a complete thread-local young +generation -- bump-allocated 64KB blocks in a 64MB arena, objects <= 512 bytes, +minor collection, block tenuring, an adaptive survival-based bypass -- carrying +a comment that it "is not defined anywhere in-tree today". It is compiled by no +gate, so it had rotted. + +Its design avoids the hardest part of a generational collector: the write +barrier PROMOTES ON ESCAPE, so an old object can never reference a young one and +**no card table or remembered set is needed**. That is why it was worth +repairing rather than replacing. + +## The split is favourable -- this is the number that justifies the work + +Minor-collection survival on the self-hosting corpus, once roots were fixed: + + alloc=113715 promoted=20138 survival=17% + alloc=111235 promoted=23160 survival=20% + alloc=115809 promoted=27619 survival=23% + +**~75-85% of small objects die without ever reaching the main heap.** This is +the opposite of the shape where generational collection is pure overhead, and it +is consistent with the earlier census (~680MB of 1.19GB allocated is garbage). + +## Six defects found, each verified by a measurable change + +1. **The nursery barrier REPLACED the SATB insertion half instead of composing + with it.** `CN1_WRITE_BARRIER` is the only barrier the translator emits at an + object store, so defining `CN1_NURSERY` silently removed insertion from the + concurrent collector. cn1_globals.m's own deletion-filter argument names this + build as its one exception -- an exception that existed only because nothing + compiled it. + +2. **The minor collection scanned only PRECISE roots.** That was complete when + the nursery was written and has not been since: frameless object/instance + codegen is default-on and exists precisely so a reference need NOT be pushed + to `threadObjectStack`. Measured: SIGSEGV after ONE minor collection with + frameless on; a clean Java-level result with it off. Fixed by scanning this + thread's own C stack + registers conservatively, which needs no signal and no + stop because it runs ON the mutator. Needs an object-start bitmap (1 bit per + 16-byte granule) so an INTERIOR pointer resolves to its object base; the + backward scan is bounded by `CN1_NURSERY_MAX_OBJECT`, so it is O(32). + +3. **`System.arraycopy` bypassed the nursery barrier**, exactly as it bypasses + both SATB halves and for the same reason -- no per-element setter runs. + `ArrayList.grow` copies through it. + +4. **`cloneArray` had the identical hole.** + +5. **`cn1InNursery()` is an ADDRESS-RANGE test, and promotion does not move the + object.** A promoted container stays physically in the arena forever, so + `!cn1InNursery(target)` read it as "still young" and skipped promoting every + value later stored into it. Replaced with `cn1IsYoungObject()` = in-arena AND + `__heapPosition == -1`. Promotion rose 14% -> 24% on the same workload. + +6. **`FusedFieldInit` emitted a raw C field assignment with NO write barrier** + -- an INDEPENDENT `allocArray` published into a field of an existing object. + This is a REAL DEFECT ON MASTER, not just a nursery one: `CN1_WRITE_BARRIER` + is the SATB insertion half, so an array allocated and installed during a + concurrent mark is recorded nowhere. `FusedConstructor`'s own children are + exempt and correctly get no barrier -- those are carved out of the OWNER's + block by `cn1FusedInstallPrimArray` and have no independent GC identity. + Five sites in the self-hosting corpus, `java.lang.String`'s `char[]` among + them. **Cost: zero.** Default build after the fix is 1261 MB -- identical to + baseline -- with byte-identical emitted C. + +## Three diagnostics added, all `#ifdef`-gated QA-only + +- **`CN1_NURSERY_POISON`** stamps every non-promoted object in a retiring block + with a poisoned class pointer, converting a silent use-after-free into an + immediate fault on the instruction that holds the stale reference. The poison + ENCODES A CLASS ID and prints a legend, so the fault address decodes back to a + class name -- a flat sentinel proves a stale reference exists but not what was + missed, which is the whole diagnosis. +- **`CN1_NURSERY_VERIFY`** checks the generational invariant directly ("nothing + outside the young generation may reference something inside it") by re-running + every object's mark function in a reporting mode. It found 1552 violations of + exactly one shape -- BiBOP owner -> young array -- which is what identified + defect 6, and reports 0 after the fix. +- **`CN1_NURSERY_PROMOTE_ALL`** promotes every object in every retiring block. + It answers one question: is a remaining failure a REACHABILITY gap or a defect + in the promotion machinery? + +### Two verifier traps, both of which first reported a confident clean + +- The first version walked only `allObjectsInHeap`. **BiBOP objects are + deliberately absent from that table** -- that is the point of the page heap -- + so every BiBOP holder read clean. It printed 0 violations against a heap that + was demonstrably corrupt. +- The second still missed **promoted-but-not-yet-registered** objects, which + reach `allObjectsInHeap` only at the next paused mark. Everything the pass had + just promoted was in neither walk. +- Both the verifier and the conservative scan therefore SELF-COUNT + (`holdersScanned=`, `stackScan words= found=`). "0 violations" is exactly what + a verifier that never ran also prints. + +## STATUS: NOT YET CORRECT, AND NOT MEASURED FOR PERFORMANCE + +With all six fixes the nursery still faults on the self-hosting corpus, so no +performance number for it is reported -- a speed figure from a run that crashes +would be meaningless. + +What is established about the remaining gap: + +- It is a **reachability gap, not a promotion-machinery defect**: + `CN1_NURSERY_PROMOTE_ALL` runs the whole corpus to completion (exit 0). +- It is **not root scanning**: the precise-root build (frameless off) and the + conservative build now fail identically. +- It is **not an old->young reference at minor-collect time**: the verifier + reports 0 over BiBOP, the legacy table and pending promotions. +- It is **not memory reuse**. An earlier draft of this section said it was, and + that was wrong: the no-reclaim arm cannot reuse a byte (every reclaim path + disabled, 4GB arena, 192MB allocated) and still fails. What separates the two + no-reuse arms is REGISTRATION -- `PROMOTE_ALL` gives every object a + `__heapPosition` of -2 and hands it to `cn1AddPending`; `NO_RECLAIM` leaves + them at -1 forever. + +## ROOT CAUSE: the main collector cannot see the young generation + +`cn1ConservativeResolve` has no nursery handling, and nothing on the main mark +path traces a young object. So an unpromoted nursery object is INVISIBLE to the +concurrent collector even while it is live -- and every HEAP object reachable +only through it is therefore never marked, and the sweep frees it. The young +object is then holding a dangling pointer into reclaimed heap memory. + +This is a design gap, not rot. The nursery's safety argument covers exactly one +direction: eager promotion on escape guarantees old-never-references-young. The +CONVERSE -- young references old, so live young objects must be roots for the +major collection -- has no mechanism at all. + +It explains every observation: `PROMOTE_ALL` works because registration makes +the whole young generation visible to the major mark; `NO_RECLAIM` fails because +nothing is registered; the invariant verifier reports 0 because it checks +old->young, which is the direction that IS maintained; and the failure needs +both several minor collections and a major cycle to appear. + +### The young-root pass is implemented, and it is NOT sufficient on its own + +`cn1NurseryMarkYoungRoots` now runs every young object's mark function during the +major mark's root phase, inside the stopped-thread region, once per cycle (same +idiom as `cn1GcScanParkedVirtualThreads`). The object-start bit is also published +LAST and release-ordered, so "bit set" means "header complete" for that walker. + +It runs -- self-counted, `pass=2 blocks=4 objects=1298` -- and the corpus still +faults, at the same site (`String.equals` reached from `findDeclaredMethod`) +after the same 8 minor collections. Note how FEW blocks it sees: with the nursery +absorbing most small objects, BiBOP volume drops and only TWO major cycles happen +before the failure, so this pass is not where the run is spending its risk. + +So the analysis above is necessary but incomplete: there is at least one more +defect. The next thing to examine is per-object teardown -- an object dying in +the nursery never reaches `cn1BibopReclaimSlot`-equivalent cleanup, so monitor +data, finalizers and native peers keyed by ADDRESS survive it, and a recycled +address inherits them. + +## Also found: an ablation arm that does not compile + +`-DCN1_DISABLE_CONSERVATIVE_GC_ROOTS` fails to build: `gcPthreadValid` is +declared only under `CN1_CONSERVATIVE_GC_ROOTS`, while `CN1_RESUME_THREAD` uses +it unconditionally. Same class of rot as the nursery itself -- a documented +configuration that no gate compiles. + +## Reproducing + +`build-selfhost.sh` gained `CN1_SELFHOST_JAVA_OPTS`, which reaches the +TRANSLATOR that emits the C rather than the C compiler -- that is how a +codegen-level ablation is run: + +```bash +CN1_SELFHOST_JAVA_OPTS="-Dcn1.frameless.objects=false -Dcn1.frameless.instance=false" \ +CN1_SELFHOST_CFLAGS="-DCN1_NURSERY -DCN1_NURSERY_VERIFY" ./build-selfhost.sh -O3 +``` + +--- + +# Round 4: where the memory actually is + +## Baseline re-measured on a QUIET machine + +The 1.17x figure in round 3 was a LOAD ARTIFACT. Re-run at load ~2 (serial marking, +the shipping default): + +| arm | wall (min of 5) | peak footprint | +|--------|-----------------|----------------| +| parpar | 0.92s | 1241 MB | +| jdk25 | 0.66s | 522 MB | + +**1.39x time, 2.38x memory.** Any wall-clock ratio quoted from a loaded run on this +box is worthless -- jdk25 itself moved 1.09s -> 0.66s between the two runs. + +## Three memory levers tried; two are dead ends + +- **Parallel marking is neutral-to-worse here.** The source default is SERIAL + (`gcMarkResolveThreadCount` returns a hardcoded 1 from an old isolation + experiment). mt4: 0.91s / 1325MB against mt1's 0.92s / 1241MB -- same time, MORE + memory. The round-2 finding that mark parallelism is decisive held only for a + corpus ~7x larger. +- **The pacing run-ahead cap is NO LONGER BINDING.** Sweeping the multiplier the + choke fix introduced (1x / 2x / 4x of last cycle's occupied bytes) moves peak + by less than noise: 1283-1298MB, 1273-1294MB, 1304-1309MB. Whatever sets the + peak, it is not admission control any more. +- **`malloc_zone_pressure_relief` after every sweep changes nothing.** 1211-1236MB + with it, 1211-1237MB without. Same result as the earlier page-trim experiment + and for the same reason: peak is a high-water mark that is re-dirtied at once. + The call is kept (it is free when it has nothing to return) with + `CN1_GC_NO_MALLOC_RELIEF` to turn it off. + +## The attribution, from vmmap at peak + +Sampling `vmmap --summary` in a tight loop against the live process and keeping the +largest sample (footprint 1.2G): + +| region | dirty | what it is | +|-----------------------|--------|----------------------------------| +| MALLOC_LARGE | 822.5M | the BiBOP arena, genuinely in use | +| MALLOC_SMALL | 207.2M | legacy heap + VM buffers | +| MALLOC_LARGE (empty) | 160.9M | freed by malloc, never returned | +| MALLOC metadata | 4.1M | | + +BiBOP is NOT wasteful: 11287 pages x 64KB = 705.44MB reserved against 708.00MB of +arena taken from malloc, so the slab allocator loses nothing to alignment. The heap +report now prints `PROCESS footprint`, `MALLOC inUse/allocated/idle` and +`ARENA taken` on the same line as the Java total, because the gap between the Java +heap and the process is the whole question and nothing printed them together. + +**~647MB of malloc's reported in-use bytes is attributed by none of these +instruments.** That is recorded as unexplained rather than guessed at. + +## THE ONE THAT PAYS: 30% of the heap is dead objects being held + +`CN1_ALLOC_CENSUS` + `CN1_HEAP_REPORT` at exit: + +``` +occupied 7,606,861 objects 705.10MB + traced 5,249,747 (69%) fresh 2,710 (0%) aging 2,319,546 (30%) dead 34,858 (0%) +``` + +"aging" is `mark == currentGcMarkValue - 1`: marked last cycle, NOT reached this +cycle, and held one more cycle by the sweep's grace rule alone. It is 30% of +occupied bytes, and it lands on exactly the churn classes: + +| class | size | aging | +|------------------------|---------|-------| +| java.lang.Object[] | 161.7MB | 41% | +| char[] | 134.2MB | 30% | +| java.lang.String | 73.6MB | 20% | +| ArrayListIterator | 41.6MB | 29% | +| boolean[] | 13.4MB | 82% | +| asm Subroutine | 9.4MB | 82% | + +That population -- short-lived arrays and iterators -- is precisely what a young +generation removes before it ever reaches the aging pipeline, which is the +independent argument for finishing the nursery. + +### The rule now lives in ONE place + +The liveness test was spelled out at FIVE sites (legacy table scan, BiBOP per-slot +walk, two reference-clearing passes, the weak-child check), each with a comment +insisting they must agree exactly -- because clearing a reference the sweep then +keeps merely wastes a cache entry, while failing to clear one the sweep frees hands +out a dangling pointer. They are now one predicate, `cn1GcSweepReclaims`, with the +window exposed as `CN1_GC_AGING_SLACK` (default 1 = historical behaviour). + +### Measured: CN1_GC_AGING_SLACK=0 + +| slack | wall (3 reps) | peak (3 reps) | files | +|-------|--------------------|----------------------|-------| +| 1 | 1.04 / 0.94 / 0.96 | 1232 / 1212 / 1216MB | 796 | +| 0 | 0.92 / 0.96 / 0.98 | 1051 / 1184 / 1183MB | 796 | + +**60-160MB (5-13%) off peak, no time cost, byte-identical output.** Less than the +~210MB the census suggests, because the aging population is re-created as fast as it +is reclaimed. + +Gates under `CN1_GC_AGING_SLACK=0`: BibopPageFloor, GcOverflowSpiral, +GcUncooperativeThread and **GcHeapIntegrity** (the dangling-reference / +type-confusion gate, i.e. the one that would catch a premature free) all pass. + +DEFAULT NOT CHANGED. The slack is what covers a mark that was incomplete rather +than a heap that was empty -- a page-index miss, a conservative-scan gap -- and on +ParparVM a dangling read is a native crash no Java catch can see. Passing a +minutes-long gate is not the same as a soak. + +## Two pre-existing failures found and separated from this work + +- **`BibopPageFloorIntegrationTest` and `GcOverflowSpiralIntegrationTest` did not + build at all.** Both called the 3-arg `runTranslator`, whose default appType is + `ios` -- which emits the C runtime as OBJECTIVE-C (`cn1_globals.m`, + `nativeMethods.m`) while the CMake project they then build globs only `*.c`. The + runtime was silently excluded and the link failed on whichever natives the app + retained (`cn1Value` for five boxed types, `System.gcIdleWaitMillis`, + `StandardInputStream.readImpl`). Sibling GC tests pass only because their apps + cull all of those. FIXED by passing `"clean"`. Confirmed pre-existing by + stashing every local change and reproducing the identical undefined-symbol set. +- **`GcSteadyStateIntegrationTest` fails on pristine sources too** (its scenario-3 + precondition: "the ceiling is not pressuring this workload", 641MB headroom on + pristine against 524MB with these changes). Not caused by this work, and the + pacing change moves it TOWARD binding, not away. + +--- + +# CORRECTION: the fast numbers came from an uncommitted change that is not safe + +Every wall-clock figure above that looks good (1.28s, 0.92s, "1.39x") was measured +with an UNCOMMITTED working-tree change -- the occupied-derived pacing ceiling. It +is not in HEAD. The committed branch uses `capCeiling = trigger * MAX_CAP_MULTIPLIER` +and is far slower. With that ceiling restored, on a quiet machine, 3 interleaved +rounds, byte-identical output across all arms: + +| arm | wall (min of 3) | peak footprint | +|--------|-----------------|----------------| +| parpar | 3.36s | 1230 MB | +| jdk25 | 0.66s | 523 MB | + +**5.09x slower, 2.35x the memory.** That is the honest position. + +## Why the lift is not a fix, and is now defaulted OFF + +It is genuinely faster -- 0.96s against 4.18-6.05s -- but the speed comes from the +BOUND GOING AWAY, not from the collector improving. `bibopLastCycleOccupiedBytes` +grows without limit in a garbage-heavy workload, so a ceiling derived from it grows +with the garbage and the mutator is never throttled at all. +`GcOverflowSpiralIntegrationTest` measures the consequence directly: + +``` +With no process ceiling the workload peaked at 10863112KB against a live set of a +few hundred bytes ... a number this size means it is tracking the HOST's free RAM +again, and the app grows until the machine complains. +``` + +10.86GB. The test's own text names the invariant the lift violates. + +THREE alternative signals were tried and all measured, so they are not re-tried: + +| ceiling signal | self-hosting wall | verdict | +|---------------------------|-------------------|-------------------------------------| +| none (trigger-derived) | 4.18-6.05s | correct, slow | +| 2x occupied, unbounded | 0.96-1.01s | fast, 10.86GB spiral -- UNSAFE | +| 2x occupied, capped at | 4.90-6.74s | correct, and no faster than none | +| maxTrigger*multiplier | | | +| 8x bibopLastCycleLiveBytes| 3.96-6.05s | that field is NOT the live set | +| 2x (occupied - reclaimed) | 6.53-9.75s | worse | + +The capped-lift row is the important one: the run-ahead this workload needs to avoid +parking is LARGER than any trigger-derived bound, so admission control is simply the +wrong place to fix it. The choke is a symptom of cycles that are long because the +heap is large. + +`CN1_GC_RUNAHEAD_MULT` keeps the knob (0 = off, the default) so the trade is +measurable, and the gates -- GcOverflowSpiral, BibopPageFloor, GcUncooperativeThread, +GcHeapIntegrity -- all pass with it off. + +## Why the trigger cannot simply be allowed to grow + +The obvious alternative is to let the adaptive trigger track the heap, since +`cap = trigger * MAX_CAP_MULTIPLIER` would then lift legitimately. It does not grow +here because its survival ratio uses `liveBytes`, which EXCLUDES grace-marked slots +(`policySurvivors = policyLiveCount - graceMarked`). + +That exclusion is deliberate and correct: including grace-marked slots is what made a +pure-garbage workload look survivor-heavy. Undo it and the spiral's trigger grows, its +cap grows with it, and the runaway is back. The policy is self-consistent -- a +garbage-heavy workload is supposed to keep a low trigger, and the self-hosting +translation IS garbage-heavy (75-85% of small objects die young). + +**Every route out of the choke leads to the same place: stop the garbage reaching the +old generation.** That is the young generation, and it is the one thing that makes the +survival ratio honest rather than gamed. + +## The one memory win that IS safe, measured on the shipping default + +`CN1_GC_AGING_SLACK=0`, lift off, same binary, 3 reps each: + +| slack | peak | wall | +|-------|----------------------|---------------------| +| 1 | 1199 / 1204 / 1166MB | 3.30 / 4.72 / 4.22s | +| 0 | 1085 / 1085 / 1048MB | 4.44 / 3.52 / 4.40s | + +**~120MB, ~10% of peak, consistently, with byte-identical output** and time within +this box's noise. Takes memory from 2.35x jdk25 to ~2.10x. Default still 1; see the +note at `cn1GcSweepReclaims` for why removing a grace window is the user's call. + +--- + +# Round 5: the fix. Parallel marking, measured in the RIGHT configuration + +## The earlier "parallel marking is neutral" result was measured wrong + +Round 4 concluded mt4 was neutral-to-worse. That run had the occupied-derived pacing +lift active, which meant THE CAP NEVER BOUND -- the mutator never parked, so mark +throughput could not affect wall clock by construction. It measured nothing. + +With the pacing bound restored (the shipping configuration, where the mutator really +does wait on the collector), 3 reps each, byte-identical output at every arm: + +| markers | wall | peak | +|---------|----------------------|---------------| +| 1 | 5.24 / 5.31 / 6.85s | 1200-1221MB | +| 4 | 0.99 / 1.07 / 1.12s | 1070-1173MB | +| 8 | 0.99 / 1.04 / 1.38s | 1083-1152MB | + +**~5x faster AND ~10% less memory.** They move together rather than trading off: a +shorter cycle gives the mutator less time to run ahead, so the heap is smaller too. +Flat past 4 markers, which is the same diminishing return earlier rounds saw. + +The default was `int n = 1`, hardcoded behind `#elif 1` by an isolation experiment +from 2026-07-03 whose own note already said "Parallel marking was never re-tested +after the experiment". It is now CPU-derived (the existing path, capped at 4), with +`-DCN1_GC_SERIAL_MARK` to restore the serial arm. + +## Headline, self-hosting corpus, 5 interleaved rounds + +| | wall (min of 5) | peak | +|---|---:|---:| +| parpar | 1.00s | 1093 MB | +| jdk25 | 0.70s | 528 MB | +| jdk8 | 0.99s | 503 MB | + +**1.43x JDK 25 on time, 2.07x on memory. Level with JDK 8 (1.01x).** + +From where this round started -- 5.09x time, 2.35x memory -- with the pacing bound +INTACT rather than removed. + +## ONE UNEXPLAINED RUN, recorded rather than dismissed + +During the slack A/B, one run of the parallel build emitted **11 files instead of +796** (0.37s, 434MB) and was not caught because that loop did not check the exit code. +It has not reproduced in **32 subsequent runs** (12 plain + 20 under the identical +invocation, env var and `/usr/bin/time -l` wrapper). + +It is on the record because parallel marking is exactly the mechanism the 2026-07-03 +isolation experiment suspected of heap corruption, and "did not reproduce in 32 runs" +is not "did not happen". Every benchmark loop now checks the emitted file count. + +## The nursery: FIXED, and it does not pay + +The remaining defect was found, and it was not roots, barriers or the worklist: + +**`gcMarkObject` rejected every nursery object before reaching the promotion branch.** +The conservative-resolve guard -- + +```c +if(!cn1GcTrustedRoots && cn1ConservativeResolve((void*)obj) != obj + && !cn1GcImmortalObjContains(obj)) return; +``` + +-- drops anything the resolver cannot map back to itself, which is how a reference +into a freed slot is refused. A nursery object lives in neither a BiBOP page nor +allObjectsInHeap, so it NEVER resolves. With the nursery branch sitting after that +guard, every promotion routed through `gcMarkObject` was silently discarded: +`cn1PromoteDrain` walked each promoted object and called `gcMarkObject` on its fields, +and every one returned early. Moving the nursery decision to the very top of +`gcMarkObject` fixes it, and the corpus then translates to completion (exit 0, 796 +files) for the first time. + +The finder that produced this answer is the inverse of the invariant verifier: after +promotion, scan every region for words pointing at objects the collection just +declared dead. It reported `stack=0 bibop=0 legacy=0 nursery=15989`, first holder a +promoted `java.util.ArrayList` (heapPos=-2, mark function present) with the dangling +word at **+32 of 48 bytes** -- exactly `java_util_ArrayList_array`. That ruled out +roots, write barriers and the worklist in one measurement, after four rounds of each +being suspected in turn. + +### And with it working, the generational premise does not hold here + +True survival is **51-57%**, not the 14-23% measured earlier -- that low figure was an +ARTIFACT of the dropped promotions. Roughly half of all small objects survive, and +promotion moves them from BiBOP (page-based, cheap sweep) into the LEGACY heap +(table-based, per-object malloc/free). The nursery is a third tier whose survivors +land in the slowest one. + +Measured against the 1.00s/1093MB default: + +| arm | wall | peak | +|---------------------------------------|-------------|-------------| +| nursery, 8MB trigger / 64MB arena | 6.62s | 1321MB | +| nursery, 64MB trigger / 256MB arena | 6.62-8.56s | 1464-1600MB | +| nursery, static scan skipped | 5.97-8.56s | 1464-1474MB | + +**A net loss on every axis.** BiBOP already IS the fast young-object path; what this +VM needed was not a fourth heap but a collector that keeps up with the one it has. +`CN1_NURSERY` stays off, now correct rather than broken, with the finding recorded so +it is not re-attempted on the strength of the old survival number. + +## Final numbers, and a metric that got noisier + +Three independent 5-round bench runs of the shipping configuration: + +| run | parpar wall (min) | jdk25 wall | parpar peak (MAX) | per-round peaks | +|-----|-------------------|------------|-------------------|------------------------| +| 1 | 1.00s | 0.70s | 1093 MB | -- | +| 2 | 0.98s | 0.65s | 1230 MB | -- | +| 3 | 0.98s | 0.65s | 1225 MB | 1070/1056/1225/1143/1068 | + +**Wall clock: 1.51x JDK 25, 1.08x JDK 8** -- level with the JDK the builders actually +fork. Wall clock is now tight (0.98-1.00s across runs), which it never was before. + +**Memory: 2.32x by the MAX definition, ~2.05x by the median round.** Report the MAX, +since a peak is a max -- but note the spread. Peak footprint used to be stable to ~3% +and is now 1056-1225MB run to run. That is a REAL change from parallel marking: cycle +completion now lands at different points relative to the allocation stream, so how much +float is outstanding at the high-water mark varies. Memory conclusions from single runs +are no longer safe; take several rounds. + +## Where the remaining memory is, and what it is NOT + +Census at exit under parallel marking: + +``` +occupied 5,714,416 objects 668.75MB + traced 1,773,219 (31%) fresh 1,171 (0%) aging 2,263,476 (40%) dead 1,676,550 (29%) + bibop 481.58MB legacy 187.17MB +``` + +Only **31% of occupied bytes are reachable**; 69% is float the sweep has not yet +returned. Levers tried against it, all measured, none of which pay: + +| lever | result | +|--------------------------------|-------------------------------------------| +| `CN1_GC_AGING_SLACK=0` | 1053-1133MB vs 1032-1054MB -- gone. Shorter cycles already shrank the aging set, so the knob that was worth 10% under serial marking is worth nothing now | +| GC trigger 48 / 24 / 12 MB | 984-1235 / 1055-1058 / 1119-1159MB -- no trend; 12MB is slower AND larger | +| `malloc_zone_pressure_relief` | nothing (round 4) | +| `CN1_ADOPT_POLICY=0` | 994-1019MB, ~4% -- but adoption is a CORRECTNESS mechanism (split reachability), so not a knob to turn | + +Note the aging-slack result specifically: it was a validated 10% win under serial +marking and is worthless under parallel. A tuning result is only true for the +configuration it was measured in. + +## Validation of the shipping configuration + +- Full `vm/tests` suite: **584 tests, 1 failure** -- `GcSteadyStateIntegrationTest`, + which fails identically on pristine sources (verified by stashing every local change). +- GC gates: GcOverflowSpiral, BibopPageFloor, GcUncooperativeThread, GcHeapIntegrity, + GcMarkCompleteness -- all pass. +- 32 consecutive self-hosting translations, output byte-identical to the JVM + translator's, with the emitted file count checked every run. +- Pacing: **1 park event** for the whole run. The collector keeps up, which is what + makes the remaining 1.51x an AOT-vs-JIT throughput gap rather than a GC stall. + +--- + +# Round 6: the mark worklist. 25% of peak memory was a rescan doing nothing + +`-DCN1_GC_CONFORM` on the parallel build, one run of the self-hosting corpus +(wallMs=1136, 5 GC cycles): + +``` +rescanPasses=68 rescanUseful=0 rescanSlots=10,107,804 rescanPushes=3,384,826 +extSearches=259367 extHits=0 bloomRejects=2,263,038 +graceLegMs=139.2 sortTotalMs=95.1 sortedTotal=2,662,940 +cause=pacingVolume count=1 totalMs=117 +``` + +**Sixty-eight rescan passes walked ten million slots and found ZERO objects the drain +had missed.** The rescan is the fallback for a mark worklist that cannot hold the +frontier; at 65536 entries it overflowed constantly on a heap with millions of live +objects, and every fallback was wasted work. + +| worklist | wall (3 reps) | peak | +|----------|-----------------------|---------------| +| 65536 | 0.97 / 1.01 / 1.24s | 1050-1265MB | +| 1048576 | 0.90 / 0.90 / 0.90s | 791-839MB | +| 4194304 | 0.88 / 0.90 / 0.90s | 777-831MB | + +**~25% off peak memory, a faster and far tighter wall clock.** 4M is no better than 1M +and costs 32MB of table against 8MB, so the default is 1M. + +Time and memory moved together again, which is now the third time: work the collector +does not waste is cycle time the mutator does not spend running ahead. + +## THIS CONTRADICTS ROUNDS 1-2, WHICH WERE RIGHT AT THE TIME + +Earlier rounds concluded "a bigger worklist is neutral-to-worse everywhere it has been +tried" and even that overflow behaves as a MITIGATION. Those were measured under SERIAL +marking. Three tuning results have now flipped when the configuration changed: + +| knob | serial marking | parallel marking | +|-------------------|-----------------------|----------------------| +| marker count | 1 (pinned, "isolation")| 4 -- ~5x faster | +| mark worklist | bigger = worse | bigger = 25% less mem| +| CN1_GC_AGING_SLACK| 0 = 10% less memory | no effect at all | + +**A GC tuning result is only true for the configuration it was measured in.** Every +number in this file should be read with the arm it was taken on. + +## Headline + +| | wall (min of 5) | peak (MAX of 5) | +|---|---:|---:| +| parpar | 0.91s | 860 MB | +| jdk25 | 0.66s | 532 MB | +| jdk8 | 0.93s | 518 MB | + +**1.38x JDK 25 on time, 1.62x on memory -- and FASTER than JDK 8 (0.98x).** +Per-round peaks 810/823/842/853/860MB, so the run-to-run noise parallel marking +introduced in round 5 is gone too. + +Session start was 5.09x time / 2.35x memory, with the pacing bound intact throughout. + +## Test maintenance this forced + +`GcOverflowSpiralIntegrationTest` began failing -- correctly, on its own precondition: +"the grace pass never drained mid-walk, so this workload never pushed enough to +approach the worklist limit and the assertion above proves nothing." A 16x larger +default put the overflow path out of reach of its fixed workload. + +Fixed by pinning `-DCN1_GC_MARK_WORKLIST_SIZE=65536` in that test's own cmake flags. +What it tests is the overflow MECHANISM, which must keep working whatever the default +is; pinning also means a future change to the default cannot silently turn the gate +into a no-op. + +--- + +# FINAL: 1.35x JDK 25 on time, FASTER than JDK 8; 1.59x on memory + +Self-hosting corpus, 5 interleaved rounds, quiet machine, byte-identical output: + +| | wall (min of 5) | peak (MAX of 5) | +|---|---:|---:| +| parpar | 0.88s | 835 MB (789-835) | +| jdk25 | 0.65s | 525 MB | +| jdk8 | 0.91s | 518 MB | + +**vs JDK 25: 1.35x time, 1.59x memory. vs JDK 8: 0.97x time (faster), 1.61x memory.** + +Session start: 5.09x time, 2.35x memory. The pacing bound is intact throughout -- none +of this came from removing admission control. + +## Three changes, all measured, all validated + +1. **Marker threads 1 -> CPU-derived (capped 4).** ~5x wall. The `int n = 1` was pinned + by a 2026-07-03 isolation experiment that was never re-tested. +2. **Mark worklist 65536 -> 1048576 entries.** ~25% peak memory. The 64K worklist + overflowed constantly, and the rescan fallback did 10.1M slot walks for zero useful + work across 68 passes. +3. **`FusedFieldInit` emits `CN1_WRITE_BARRIER`.** A live SATB insertion hole, + independent of any of the above. Zero cost. + +Plus: the pacing lift is defaulted OFF (it was disabling admission control -- 10.86GB +spiral), the sweep's liveness rule is one predicate instead of five copies, and +`CN1_NURSERY` is correct rather than broken though it does not pay. + +## GC is no longer the bottleneck + +`-DCN1_GC_CONFORM` on the final build: + +``` +wallMs=938 cyclesOnDemand=8 +rescanPasses=0 rescanUseful=0 rescanSlots=0 +cause=nativeResume count=198 totalMs=0 +cause=signalStop count=12 totalMs=0 +(no pacingVolume stall at all) +``` + +**The mutator's total GC stall is ~0ms**, against 68 rescan passes and a 117ms pacing +park before. The remaining 1.35x is AOT-vs-JIT execution throughput, which is a codegen +problem, not a collector one. + +Census at exit went from `traced 31% / aging 40% / dead 29%` to +**`traced 70% / aging 11% / dead 19%`** -- the collector now keeps up, and the Java heap +(576MB resident) is comparable to HotSpot's ENTIRE process (525MB). + +## What remains, attributed by vmmap at an 823.5MB peak + +| region | dirty | note | +|----------------------|---------|-------------------------------------| +| MALLOC_LARGE | 488.5MB | BiBOP arena -- accounted (460MB) | +| MALLOC_SMALL | 227.5MB | legacy heap 119.7MB + VM buffers | +| MALLOC_LARGE (empty) | 92.9MB | freed by malloc, never returned | +| __DATA / metadata | 12.0MB | | + +`malloc_zone_pressure_relief` was re-tested against the 92.9MB in THIS configuration +(three results having already flipped on configuration) and still does nothing: +798-820MB with it, 800-813MB without. Peak is a high-water mark that is re-dirtied. + +## Validation + +- Full `vm/tests`: **584 tests, 1 failure** -- `GcSteadyStateIntegrationTest`, which + fails identically on pristine sources. +- GC gates: GcOverflowSpiral, BibopPageFloor, GcUncooperativeThread, GcHeapIntegrity, + GcMarkCompleteness -- all pass. +- Output byte-identical to the JVM translator across every benchmark arm and every + A/B run; the emitted file count is checked on every run. + +--- + +# Round 7: pushed, and what review found + +## The performance is now gated, not just recorded + +`vm/selfhost/perf-guard.sh` fails when the JDK 25 ratio exceeds 2.00x time or 2.10x +memory. Both defaults it protects were set by a measurement that a later configuration +change invalidated, and -- this is the point -- NEITHER REGRESSION WOULD FAIL A SINGLE +FUNCTIONAL TEST IN THIS TREE. Losing parallel marking costs ~5x wall clock and losing +the 1M worklist costs ~25% of peak, and both look like ordinary code. + +The ceilings carry headroom over measured (1.25-1.35x time, 1.59-1.85x memory) while +sitting well under the 5.09x / 2.35x they exist to catch. The bench refuses to print a +ratio unless every arm emitted identical C, so a green guard is also a correctness +result. + +## Re-verified across the master merge + +A merge that touches GC code can cost the gains silently, so the ratios were re-measured +either side of it: time 1.35x -> 1.33-1.34x, memory medians ~2% apart. Held. + +Note how load distorts ABSOLUTE numbers -- jdk25's own peak read 525MB on a quiet box +and 709MB on a loaded one, for the same work. Only ratios measured in one sitting mean +anything here. + +## Two review findings, both real, both mine to have caused + +**The Windows marker pool was uncapped.** The POSIX branch capped a CPU-derived count at +4 and the `_WIN32` branch did not. Harmless while the hardcoded serial default made both +branches unreachable -- defaulting to CPU-derived marking is what turned it into an +exposure. gcMarkPoolEnsure creates a persistent helper per marker, each reserving 16MB of +stack, so a 64-logical-CPU Windows host would reserve ~1GB for helpers the measurements +say do nothing past 4. + +**The staleness guard could not see a DELETION.** It tested existence plus `find -newer`; +a removed or renamed file makes no remaining file newer, so maven never re-ran, `mvn +clean` never ran, and the deleted file survived in target/classes. The JVM translator +then keeps embedding a runtime resource that is gone from the tree -- and because BOTH +sides of the self-host comparison consume that same stale copy, GATE A STILL PASSES. A +gate that cannot fail on a deleted file is not covering deletions. Fixed with a manifest +diff, which the JavaAPI block twenty lines below already used for this exact reason. + +### The negative control caught a bug in the FIX + +The first version staged the manifest inside `target/`, which `mvn clean` deletes -- so +it was gone before it could be compared, the guard rebuilt on every run, and then failed +on the missing file. Only the "must stay SILENT when nothing changed" half of the control +exposed that; "fires on a deletion" passed happily. Now verified three ways: fires on a +deleted resource, fires when it is restored, silent when nothing changed. + +## One failure correctly NOT chased + +`BackendJavaSeRuntimeTest.javaSeSelfTest` fails locally and is master's new backend +module meeting a partially-installed local Maven repository. The whole branch touches +nothing under `backend/`, `demo/` or `maven/` -- verified by listing every changed file +against master -- and CI's build-test (17) and (21) pass. The test SKIPS on a fresh +worktree and runs on a populated one, so a worktree A/B is confounded by build state +rather than by code; scope is the honest discriminator here, not the worktree. diff --git a/vm/selfhost/experiments/src/com/exp/PinProbe.java b/vm/selfhost/experiments/src/com/exp/PinProbe.java new file mode 100644 index 00000000000..f4ccd7baa4b --- /dev/null +++ b/vm/selfhost/experiments/src/com/exp/PinProbe.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.exp; + +/** + * Does ParparVM's conservative stack scan pin objects that are provably dead? + * + * Three arms, three distinct classes so one run compares them in one census: + * + * PinShallow allocated and dropped in a shallow frame + * PinDeep allocated at the bottom of a deep recursion, then unwound + * PinScrub same as PinDeep, then the stack is overwritten before collecting + * + * Every instance is unreachable by the time the collector runs -- nothing holds a + * reference. So a correct precise collector reclaims all of them, and any survivor + * is something the conservative scan mistook a stale stack word for. If Deep >> + * Shallow the depth is what pins; if Scrub << Deep the stale words are the + * mechanism and overwriting them frees the objects. + */ +public class PinProbe { + static final int BATCH = 200000; + static final int DEPTH = 400; + + // Sinks so the allocations cannot be optimised away, without retaining anything. + static int shallowSink, deepSink, scrubSink, scrubberSink; + + static class PinShallow { int a; } + static class PinDeep { int a; } + static class PinScrub { int a; } + + static void allocShallow() { + for (int i = 0; i < BATCH; i++) { + PinShallow p = new PinShallow(); + p.a = i; + shallowSink += p.a; + } + } + + static void allocDeep(int depth) { + if (depth > 0) { + allocDeep(depth - 1); + return; + } + for (int i = 0; i < BATCH; i++) { + PinDeep p = new PinDeep(); + p.a = i; + deepSink += p.a; + } + } + + static void allocScrub(int depth) { + if (depth > 0) { + allocScrub(depth - 1); + return; + } + for (int i = 0; i < BATCH; i++) { + PinScrub p = new PinScrub(); + p.a = i; + scrubSink += p.a; + } + } + + /** + * Walks back down to the same depth writing non-pointer values into locals, so + * every stack slot the allocation loops left behind is overwritten with an + * integer that cannot be mistaken for a heap address. + */ + static void scrub(int depth) { + int a = depth * 3 + 1, b = depth * 5 + 2, c = depth * 7 + 3, d = depth * 11 + 4; + int e = depth * 13 + 5, f = depth * 17 + 6, g = depth * 19 + 7, h = depth * 23 + 8; + if (depth > 0) { + scrub(depth - 1); + } + scrubberSink += a + b + c + d + e + f + g + h; + } + + static void collect(String label) throws Exception { + System.gc(); + // gc() only signals the collector; give it room to finish a cycle so the + // census that follows is reading fresh marks. + Thread.sleep(1500); + System.err.println("[PROBE] after " + label); + } + + public static void main(String[] args) throws Exception { + System.err.println("[PROBE] batch=" + BATCH + " depth=" + DEPTH); + + allocShallow(); + collect("shallow"); + + allocDeep(DEPTH); + collect("deep"); + + allocScrub(DEPTH); + scrub(DEPTH); + collect("deep+scrub"); + + System.err.println("[PROBE] sinks " + shallowSink + " " + deepSink + " " + + scrubSink + " " + scrubberSink); + System.out.println("DONE"); + } +} diff --git a/vm/selfhost/perf-guard.sh b/vm/selfhost/perf-guard.sh new file mode 100755 index 00000000000..91006b92f4c --- /dev/null +++ b/vm/selfhost/perf-guard.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Performance ratchet for the self-hosted translator. +# +# WHY A GATE AND NOT A README LINE: the two defaults this protects were each set by a +# measurement that a later configuration change silently invalidated, and neither +# regression would have failed a single functional test. Marker count and mark-worklist +# size are worth ~5x wall clock and ~25% of peak memory between them, and losing either +# looks exactly like normal code that passes every gate in the tree. +# +# scripts/../vm/selfhost/perf-guard.sh [rounds] +# +# Fails when the ratio against JDK 25 exceeds the ceilings below. The bench itself +# refuses to print a ratio unless every arm emitted byte-identical C, so a green result +# here is also a correctness result. +# +# CEILINGS carry headroom over the measured values rather than hugging them, because +# this box's wall clock moves with load and peak footprint moves run to run: +# +# measured (5 rounds, quiet) ceiling what breaching it means +# time 1.33-1.35x 2.00x serial marking is back +# memory 1.59-1.80x 2.10x the 64K mark worklist is back +# +# Before these two landed the same corpus measured 5.09x time and 2.35x memory, so both +# ceilings sit well clear of noise and well below the regression they exist to catch. +# +# VALIDATED BY WATCHING IT FAIL, against a deliberately regressed build +# (-DCN1_GC_SERIAL_MARK): reports time=31.74x memory=2.42x, prints both REGRESSION lines +# and exits 1. +# +# DO NOT PIPE IT. `perf-guard.sh | tee log` reports $? from tee, not from this script, so +# the run looks green while the gate is screaming -- which is exactly how the failure +# above first read as a pass. Redirect instead (`perf-guard.sh > log 2>&1`), or set +# `set -o pipefail` in the caller. +set -e +cd "$(dirname "$0")" +ROUNDS="${1:-5}" +T="$(cd ../.. && pwd)/vm/selfhost/target" +: "${CN1_SELFHOST_BIN:=$T/parpar-O3}" +export CN1_SELFHOST_BIN +MAX_TIME="${CN1_PERF_MAX_TIME:-2.00}" +MAX_MEM="${CN1_PERF_MAX_MEM:-2.10}" +OUT=$(mktemp -t cn1perf) +./bench-selfhost.sh "$T/javaapi-classes;$T/asm-classes;$T/classes" \ + com_codename1_tools_translator_ByteCodeTranslator com.codename1.tools.translator \ + "$ROUNDS" | tee "$OUT" +LINE=$(grep '^vs jdk25:' "$OUT" || true) +if [ -z "$LINE" ]; then + echo "perf-guard: FAIL -- no jdk25 ratio line. The bench refuses to print ratios when" + echo " the arms did not emit identical C, so treat this as a correctness failure." + exit 1 +fi +TIME=$(echo "$LINE" | sed -n 's/.*time \([0-9.]*\)x.*/\1/p') +MEM=$(echo "$LINE" | sed -n 's/.*memory \([0-9.]*\)x.*/\1/p') +echo "perf-guard: time=${TIME}x (max ${MAX_TIME}x) memory=${MEM}x (max ${MAX_MEM}x)" +FAIL=0 +awk -v v="$TIME" -v m="$MAX_TIME" 'BEGIN{exit !(v>m)}' && { echo "perf-guard: TIME REGRESSION"; FAIL=1; } +awk -v v="$MEM" -v m="$MAX_MEM" 'BEGIN{exit !(v>m)}' && { echo "perf-guard: MEMORY REGRESSION"; FAIL=1; } +[ "$FAIL" = 0 ] && echo "perf-guard: OK" +exit $FAIL diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java b/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java new file mode 100644 index 00000000000..dd815624bde --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/ArchiveClassScanner.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * Stub for the self-hosted translator build. See {@code vm/selfhost/README.md}. + * + * The real class reads a jar with java.util.zip, which JavaAPI has no business + * gaining -- it is mirrored by Ports/CLDC11, where the package does not belong. + * It is reachable only from NativeSignatureVerifier's offline command-line entry + * point; a translation never reads an archive, because every caller extracts a jar + * into a directory of class files first. + */ +final class ArchiveClassScanner { + private ArchiveClassScanner() { + } + + static void collect(File archive, List into) throws IOException { + throw new UnsupportedOperationException( + "archive scanning is not built into this translator; pass a directory of class files"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java b/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java new file mode 100644 index 00000000000..b371d7d2afc --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/DebugSymbolCompressor.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +/** + * Stub for the self-hosted translator build. See {@code vm/selfhost/README.md}. + * + * The real class gzips the on-device-debug symbol table with java.util.zip, which + * JavaAPI has no business gaining -- it is mirrored by Ports/CLDC11, where the + * package does not belong. Reached only when cn1.onDeviceDebug is set, which is + * off by default. + */ +final class DebugSymbolCompressor { + private DebugSymbolCompressor() { + } + + static byte[] gzip(ByteArrayOutputStream raw) throws IOException { + throw new UnsupportedOperationException( + "on-device-debug symbols are not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java new file mode 100644 index 00000000000..e29a755c8dd --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptBundleWriter.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptBundleWriter { + private JavascriptBundleWriter() { + } + + static void write(File outputDirectory, List classes) throws IOException { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java new file mode 100644 index 00000000000..113c3b99a6e --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptMethodGenerator.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptMethodGenerator { + private JavascriptMethodGenerator() { + } + + static String generateClassJavascript(ByteCodeClass cls, List allClasses) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java new file mode 100644 index 00000000000..a6c9bec6385 --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptReachability.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptReachability { + private JavascriptReachability() { + } + + /// Stub: the JavaScript target is excluded from the self-hosted build, so the + /// per-application fact cache it clears does not exist here. + static void resetExportedFacts() { + } + + static int run(List classes, List classPool, + String[] nativeSources) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java new file mode 100644 index 00000000000..4d1c96eced1 --- /dev/null +++ b/vm/selfhost/stubs/com/codename1/tools/translator/JavascriptSuspensionAnalysis.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import java.util.List; + +/** + * See {@code vm/selfhost/README.md}. + * + * Stub for the self-hosted translator build, which does the clean/ios/macos + * targets only. Replaced on the source path -- the real class is never compiled + * into that binary, and none of these methods is reachable in it. + * + * They throw rather than returning a plausible value: the JavaScript target is + * selected explicitly, so reaching one of these would mean the binary was asked + * for a target it was not built with, and that should be loud. + */ +final class JavascriptSuspensionAnalysis { + private JavascriptSuspensionAnalysis() { + } + + static int run(List classes, java.io.File outputDirectory) { + throw new UnsupportedOperationException("JavaScript target not built into this translator"); + } +} diff --git a/vm/selfhost/verify-output-neutral.sh b/vm/selfhost/verify-output-neutral.sh new file mode 100755 index 00000000000..54073a66a59 --- /dev/null +++ b/vm/selfhost/verify-output-neutral.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# Proves a translator source change does not alter the emitted C. +# +# verify-output-neutral.sh capture # run the JVM translator, save the tree +# verify-output-neutral.sh compare # diff two captured trees +# +# Gate A (in verify-selfhost.sh) compares the JVM translator against the native one +# and CANNOT see this: a refactor lands on both sides at once, so both move together +# and the gate stays green while every emitted signature changes. This runs the JVM +# translator alone, before and after, over the same corpus. +# +# Same fixed output path and constructed environment as verify-selfhost.sh, and for +# the same reasons: the generated CMakeLists embeds srcRoot.getAbsolutePath(), and +# the translator reads its knobs from getenv. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +W="$REPO/vm/selfhost/target/neutral" +OUT="$W/out" + +# Renumber label_L by ORDER OF FIRST APPEARANCE within the file, rather than +# erasing every label to one token. Erasing them makes a jump that was retargeted +# from one existing label to another compare EQUAL -- which is precisely the +# code-generation regression this comparison is for. Renumbering keeps the identity +# relationships and still absorbs the switch from identity-hash names to sequential +# ones. +cn1_canon_labels() { + awk ' + # Reset the mapping at every generated FUNCTION boundary. + # + # The branch numbers labels with a METHOD-LOCAL counter, so L0 and L1 recur + # in every method, while master derives them from ASM identities that are + # distinct across the whole file. A file-wide map therefore folds the second + # method'"'"'s L0 onto the first method'"'"'s token on one side and not the other, + # and reports identical output as a codegen difference -- the mirror of the + # erase-everything bug, generating false positives instead of hiding real + # ones. Per-function scope matches how the names are actually minted. + /^[A-Za-z_][A-Za-z0-9_ \*]*\(/ { delete seen; k = 0 } + { + line = $0 + out = "" + # The number leaks into derived identifiers too -- catch_L, + # restoreToL, tryBlockOffsetL -- which must share the numbering + # WITHIN a function or every try/catch file reports as changed forever. + while (match(line, /(label_L|catch_L|restoreToL|tryBlockOffsetL)[0-9]+/)) { + pre = substr(line, 1, RSTART - 1) + tok = substr(line, RSTART, RLENGTH) + line = substr(line, RSTART + RLENGTH) + nstart = match(tok, /[0-9]+$/) + kind = substr(tok, 1, nstart - 1) + num = substr(tok, nstart) + if (!(num in seen)) { seen[num] = ++k } + out = out pre kind seen[num] + } + print out line + }' "$1" +} + +case "${1:?usage: capture | compare | vs-master}" in +vs-master) + # Compare THIS branch's translator against MASTER's over one corpus. + # + # This is the comparison the other two modes cannot make. They run the same + # translator twice, so a change that lands on the branch is present on both + # sides and they stay green while every emitted signature moves. That blind + # spot cost a full bisect: four native screenshot legs reported a four-pixel + # layout shift, and the cause was a codegen change this script reported as + # neutral because it was neutral -- against itself. + # + # The corpus is a small app compiled against MASTER's JavaAPI, so JavaAPI is + # held constant and the translator is the only variable. Differences in + # deterministic label names and local-variable declaration order are expected + # and are normalised out; anything else is a real codegen change and should be + # a deliberate one. + MW="${CN1_MASTER_WORKTREE:-/tmp/cn3-master}" + [ -d "$MW/vm" ] || { echo "no master worktree at $MW"; echo " git worktree add $MW origin/master"; exit 1; } + MTR="$MW/vm/ByteCodeTranslator/target/classes" + MAPI="$MW/vm/JavaAPI/target/classes" + for d in "$MTR" "$MAPI"; do + [ -d "$d" ] || { echo "missing $d -- build master's translator and JavaAPI first:"; \ + echo " (cd $MW/vm && mvn -q -B -pl ByteCodeTranslator,JavaAPI package -DskipTests)"; exit 1; } + done + APP="${CN1_NEUTRAL_APP:-/tmp/cmpcls}" + [ -d "$APP" ] || { echo "no corpus app at $APP (set CN1_NEUTRAL_APP)"; exit 1; } + ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + rm -rf "$W/m-tree" "$W/b-tree" "$OUT" + for side in m b; do + [ "$side" = m ] && TR="$MTR" || TR="$REPO/vm/ByteCodeTranslator/target/classes" + mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$MAPI;$APP" "$OUT" CmpApp com.cmp CmpApp 1.0 clean none ) > "$W/$side.log" 2>&1 \ + || { echo "$side side FAILED"; tail -5 "$W/$side.log"; exit 1; } + mv "$OUT" "$W/$side-tree" + done + # The C runtime is copied verbatim and this branch edits it on purpose, so it is + # not part of the codegen question. + # Single backslash: this value goes to grep -E, where \. is a literal dot. Two + # backslashes made it "a literal backslash followed by any character", so NOTHING + # matched and the four copied runtime files were counted as codegen differences -- + # which is the 115-versus-119 gap that should have been chased when it appeared. + RUNTIME='^(cn1_globals\.[ch]|nativeMethods\.c|cn1_intrinsics\.h|java_io_File_runtime\.c|cn1-source-manifest\.txt)$' + # Walk the UNION of both trees, not master's listing. A file the branch emits and + # master does not would never be visited by a master-only loop, so a whole new + # generated class could appear and the gate would report neutral. + ( cd "$W/m-tree/dist/CmpApp-src" 2>/dev/null && ls ) > "$W/m.list" 2>/dev/null || : > "$W/m.list" + ( cd "$W/b-tree/dist/CmpApp-src" 2>/dev/null && ls ) > "$W/b.list" 2>/dev/null || : > "$W/b.list" + sort -u "$W/m.list" "$W/b.list" > "$W/all.list" + n=0 + while read -r base; do + [ -n "$base" ] || continue + echo "$base" | grep -qE "$RUNTIME" && continue + case "$base" in *.c|*.h) ;; *) continue ;; esac + mf="$W/m-tree/dist/CmpApp-src/$base"; bf="$W/b-tree/dist/CmpApp-src/$base" + if [ ! -f "$mf" ]; then echo " ONLY IN BRANCH: $base"; n=$((n+1)); continue; fi + if [ ! -f "$bf" ]; then echo " ONLY IN MASTER: $base"; n=$((n+1)); continue; fi + if ! diff -q <(cn1_canon_labels "$mf") <(cn1_canon_labels "$bf") >/dev/null; then + [ $n -lt 12 ] && echo " differs: $base" + n=$((n+1)) + fi + done < "$W/all.list" + echo "VS-MASTER: $n generated file(s) differ beyond label naming" + if [ "$n" != 0 ]; then + echo "Each one is a codegen change against master. Confirm every one is intended." + # Exit NONZERO. Printing a finding and returning 0 makes every caller read a + # real mismatch as a passing check, which is the failure mode this whole + # script exists to prevent. + exit 1 + fi + ;; +capture) + TAG="${2:?}" + TR="$REPO/vm/ByteCodeTranslator/target/classes" + ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + JAPI="$REPO/vm/selfhost/target/javaapi-classes" + # Same staleness guard verify-selfhost.sh carries: a source newer than its class + # would capture the OLD translator under the NEW tag and report a real change as + # neutral -- the exact failure this script exists to prevent. + newest="$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TR" -print -quit 2>/dev/null || true)" + [ -z "$newest" ] || { echo "STALE: $TR older than $newest -- run mvn package first" >&2; exit 1; } + rm -rf "$W/$TAG-tree" "$OUT"; mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ + CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" \ + "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator \ + clean "$JAPI;$REPO/vm/selfhost/target/asm-classes;$REPO/vm/selfhost/target/classes" \ + "$OUT" com_codename1_tools_translator_ByteCodeTranslator \ + com.codename1.tools.translator com_codename1_tools_translator_ByteCodeTranslator \ + 1.0 clean none ) > "$W/$TAG.log" 2>&1 \ + || { echo "capture $TAG FAILED"; tail -20 "$W/$TAG.log"; exit 1; } + mv "$OUT" "$W/$TAG-tree" + n=$(find "$W/$TAG-tree" -type f | wc -l | tr -d ' ') + [ "$n" -gt 10 ] || { echo "VACUOUS: only $n files"; exit 1; } + echo "captured $TAG: $n files" + ;; +compare) + A="$W/${2:?}-tree"; B="$W/${3:?}-tree" + for d in "$A" "$B"; do [ -d "$d" ] || { echo "no $d"; exit 1; }; done + na=$(find "$A" -type f | wc -l | tr -d ' ') + if diff -rq "$A" "$B" > "$W/neutral.txt" 2>&1; then + echo "OUTPUT-NEUTRAL: PASS -- $na files byte-identical" + else + echo "OUTPUT-NEUTRAL: FAIL -- $(grep -c . "$W/neutral.txt") differing paths" + head -20 "$W/neutral.txt"; exit 1 + fi + ;; +*) echo "usage: capture | compare " >&2; exit 1 ;; +esac diff --git a/vm/selfhost/verify-selfhost.sh b/vm/selfhost/verify-selfhost.sh new file mode 100755 index 00000000000..28fb280724d --- /dev/null +++ b/vm/selfhost/verify-selfhost.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Validation gates for the self-hosted translator. +# +# verify-selfhost.sh +# +# Compares the C emitted by the JVM-hosted translator against the C emitted by the +# native one. The comparison is on the emitted SOURCE, never on the compiled binary: +# clang is not what is under test, and gating on object code would fail for toolchain +# reasons that have nothing to do with the VM. +# +# Gate D runs first and is the cheap one: the native translator against itself. If it +# is not self-consistent, nothing downstream means anything, and the cause is VM +# nondeterminism rather than a difference between the two runtimes. +# +# Gate A is the headline: same program, different runtime, identical output. +# +# Both sides run into the SAME absolute output path, sequentially, with the tree +# moved aside between runs. The generated CMakeLists embeds +# srcRoot.getAbsolutePath(), so running in one place removes a whole class of false +# differences rather than normalizing it away afterwards. Both also run under a +# constructed environment: the translator reads its knobs from getenv (see +# Util.getProperty), so a stray CN1_* variable would change one side's output. +set -e +cd "$(dirname "$0")" +REPO="$(cd ../.. && pwd)" +J8="${JDK_8_HOME:?set JDK_8_HOME to a working JDK 8}" +CLASSES="${1:?usage: verify-selfhost.sh }" +APP="${2:?}" +PKG="${3:?}" + +PARPAR="$REPO/vm/selfhost/target/parpar" +[ -x "$PARPAR" ] || { echo "no $PARPAR -- run build-selfhost.sh first"; exit 1; } +JAPI="$REPO/vm/selfhost/target/javaapi-classes" +TR="$REPO/vm/ByteCodeTranslator/target/classes" +ASM="$(cat "$REPO/vm/ByteCodeTranslator/target/selfhost-asm-classpath.txt")" + +# The JVM side of gate A runs target/classes, which nothing in this script builds +# -- build-selfhost.sh compiles the translator only for the NATIVE side. A source +# edit that has not been through `mvn package` therefore makes gate A compare the +# new translator against the old one, and it reports the intended change as a VM +# divergence. That has happened; the diff pointed at java_util_ArrayDeque.c and +# looked exactly like a real one. Maven's own incremental check does not save us +# here either -- it answered "Nothing to compile - all classes are up to date" +# for a source three hours newer than its class, so this compares the trees +# directly rather than trusting it. +newest_src="$(find "$REPO/vm/ByteCodeTranslator/src" -name '*.java' -newer "$TR" -print -quit 2>/dev/null || true)" +if [ -n "$newest_src" ]; then + echo "STALE: $TR is older than $newest_src" >&2 + echo "gate A would compare the new translator against the old one. Run:" >&2 + echo " (cd $REPO/vm && mvn -q -B -pl ByteCodeTranslator clean package -DskipTests)" >&2 + echo "and restore target/selfhost-asm-classpath.txt, which clean removes." >&2 + exit 1 +fi + +W="$REPO/vm/selfhost/target/verify" +rm -rf "$W"; mkdir -p "$W" +OUT="$W/out" + +# CN1_NATIVE_VERIFY is forwarded explicitly. `env -i` starts from an EMPTY +# environment, so a workflow-level `CN1_NATIVE_VERIFY: strict` never reached the +# translator here and NativeSignatureVerifier.mode() defaulted to OFF -- the gate +# reported a mode it was not running in, which is the failure this whole script +# exists to prevent. Forwarded rather than hard-coded so a local run without it set +# behaves as it always did. +run() { + local tag=$1; shift + mkdir -p "$OUT" + ( cd "$W" && env -i PATH=/usr/bin:/bin HOME="$HOME" TMPDIR=/tmp LC_ALL=C \ + CN1_NATIVE_VERIFY="${CN1_NATIVE_VERIFY:-}" \ + CN1_RESOURCE_PATH="$REPO/vm/ByteCodeTranslator/src" "$@" ) > "$W/$tag.log" 2>&1 \ + || { echo "$tag FAILED"; tail -20 "$W/$tag.log"; exit 1; } + mv "$OUT" "$W/$tag-tree" +} + +jvm_args=( "$J8/bin/java" -cp "$TR:$ASM" com.codename1.tools.translator.ByteCodeTranslator ) +common=( clean "$JAPI;$CLASSES" "$OUT" "$APP" "$PKG" "$APP" 1.0 clean none ) + +run parpar1 "$PARPAR" "${common[@]}" +run parpar2 "$PARPAR" "${common[@]}" +run jvm "${jvm_args[@]}" "${common[@]}" + +files=$(find "$W/jvm-tree" -type f | wc -l | tr -d ' ') +bytes=$(find "$W/jvm-tree" -type f -exec cat {} + | wc -c | tr -d ' ') +# A comparison of two empty trees is not a passing comparison. +[ "$files" -gt 10 ] || { echo "VACUOUS: only $files files emitted"; exit 1; } +echo "corpus: $APP -- $files files, $bytes bytes" + +fail=0 +if diff -rq "$W/parpar1-tree" "$W/parpar2-tree" > "$W/gateD.txt" 2>&1; then + echo "GATE D (parpar vs parpar): PASS" +else + echo "GATE D (parpar vs parpar): FAIL -- $(grep -c . "$W/gateD.txt") paths"; fail=1 +fi +if diff -rq "$W/jvm-tree" "$W/parpar1-tree" > "$W/gateA.txt" 2>&1; then + echo "GATE A (jvm vs parpar): PASS -- $files files byte-identical" +else + echo "GATE A (jvm vs parpar): FAIL -- $(grep -c . "$W/gateA.txt") of $files paths differ" + sed 's|.*/'"$APP"'-src/||;s| and .*||' "$W/gateA.txt" | head -20 + fail=1 +fi + +# Negative control: a comparator nobody has watched fail is not a comparator. Flip one +# byte and require the comparison to notice, so a pass above cannot be a pass by +# accident (a mis-set path, an empty tree, a diff invocation that never ran). +victim=$(find "$W/parpar1-tree" -name '*.c' | sort | head -1) +cp "$victim" "$W/victim.bak" +printf 'x' | dd of="$victim" bs=1 seek=40 conv=notrunc status=none +if diff -rq "$W/jvm-tree" "$W/parpar1-tree" > /dev/null 2>&1; then + echo "NEGATIVE CONTROL: FAIL -- a corrupted tree still compared equal"; fail=1 +else + echo "NEGATIVE CONTROL: PASS -- corruption detected" +fi +cp "$W/victim.bak" "$victim" + +exit $fail diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java index fcd133f7722..bd8168e9c39 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java @@ -196,7 +196,11 @@ private void runFloorProbe(List tempDirs) throws Exception { Path outputDir = Files.createTempDirectory("bibop-page-floor-output"); tempDirs.add(outputDir); - CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "BibopPageFloorApp"); + // "clean", NOT the 3-arg default -- see the matching note in + // GcOverflowSpiralIntegrationTest. The default appType "ios" emits the C runtime + // as Objective-C, which the CMake glob (*.c) silently drops, so the build fails at + // link on natives that have nothing to do with what this test measures. + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "BibopPageFloorApp", "clean"); Path distDir = outputDir.resolve("dist"); Path cmakeLists = distDir.resolve("CMakeLists.txt"); diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java index e3031959ee8..d770ca2763e 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BytecodeInstructionIntegrationTest.java @@ -972,15 +972,19 @@ void handleDefaultOutputWritesOutput(CompilerHelper.CompilerConfig config) throw } @Test - void readFileAsStringBuilderReadsContent() throws Exception { + void readFileAsStringReadsContent() throws Exception { File temp = File.createTempFile("readfile", ".txt"); Files.write(temp.toPath(), "Hello World".getBytes(StandardCharsets.UTF_8)); - Method m = ByteCodeTranslator.class.getDeclaredMethod("readFileAsStringBuilder", File.class); + // readFileAsStringBuilder until the translator had to compile against + // ParparVM's own JavaAPI in order to translate itself: StringBuilder there + // has no indexOf/replace, so replaceInFile works on a String instead and + // this helper returns one. + Method m = ByteCodeTranslator.class.getDeclaredMethod("readFileAsString", File.class); m.setAccessible(true); - StringBuilder sb = (StringBuilder) m.invoke(null, temp); + String contents = (String) m.invoke(null, temp); - assertEquals("Hello World", sb.toString()); + assertEquals("Hello World", contents); temp.delete(); } diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java index 6b6d8ba29f5..f2f50047645 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CleanTargetLinuxIntegrationTest.java @@ -287,6 +287,16 @@ static Path buildHelloCodenameOneElf(String launcherSource) throws Exception { if (Boolean.parseBoolean(System.getenv("CN1_LINUX_FULL_DEBUG"))) { configure.add("-DCN1_DEBUG_INFO_LEVEL=3"); } + // Diagnostic defines, e.g. CN1_GC_VERIFY for the collector's heap-integrity + // checker. Unset it and the build is exactly what it was. + String extraDefines = System.getenv("CN1_LINUX_EXTRA_DEFINES"); + if (extraDefines != null && !extraDefines.trim().isEmpty()) { + configure.add("-DCN1_EXTRA_DEFINES=" + extraDefines.trim()); + } + // Printed so a diagnostic build proves itself from the job log. A define + // that silently fails to reach the compiler leaves a clean-looking run that + // measured nothing, which is worse than no diagnostic at all. + System.out.println("CN1SS:HARNESS: cmake configure: " + String.join(" ", configure)); CleanTargetIntegrationTest.runCommand(configure, cmakeRoot); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), cmakeRoot); Path elf = buildDir.resolve("LinuxHelloMain"); diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java new file mode 100644 index 00000000000..b69e0314c4d --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/CollectionSemanticsIntegrationTest.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Pins ArrayList and IdentityHashMap against a real JDK after both were changed to + * stop allocating. + * + *

ArrayList no longer allocates a backing array in its no-arg constructor -- it + * shares a zero-length one until the first growth, which then allocates exactly ten + * so a small list stays in the size class it always occupied. IdentityHashMap's key + * and value iterators no longer build an Entry per step; only entrySet does, which is + * the only view where a caller can observe one. java.util.HashMap already had that + * split and this map had been missed.

+ * + *

Both changes are invisible when they work and produce wrong answers at the + * edges when they do not -- an empty list that reports the wrong size, a null key + * that reads back as the table's sentinel -- so the JDK is used as the oracle rather + * than a hand-written expectation.

+ */ +class CollectionSemanticsIntegrationTest { + + @Test + void collectionSemanticsMatchTheJvm() throws Exception { + Parser.cleanup(); + + Path sourceDir = Files.createTempDirectory("collection-semantics-sources"); + Path classesDir = Files.createTempDirectory("collection-semantics-classes"); + Path javaApiDir = Files.createTempDirectory("collection-semantics-java-api"); + + Path source = sourceDir.resolve("CollectionSemanticsApp.java"); + Files.write(source, loadAppSource().getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + if (config == null) { + fail("No compatible compiler available for the collection semantics integration test"); + } + assertTrue(CompilerHelper.isJavaApiCompatible(config), + "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); + + CompilerHelper.compileJavaAPI(javaApiDir, config); + + List compileArgs = new ArrayList<>(); + compileArgs.add("-source"); + compileArgs.add(config.targetVersion); + compileArgs.add("-target"); + compileArgs.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + compileArgs.add("-classpath"); + compileArgs.add(javaApiDir.toString()); + } else { + compileArgs.add("-bootclasspath"); + compileArgs.add(javaApiDir.toString()); + compileArgs.add("-Xlint:-options"); + } + compileArgs.add("-d"); + compileArgs.add(classesDir.toString()); + compileArgs.add(source.toString()); + + assertEquals(0, CompilerHelper.compile(config.jdkHome, compileArgs), + "CollectionSemanticsApp should compile against the JavaAPI"); + + Map expected = parseCases(runJavaMain(config, classesDir, javaApiDir)); + assertFalse(expected.isEmpty(), "JVM run should emit cases"); + + CompilerHelper.copyDirectory(javaApiDir, classesDir); + + Path outputDir = Files.createTempDirectory("collection-semantics-output"); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "CollectionSemanticsApp"); + + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "CollectionSemanticsApp-src"); + + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList( + "cmake", + "-S", distDir.toString(), + "-B", buildDir.toString(), + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_OBJC_COMPILER=clang" + ), distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + Path executable = buildDir.resolve("CollectionSemanticsApp"); + String parparOutput = CleanTargetIntegrationTest.runCommand( + Arrays.asList(executable.toString()), buildDir); + assertTrue(parparOutput.contains("DONE"), + "ParparVM run should complete. Output: " + parparOutput); + + Map actual = parseCases(parparOutput); + assertEquals(expected.keySet(), actual.keySet(), "ParparVM should emit the same cases"); + + List differences = new ArrayList<>(); + for (Map.Entry entry : expected.entrySet()) { + if (!entry.getValue().equals(actual.get(entry.getKey()))) { + differences.add(entry.getKey() + + "\n jvm : " + entry.getValue() + + "\n parparvm: " + actual.get(entry.getKey())); + } + } + assertTrue(differences.isEmpty(), + "Collection semantics diverged from the JVM:\n" + String.join("\n", differences)); + + // Named explicitly so a regression points at the change rather than at a + // generic diff. + assertEquals("0", actual.get("empty.size"), "a list never added to must be empty"); + assertEquals("IndexOutOfBounds", actual.get("empty.get0"), + "get(0) on an empty list must still throw"); + assertEquals("1", actual.get("ihm.keyNulls"), + "the key iterator must hand back the null key as null, not the table's sentinel"); + assertEquals("1", actual.get("ihm.valNulls"), + "the value iterator must hand back a null value as null"); + assertEquals("500", actual.get("ihm.bigSeen"), + "key iteration must survive a rehash"); + } + + private Map parseCases(String output) { + Map cases = new LinkedHashMap<>(); + for (String line : output.split("\\R")) { + if (!line.startsWith("CASE|")) { + continue; + } + String body = line.substring("CASE|".length()); + int separator = body.indexOf('|'); + assertTrue(separator > 0, "Malformed case line: " + line); + cases.put(body.substring(0, separator), body.substring(separator + 1)); + } + return cases; + } + + private String loadAppSource() throws Exception { + java.io.InputStream in = CollectionSemanticsIntegrationTest.class + .getResourceAsStream("/com/codename1/tools/translator/CollectionSemanticsApp.java"); + assertNotNull(in, "CollectionSemanticsApp.java test resource should exist"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")) + "\n"; + } + } + + private String runJavaMain(CompilerHelper.CompilerConfig config, Path classesDir, Path javaApiDir) + throws Exception { + String javaExe = config.jdkHome.resolve("bin").resolve(CompilerHelper.executableName("java")).toString(); + ProcessBuilder pb = new ProcessBuilder( + javaExe, + "-cp", + classesDir + System.getProperty("path.separator") + javaApiDir, + "CollectionSemanticsApp" + ); + pb.redirectErrorStream(true); + + Process process = pb.start(); + String output; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, process.waitFor(), "JVM run should exit cleanly. Output: " + output); + return output; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + List configs = CompilerHelper.getAvailableCompilers(target); + for (CompilerHelper.CompilerConfig config : configs) { + if (CompilerHelper.isJavaApiCompatible(config)) { + return config; + } + } + } + return null; + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java index 614023a7b0c..4f7826b7f68 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcHeapIntegrityIntegrationTest.java @@ -169,6 +169,26 @@ private void runGate(List tempDirs) throws Exception { assertTrue(!clean.output.contains("DANGLING REFERENCE"), "The sweep left a surviving object pointing at reclaimed memory.\n" + violationExcerpt(clean.output)); + // A recycled slot is NOT dangling: it holds a live, valid object, just not + // the one the field pointed at. That is why the dangling check above passed + // through the failure a Linux core caught -- ArrayList.add running on a + // charts.compat.Canvas. cn1GcVerifyFieldType asks the other question, whether + // what a field HOLDS is assignable to what it was DECLARED as. + assertTrue(!clean.output.contains("TYPE CONFUSION"), + "A reference field holds an object of an unrelated type -- a live " + + "object was reclaimed and its slot recycled.\n" + + violationExcerpt(clean.output)); + // And prove that detector RAN. Inverting its condition on the first version + // produced no output whatever, which is how it was found to be checking + // nothing; silence from a detector that never executes is indistinguishable + // from silence from a clean heap. + java.util.regex.Matcher ft = java.util.regex.Pattern + .compile("FIELDTYPE checks=(\\d+) findings=(\\d+)").matcher(clean.output); + assertTrue(ft.find(), + "the field-type verifier never reported, so it did not run: " + clean.output); + assertTrue(Long.parseLong(ft.group(1)) > 0, + "the field-type verifier ran but checked no field, which is not a pass: " + + ft.group(0)); assertTrue(clean.output.contains("GC_VERIFY_APP_DONE"), "The workload should run to completion. Output: " + clean.output); // A workload that never finishes a collection cycle never runs the diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java new file mode 100644 index 00000000000..62a89c00af8 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcMarkCompletenessTest.java @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * Every non-static object field a class declares must be traced by that class's + * {@code __GC_MARK_} function. + * + * A field the collector cannot see is a live object it will reclaim, and the + * failure is neither an exception nor a null dereference: the slot is recycled, + * some later object moves in, and the next method called through the stale + * reference reads ITS OWN field layout out of an unrelated object. That is + * indistinguishable from the unchecked-CHECKCAST hazard and just as invisible -- + * a SIGSEGV a long way from the cause, with no Java frame that could catch it. + * + * The shape is not hypothetical. A Linux suite core showed + * java_util_ArrayList_add running on an object whose class word said + * com_codename1_charts_compat_Canvas: the list's backing-array slot held two of + * Canvas's int fields, so the length load faulted. The list was + * Display.pendingIdleSerialCalls, reachable from a static root through a + * private final instance field, and the Canvas in its place was itself live + * (mark epoch 18, not the -1 that means fresh) -- a recycled slot, not garbage. + * + * What this checks is the ONE structural property that makes such a reclaim + * possible from the translator's side: ByteCodeClass emits a mark body from + * `fullFieldList`, filtered to non-static object fields DECLARED by the class + * (inherited ones are the base class's mark function's job). Anything that + * makes a field fall out of that filter -- a descriptor the parser types + * wrongly, a new field kind, a refactor of isObjectType -- silently stops the + * field being traced. Reading the emitted C is the only place that assumption + * is observable. + */ +class GcMarkCompletenessTest { + + /** struct obj__X { ... } -- the layout the mark function has to cover. */ + private static final Pattern STRUCT = + Pattern.compile("struct obj__(\\w+)\\s*\\{(.*?)\\n\\};", Pattern.DOTALL); + /** void __GC_MARK_X(...) { ... } */ + private static final Pattern MARKFN = + Pattern.compile("void __GC_MARK_(\\w+)\\(CODENAME_ONE_THREAD_STATE[^)]*\\)\\s*\\{(.*?)\\n\\}", + Pattern.DOTALL); + /** A JAVA_OBJECT member, i.e. exactly what the collector must follow. */ + private static final Pattern OBJ_FIELD = + Pattern.compile("^\\s*JAVA_OBJECT\\s+(\\w+)\\s*;", Pattern.MULTILINE); + + @Test + void everyDeclaredObjectFieldIsTracedByItsMarkFunction() throws Exception { + Path classes = Files.createTempDirectory("gcmark-classes"); + Path out = Files.createTempDirectory("gcmark-out"); + Path src = Files.createTempDirectory("gcmark-src"); + + // Deliberately covers the shapes that have gone wrong or could: a field + // declared on a BASE class and inherited, a collection field like the one + // the core implicated, an array field, an interface-typed field, and a + // class whose object fields sit among primitives so an offset mistake is + // visible. + Path app = src.resolve("GcMarkApp.java"); + Files.write(app, ("import java.util.*;\n" + + "class MarkBase { Object baseRef; int basePrim; }\n" + + "class MarkMid extends MarkBase { String midRef; }\n" + + "class MarkLeaf extends MarkMid {\n" + + " final ArrayList pending = new ArrayList();\n" + + " int a; Object mixedOne; long b; String[] arrayRef; int c;\n" + + " Runnable iface; Map mapRef;\n" + + "}\n" + + "public class GcMarkApp {\n" + + " static MarkLeaf keep;\n" + + " public static void main(String[] args) {\n" + + " keep = new MarkLeaf();\n" + + " keep.pending.add(new Runnable(){ public void run(){} });\n" + + " keep.mixedOne = new Object();\n" + + " keep.arrayRef = new String[2];\n" + + " keep.iface = new Runnable(){ public void run(){} };\n" + + " keep.mapRef = new HashMap();\n" + + " keep.baseRef = new Object();\n" + + " keep.midRef = \"x\";\n" + + " System.out.println(keep.pending.size());\n" + + " }\n" + + "}\n").getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + org.junit.jupiter.api.Assumptions.assumeTrue(config != null, + "no compiler available that targets a JavaAPI-compatible bytecode level"); + + Path javaApi = Files.createTempDirectory("gcmark-java-api"); + CompilerHelper.compileJavaAPI(javaApi, config); + + List args = new ArrayList(); + args.add("-source"); args.add(config.targetVersion); + args.add("-target"); args.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + args.add("-classpath"); args.add(javaApi.toString()); + } else { + args.add("-bootclasspath"); args.add(javaApi.toString()); + args.add("-Xlint:-options"); + } + args.add("-nowarn"); + args.add("-d"); args.add(classes.toString()); + args.add(app.toString()); + assertTrue(CompilerHelper.compile(config.jdkHome, args) == 0, + "the fixture must compile against JavaAPI"); + + // The translator needs the class library beside the app, as every other + // integration test here stages it. + CompilerHelper.copyDirectory(javaApi, classes); + CleanTargetIntegrationTest.runTranslator(classes, out, "GcMarkApp"); + Path srcRoot = findSrcRoot(out); + + List missing = new ArrayList(); + int classesChecked = 0; + int fieldsChecked = 0; + + try (Stream files = Files.walk(srcRoot)) { + for (Path c : (Iterable) files.filter(p -> p.toString().endsWith(".c"))::iterator) { + String body = new String(Files.readAllBytes(c), StandardCharsets.ISO_8859_1); + Path header = c.resolveSibling(c.getFileName().toString().replaceAll("\\.c$", ".h")); + if (!Files.exists(header)) { + continue; + } + String head = new String(Files.readAllBytes(header), StandardCharsets.ISO_8859_1); + + Matcher mf = MARKFN.matcher(body); + while (mf.find()) { + String cls = mf.group(1); + String markBody = mf.group(2); + // struct obj__X FLATTENS the inherited fields, but the mark + // function deliberately marks only what the class DECLARES and + // chains to its base for the rest -- so requiring every struct + // member here would demand that Error re-mark Throwable's fields. + // The mangled name carries its declaring class, which is the same + // filter ByteCodeClass applies (fld.getClsName().equals(clsName)). + Set declared = new LinkedHashSet(); + for (String f : declaredObjectFields(head, cls)) { + if (f.startsWith(cls + "_")) { + declared.add(f); + } + } + if (declared.isEmpty()) { + continue; + } + classesChecked++; + for (String f : declared) { + fieldsChecked++; + // The emitted body names the field directly, whether it goes + // through gcMarkObject, gcMarkArrayObject or + // cn1GcDiscoverReference (the WeakReference referent, which is + // deliberately not traced but IS handed to the collector). + if (!markBody.contains(f)) { + missing.add(cls + "." + f); + } + } + } + } + } + + // The filter above is only sound because a class DELEGATES to its base, so + // check the delegation actually exists wherever the base declares object + // fields. Without this, "declared by me" and "marked by me" could both be + // empty for a whole hierarchy and the test would still pass. + List brokenChain = new ArrayList(); + try (Stream files2 = Files.walk(srcRoot)) { + for (Path c : (Iterable) files2.filter(p -> p.toString().endsWith(".c"))::iterator) { + String body = new String(Files.readAllBytes(c), StandardCharsets.ISO_8859_1); + Matcher mf = MARKFN.matcher(body); + while (mf.find()) { + String cls = mf.group(1); + String markBody = mf.group(2); + String base = baseOf(body, cls); + if (base != null && !base.equals("java_lang_Object") + && !markBody.contains("__GC_MARK_" + base)) { + brokenChain.add(cls + " -> " + base); + } + } + } + } + assertTrue(brokenChain.isEmpty(), + "__GC_MARK_ must chain to the base class, or the base's declared fields " + + "are traced by nobody: " + brokenChain); + + // A pass that inspected nothing is not a pass. The fixture alone declares + // eight object fields across three classes in one hierarchy. + assertTrue(classesChecked >= 3, + "expected to inspect several classes, saw " + classesChecked); + assertTrue(fieldsChecked >= 8, + "expected to inspect the fixture's object fields, saw " + fieldsChecked); + assertTrue(missing.isEmpty(), + "object field(s) declared but never traced by the class's __GC_MARK_ function -- " + + "the collector cannot see them, so it will reclaim live objects and " + + "recycle their slots: " + missing); + } + + /** + * Proves the check can fail, by deleting one field's mark from a body and + * confirming the comparison notices. A gate nobody has watched fail is not a + * gate, and this one is a string search over generated code -- exactly the kind + * that silently matches everything or nothing. + */ + @Test + void theCheckDetectsAnUntracedField() { + String head = "struct obj__Foo {\n JAVA_OBJECT Foo_kept;\n JAVA_OBJECT Foo_dropped;\n};"; + Set declared = declaredObjectFields(head, "Foo"); + assertTrue(declared.contains("Foo_kept") && declared.contains("Foo_dropped"), + "fixture parse: " + declared); + String markBody = " gcMarkObject(threadStateData, objInstance->Foo_kept, force);"; + List missing = new ArrayList(); + for (String f : declared) { + if (!markBody.contains(f)) { + missing.add(f); + } + } + assertFalse(missing.isEmpty(), "the check must notice a field that is not marked"); + assertTrue(missing.contains("Foo_dropped") && missing.size() == 1, + "it must name exactly the untraced field, got " + missing); + } + + /** The base class name from the emitted `struct clazz` initialiser, or null. */ + private static String baseOf(String body, String cls) { + Matcher m = Pattern.compile("struct clazz class__" + Pattern.quote(cls) + + "\\s*=\\s*\\{(.*?)\\};", Pattern.DOTALL).matcher(body); + if (!m.find()) { + return null; + } + Matcher b = Pattern.compile("&class__(\\w+)\\s*,\\s*(?:base_interfaces|EMPTY_INTERFACES)").matcher(m.group(1)); + return b.find() ? b.group(1) : null; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + for (CompilerHelper.CompilerConfig c : CompilerHelper.getAvailableCompilers(target)) { + if (CompilerHelper.isJavaApiCompatible(c)) { + return c; + } + } + } + return null; + } + + private static Set declaredObjectFields(String header, String cls) { + Set out = new LinkedHashSet(); + Matcher s = STRUCT.matcher(header); + while (s.find()) { + if (!s.group(1).equals(cls)) { + continue; + } + Matcher f = OBJ_FIELD.matcher(s.group(2)); + while (f.find()) { + String name = f.group(1); + // The object header's own slots are not Java fields. + if (name.startsWith("__codenameOne") || name.equals("__heapPosition")) { + continue; + } + out.add(name); + } + } + return out; + } + + private static Path findSrcRoot(Path out) throws IOException { + try (Stream w = Files.walk(out)) { + return w.filter(Files::isDirectory) + .filter(p -> p.getFileName().toString().endsWith("-src")) + .findFirst() + .orElseThrow(() -> new IOException("no generated -src directory under " + out)); + } + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/GcOverflowSpiralIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/GcOverflowSpiralIntegrationTest.java index 403f1ea080b..503e6db764f 100644 --- a/vm/tests/src/test/java/com/codename1/tools/translator/GcOverflowSpiralIntegrationTest.java +++ b/vm/tests/src/test/java/com/codename1/tools/translator/GcOverflowSpiralIntegrationTest.java @@ -232,7 +232,14 @@ private void runSpiralLoad(List tempDirs) throws Exception { Path outputDir = Files.createTempDirectory("gc-overflow-output"); tempDirs.add(outputDir); - CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "GcOverflowSpiralApp"); + // "clean", NOT the 3-arg default. That default is appType "ios", which emits the C + // runtime as OBJECTIVE-C -- cn1_globals.m and nativeMethods.m -- while the CMake + // project this test then builds globs only *.c. The runtime is therefore excluded + // silently, and the link fails on whichever natives the app happens to retain + // (cn1Value for every boxed type, System.gcIdleWaitMillis, readImpl). It looks + // like a GC regression and is a target-type mismatch. Tests whose apps cull all + // of those link anyway, which is why this stayed hidden. + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "GcOverflowSpiralApp", "clean"); Path distDir = outputDir.resolve("dist"); Path cmakeLists = distDir.resolve("CMakeLists.txt"); @@ -241,9 +248,25 @@ private void runSpiralLoad(List tempDirs) throws Exception { Path buildDir = distDir.resolve("build"); Files.createDirectories(buildDir); + // PIN THE WORKLIST SIZE THIS TEST EXISTS TO STRESS. + // + // What is under test here is the OVERFLOW path: when the mark worklist cannot + // hold the frontier, the collector must fall back to the page rescan and still + // produce a sound mark. That mechanism is independent of how big the worklist + // happens to be by default. + // + // The product default was raised from 65536 to 1048576 entries (see + // CN1_GC_MARK_WORKLIST_SIZE -- the rescan was measured doing 10.1 million slot + // walks across 68 passes for zero useful work). At the larger size this test's + // fixed workload no longer comes near the limit, and the test correctly refused + // to pass: "the grace pass never drained mid-walk ... the assertion above proves + // nothing". Pinning the size keeps the test measuring the mechanism rather than + // the current default, and means raising or lowering that default again cannot + // silently turn this gate into a no-op. List cmakeArgs = new ArrayList<>(Arrays.asList( "cmake", "-S", distDir.toString(), "-B", buildDir.toString(), - "-DCMAKE_BUILD_TYPE=Release")); + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_C_FLAGS=-DCN1_GC_MARK_WORKLIST_SIZE=65536")); cmakeArgs.addAll(CompilerHelper.cmakeToolchainArgs()); CleanTargetIntegrationTest.runCommand(cmakeArgs, distDir); CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java new file mode 100644 index 00000000000..9e84582446a --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/UtilStringHelperTest.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Holds {@link Util}'s hand-written string helpers to the JDK regex behaviour they + * replaced. + * + * The translator has to compile against ParparVM's JavaAPI in order to translate + * itself, and String.split/replaceAll are not declared there -- they are among the + * methods BytecodeComplianceMojo rewrites onto com.codename1.util.regex precisely + * because JavaAPI lacks them. The call sites lost the regex rather than JavaAPI + * gaining a second engine, so the risk is that a replacement quietly disagrees and + * changes generated C. These tests compare against the originals directly, so the + * JDK is the oracle rather than a hand-written expectation. + */ +class UtilStringHelperTest { + + private static final String LOCALS_REGEX = "locals\\[(\\d+)\\]\\.data\\.o"; + private static final String LOCALS_REPLACEMENT = "olocals_$1_"; + + @Test + void rewriteLocalObjectRefsMatchesReplaceAll() { + for (String s : localsCases()) { + assertEquals(s.replaceAll(LOCALS_REGEX, LOCALS_REPLACEMENT), + Util.rewriteLocalObjectRefs(s), + "rewriteLocalObjectRefs diverged on: " + s); + } + } + + @Test + void collapseWhitespaceMatchesReplaceAll() { + for (String s : whitespaceCases()) { + assertEquals(s.replaceAll("\\s+", " "), Util.collapseWhitespace(s), + "collapseWhitespace diverged on: " + escape(s)); + } + } + + @Test + void splitWhitespaceMatchesSplit() { + for (String s : whitespaceCases()) { + assertArrayEquals(s.split("\\s+"), Util.splitWhitespace(s), + "splitWhitespace diverged on: " + escape(s)); + } + } + + @Test + void splitLiteralMatchesSplit() { + String[] cases = { + "", ";", ";;", "a", "a;b", "a;b;c", ";a", "a;", "a;;b", ";;a;;b;;", + "a;b;", "a;b;;", " a ; b ", "one" + }; + for (String s : cases) { + assertArrayEquals(s.split(";"), Util.splitLiteral(s, ';'), + "splitLiteral diverged on: " + escape(s)); + } + } + + /** + * The generated-code shapes plus the near misses: a bracket with no digits, a + * digit run that is not followed by ".data.o", and a nested occurrence. These are + * where a hand-written scanner and a regex are most likely to part company. + */ + private List localsCases() { + List cases = new ArrayList(); + for (String s : new String[]{ + "", + "locals[0].data.o", + "locals[12].data.o", + "locals[0].data.o + locals[1].data.o", + "f(locals[3].data.o, locals[44].data.o)", + "locals[].data.o", + "locals[x].data.o", + "locals[0].data.i", + "locals[0].data", + "locals[", + "locals[0", + "locals[0]", + "prefix locals[7].data.o suffix", + "locals[locals[1].data.o].data.o", + "no match here at all", + "LOCALS[0].DATA.O" + }) { + cases.add(s); + } + // Randomised fuzz over the alphabet the pattern cares about, so the oracle + // sees inputs nobody thought to enumerate. + Random r = new Random(20260909L); + char[] alphabet = {'l', 'o', 'c', 'a', 's', '[', ']', '.', 'd', 't', '0', '1', '9', ' ', 'x'}; + for (int i = 0; i < 3000; i++) { + StringBuilder b = new StringBuilder(); + int len = r.nextInt(24); + for (int j = 0; j < len; j++) { + b.append(alphabet[r.nextInt(alphabet.length)]); + } + if (r.nextBoolean()) { + b.append("locals[").append(r.nextInt(200)).append("].data.o"); + } + cases.add(b.toString()); + } + return cases; + } + + private List whitespaceCases() { + List cases = new ArrayList(); + // 0x0B is the vertical tab: Java's \s includes it and Character.isWhitespace + // does not, which is the difference most likely to be got wrong. + String vt = String.valueOf((char) 0x0B); + for (String s : new String[]{ + "", " ", " ", "a", "a b", "a b", " a b ", "\ta\tb\t", "a\nb", + "a" + vt + "b", "a\fb", "a\r\nb", "JAVA_OBJECT me", " leading", "trailing ", + " both ", "a \t\n b" + }) { + cases.add(s); + } + Random r = new Random(20260910L); + char[] alphabet = {' ', '\t', '\n', 0x0B, '\f', '\r', 'a', 'b', '*'}; + for (int i = 0; i < 3000; i++) { + StringBuilder b = new StringBuilder(); + int len = r.nextInt(16); + for (int j = 0; j < len; j++) { + b.append(alphabet[r.nextInt(alphabet.length)]); + } + cases.add(b.toString()); + } + return cases; + } + + private String escape(String s) { + StringBuilder b = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < 0x20) { + b.append("\\x").append(Integer.toHexString(c)); + } else { + b.append(c); + } + } + return b.toString(); + } +} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java new file mode 100644 index 00000000000..667062c9344 --- /dev/null +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/CollectionSemanticsApp.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Exercises the parts of ArrayList and IdentityHashMap that were changed to stop + * allocating: ArrayList no longer allocates a backing array until the first growth, + * and IdentityHashMap's key and value iterators no longer build an Entry per step. + * + * Every line is compared against a real JDK run, so the JDK is the oracle rather + * than a hand-written expectation. + */ +public class CollectionSemanticsApp { + static void emit(String k, Object v) { + System.out.println("CASE|" + k + "|" + v); + } + + public static void main(String[] args) { + // ---- an ArrayList that is never added to ------------------------------- + List empty = new ArrayList(); + emit("empty.size", empty.size()); + emit("empty.isEmpty", empty.isEmpty()); + emit("empty.contains", empty.contains("x")); + emit("empty.indexOf", empty.indexOf("x")); + emit("empty.iterHasNext", empty.iterator().hasNext()); + emit("empty.toArrayLen", empty.toArray().length); + emit("empty.toString", empty.toString()); + empty.clear(); + emit("empty.afterClear", empty.size()); + try { + empty.get(0); + emit("empty.get0", "no throw"); + } catch (IndexOutOfBoundsException err) { + emit("empty.get0", "IndexOutOfBounds"); + } + + // ---- first growth, and growth past it ---------------------------------- + List grow = new ArrayList(); + for (int i = 0; i < 40; i++) { + grow.add(Integer.valueOf(i)); + if (i < 3 || i == 9 || i == 10 || i == 11 || i == 12 || i == 39) { + emit("grow.size@" + i, grow.size() + ":" + grow.get(0) + ":" + grow.get(i)); + } + } + emit("grow.toString", grow.toString()); + emit("grow.indexOf37", grow.indexOf(Integer.valueOf(37))); + + // ---- add-at-front on a fresh list (the growAtFront path) --------------- + List front = new ArrayList(); + front.add(0, "b"); + front.add(0, "a"); + front.add("c"); + emit("front.toString", front.toString()); + emit("front.size", front.size()); + + // ---- insert into the middle of a fresh list (growForInsert) ------------ + List mid = new ArrayList(); + mid.add("x"); + mid.add("z"); + mid.add(1, "y"); + emit("mid.toString", mid.toString()); + + // ---- ensureCapacity on a fresh list ------------------------------------ + ArrayList ec = new ArrayList(); + ec.ensureCapacity(100); + ec.add("only"); + emit("ec.toString", ec.toString()); + + // ---- remove down to empty and re-add ----------------------------------- + List churn = new ArrayList(); + churn.add("p"); + churn.add("q"); + churn.remove("p"); + churn.remove(0); + emit("churn.emptyAgain", churn.size()); + churn.add("r"); + emit("churn.readd", churn.toString()); + + // ---- IdentityHashMap: identity semantics, and all three views ---------- + String k1 = new String("dup"); + String k2 = new String("dup"); + IdentityHashMap ihm = new IdentityHashMap(); + ihm.put(k1, "first"); + ihm.put(k2, "second"); + emit("ihm.size", ihm.size()); + emit("ihm.get1", ihm.get(k1)); + emit("ihm.get2", ihm.get(k2)); + emit("ihm.containsKey1", ihm.containsKey(k1)); + + // Null key and null value must survive the table's NULL_OBJECT sentinel in + // BOTH directions -- this is what the key/value iterators read directly now. + ihm.put(null, "nullkey"); + ihm.put("nullval", null); + emit("ihm.getNullKey", ihm.get(null)); + emit("ihm.getNullVal", String.valueOf(ihm.get("nullval"))); + emit("ihm.sizeWithNulls", ihm.size()); + + int keyNulls = 0, keyCount = 0; + for (Iterator it = ihm.keySet().iterator(); it.hasNext();) { + String k = it.next(); + keyCount++; + if (k == null) { + keyNulls++; + } + } + emit("ihm.keyCount", keyCount); + emit("ihm.keyNulls", keyNulls); + + int valNulls = 0, valCount = 0; + for (Iterator it = ihm.values().iterator(); it.hasNext();) { + String v = it.next(); + valCount++; + if (v == null) { + valNulls++; + } + } + emit("ihm.valCount", valCount); + emit("ihm.valNulls", valNulls); + + int entryCount = 0, entryKeyNulls = 0, entryValNulls = 0; + for (Map.Entry e : ihm.entrySet()) { + entryCount++; + if (e.getKey() == null) { + entryKeyNulls++; + } + if (e.getValue() == null) { + entryValNulls++; + } + } + emit("ihm.entryCount", entryCount); + emit("ihm.entryKeyNulls", entryKeyNulls); + emit("ihm.entryValNulls", entryValNulls); + + // keySet().contains and removal through the key view + Set keys = ihm.keySet(); + emit("ihm.keysContainsK1", keys.contains(k1)); + emit("ihm.keysRemoveK1", keys.remove(k1)); + emit("ihm.sizeAfterRemove", ihm.size()); + + // iterator removal + IdentityHashMap rem = new IdentityHashMap(); + String r1 = new String("r1"); + String r2 = new String("r2"); + rem.put(r1, "1"); + rem.put(r2, "2"); + for (Iterator it = rem.keySet().iterator(); it.hasNext();) { + if (it.next() == r1) { + it.remove(); + } + } + emit("ihm.afterIterRemove", rem.size() + ":" + rem.get(r2)); + + // a map big enough to force a rehash, iterated by key + IdentityHashMap big = new IdentityHashMap(); + Object[] held = new Object[500]; + for (int i = 0; i < held.length; i++) { + held[i] = new Object(); + big.put(held[i], Integer.valueOf(i)); + } + long sum = 0; + int seen = 0; + for (Object o : big.keySet()) { + sum += big.get(o).intValue(); + seen++; + } + emit("ihm.bigSeen", seen); + emit("ihm.bigSum", sum); + + System.out.println("DONE"); + } +}