fix(wasm): size the query buffer from the bytes actually written; reject embedded NUL

.github/workflows/ci.yml

@@ -53,11 +53,13 @@ jobs:
       - name: Build WASM 🏗
         run: |
           if [ "${{ matrix.package.name }}" = "v13" ]; then
-            # Download prebuilt WASM for v13 since it can't build in CI
+            # v13 can't build its WASM in CI: reuse the prebuilt binary, but
+            # compile the wrapper from this checkout so its tests cover src/.
             mkdir -p wasm
             curl -o v13.tgz "https://registry.npmjs.org/@libpg-query/v13/-/v13-13.5.7.tgz"
-            tar -xzf v13.tgz --strip-components=1 package/wasm
+            tar -xzf v13.tgz --strip-components=1 package/wasm/libpg-query.js package/wasm/libpg-query.wasm
             rm v13.tgz
+            pnpm run build:js
           else
             pnpm run build
           fi

templates/full/index.ts

@@ -131,6 +131,7 @@ interface WasmModule {
   stringToUTF8: (str: string, ptr: number, len: number) => void;
   UTF8ToString: (ptr: number) => string;
   getValue: (ptr: number, type: string) => number;
+  HEAPU8: Uint8Array;
 }
 
 let wasmModule: WasmModule;
@@ -156,20 +157,28 @@ function awaitInit<T extends (...args: any[]) => Promise<any>>(fn: T): T {
   }) as T;
 }
 
+const utf8Encoder = new TextEncoder();
+
 function stringToPtr(str: string): number {
   ensureLoaded();
   if (typeof str !== 'string') {
     throw new TypeError(`Expected a string, got ${typeof str}`);
   }
-  const len = wasmModule.lengthBytesUTF8(str) + 1;
-  const ptr = wasmModule._malloc(len);
-  try {
-    wasmModule.stringToUTF8(str, ptr, len);
-    return ptr;
-  } catch (error) {
-    wasmModule._free(ptr);
-    throw error;
+  // libpg_query takes a NUL-terminated C string: an embedded NUL would end the
+  // parse early and silently drop everything after it.
+  if (str.includes('\0')) {
+    throw new Error('Query must not contain a NUL (U+0000) character');
+  }
+  // Size the buffer from the exact bytes written. TextEncoder writes each
+  // unpaired surrogate as U+FFFD — the same bytes a Postgres driver sends.
+  const bytes = utf8Encoder.encode(str);
+  const ptr = wasmModule._malloc(bytes.length + 1);
+  if (!ptr) {
+    throw new Error('Failed to allocate memory for query');
   }
+  wasmModule.HEAPU8.set(bytes, ptr);
+  wasmModule.HEAPU8[ptr + bytes.length] = 0;
+  return ptr;
 }
 
 function ptrToString(ptr: number): string {

templates/full/libpg-query.d.ts

@@ -10,6 +10,7 @@ declare module './libpg-query.js' {
     lengthBytesUTF8: (str: string) => number;
     stringToUTF8: (str: string, ptr: number, len: number) => void;
     UTF8ToString: (ptr: number) => string;
+    HEAPU8: Uint8Array;
   }
 
   const PgQueryModule: () => Promise<WasmModule>;

templates/index.ts

@@ -145,20 +145,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
   }) as T;
 }
 
+const utf8Encoder = new TextEncoder();
+
 function stringToPtr(str: string): number {
   ensureLoaded();
   if (typeof str !== 'string') {
     throw new TypeError(`Expected a string, got ${typeof str}`);
   }
-  const len = wasmModule.lengthBytesUTF8(str) + 1;
-  const ptr = wasmModule._malloc(len);
-  try {
-    wasmModule.stringToUTF8(str, ptr, len);
-    return ptr;
-  } catch (error) {
-    wasmModule._free(ptr);
-    throw error;
+  // libpg_query takes a NUL-terminated C string: an embedded NUL would end the
+  // parse early and silently drop everything after it.
+  if (str.includes('\0')) {
+    throw new Error('Query must not contain a NUL (U+0000) character');
+  }
+  // Size the buffer from the exact bytes written. TextEncoder writes each
+  // unpaired surrogate as U+FFFD — the same bytes a Postgres driver sends.
+  const bytes = utf8Encoder.encode(str);
+  const ptr = wasmModule._malloc(bytes.length + 1);
+  if (!ptr) {
+    throw new Error('Failed to allocate memory for query');
   }
+  wasmModule.HEAPU8.set(bytes, ptr);
+  wasmModule.HEAPU8[ptr + bytes.length] = 0;
+  return ptr;
 }
 
 function ptrToString(ptr: number): string {

versions/13/package.json

@@ -28,7 +28,7 @@
     "wasm:rebuild": "pnpm wasm:make rebuild",
     "wasm:clean": "pnpm wasm:make clean",
     "wasm:clean-cache": "pnpm wasm:make clean-cache",
-    "test": "node --test test/parsing.test.js test/errors.test.js"
+    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js"
   },
   "author": "Constructive <developers@constructive.io>",
   "license": "MIT",

versions/13/src/index.ts

@@ -152,20 +152,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
   }) as T;
 }
 
+const utf8Encoder = new TextEncoder();
+
 function stringToPtr(str: string): number {
   ensureLoaded();
   if (typeof str !== 'string') {
     throw new TypeError(`Expected a string, got ${typeof str}`);
   }
-  const len = wasmModule.lengthBytesUTF8(str) + 1;
-  const ptr = wasmModule._malloc(len);
-  try {
-    wasmModule.stringToUTF8(str, ptr, len);
-    return ptr;
-  } catch (error) {
-    wasmModule._free(ptr);
-    throw error;
+  // libpg_query takes a NUL-terminated C string: an embedded NUL would end the
+  // parse early and silently drop everything after it.
+  if (str.includes('\0')) {
+    throw new Error('Query must not contain a NUL (U+0000) character');
+  }
+  // Size the buffer from the exact bytes written. TextEncoder writes each
+  // unpaired surrogate as U+FFFD — the same bytes a Postgres driver sends.
+  const bytes = utf8Encoder.encode(str);
+  const ptr = wasmModule._malloc(bytes.length + 1);
+  if (!ptr) {
+    throw new Error('Failed to allocate memory for query');
   }
+  wasmModule.HEAPU8.set(bytes, ptr);
+  wasmModule.HEAPU8[ptr + bytes.length] = 0;
+  return ptr;
 }
 
 function ptrToString(ptr: number): string {

versions/13/test/input-encoding.test.js

@@ -0,0 +1,45 @@
+const { describe, it, before } = require('node:test');
+const assert = require('node:assert/strict');
+const query = require('../');
+
+const LONE_HIGH = '\uD800';
+const LONE_LOW = '\uDC00';
+
+function firstLiteral(result) {
+  const aConst = result.stmts[0].stmt.SelectStmt.targetList[0].ResTarget.val.A_Const;
+  return aConst.sval ? aConst.sval.sval : aConst.val.String.str;
+}
+
+describe('Query input encoding', () => {
+  before(async () => {
+    await query.parse('SELECT 1');
+  });
+
+  for (const count of [1, 2, 4, 8, 16, 64]) {
+    it(`parses every statement after ${count} unpaired high surrogates`, () => {
+      const result = query.parseSync(`SELECT '${LONE_HIGH.repeat(count)}'; SELECT 2`);
+      assert.equal(result.stmts.length, 2);
+    });
+  }
+
+  it('parses every statement after unpaired low surrogates', async () => {
+    const result = await query.parse(`SELECT '${LONE_LOW.repeat(8)}'; SELECT 2`);
+    assert.equal(result.stmts.length, 2);
+  });
+
+  it('reads an unpaired surrogate as U+FFFD, as a Postgres driver sends it', () => {
+    const result = query.parseSync(`SELECT '${LONE_HIGH}x${LONE_LOW}'`);
+    assert.equal(firstLiteral(result), '\uFFFDx\uFFFD');
+  });
+
+  it('keeps valid surrogate pairs intact', () => {
+    const result = query.parseSync("SELECT '\u{1F600}'; SELECT 2");
+    assert.equal(result.stmts.length, 2);
+    assert.equal(firstLiteral(result), '\u{1F600}');
+  });
+
+  it('rejects an embedded NUL instead of dropping the rest of the query', async () => {
+    assert.throws(() => query.parseSync('SELECT 1\u0000; SELECT 2'), /NUL/);
+    await assert.rejects(query.parse('SELECT 1\u0000; SELECT 2'), /NUL/);
+  });
+});

versions/14/package.json

@@ -28,7 +28,7 @@
     "wasm:rebuild": "pnpm wasm:make rebuild",
     "wasm:clean": "pnpm wasm:make clean",
     "wasm:clean-cache": "pnpm wasm:make clean-cache",
-    "test": "node --test test/parsing.test.js test/errors.test.js"
+    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js"
   },
   "author": "Constructive <developers@constructive.io>",
   "license": "MIT",

versions/14/src/index.ts

@@ -152,20 +152,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
   }) as T;
 }
 
+const utf8Encoder = new TextEncoder();
+
 function stringToPtr(str: string): number {
   ensureLoaded();
   if (typeof str !== 'string') {
     throw new TypeError(`Expected a string, got ${typeof str}`);
   }
-  const len = wasmModule.lengthBytesUTF8(str) + 1;
-  const ptr = wasmModule._malloc(len);
-  try {
-    wasmModule.stringToUTF8(str, ptr, len);
-    return ptr;
-  } catch (error) {
-    wasmModule._free(ptr);
-    throw error;
+  // libpg_query takes a NUL-terminated C string: an embedded NUL would end the
+  // parse early and silently drop everything after it.
+  if (str.includes('\0')) {
+    throw new Error('Query must not contain a NUL (U+0000) character');
+  }
+  // Size the buffer from the exact bytes written. TextEncoder writes each
+  // unpaired surrogate as U+FFFD — the same bytes a Postgres driver sends.
+  const bytes = utf8Encoder.encode(str);
+  const ptr = wasmModule._malloc(bytes.length + 1);
+  if (!ptr) {
+    throw new Error('Failed to allocate memory for query');
   }
+  wasmModule.HEAPU8.set(bytes, ptr);
+  wasmModule.HEAPU8[ptr + bytes.length] = 0;
+  return ptr;
 }
 
 function ptrToString(ptr: number): string {

versions/14/test/input-encoding.test.js

@@ -0,0 +1,45 @@
+const { describe, it, before } = require('node:test');
+const assert = require('node:assert/strict');
+const query = require('../');
+
+const LONE_HIGH = '\uD800';
+const LONE_LOW = '\uDC00';
+
+function firstLiteral(result) {
+  const aConst = result.stmts[0].stmt.SelectStmt.targetList[0].ResTarget.val.A_Const;
+  return aConst.sval ? aConst.sval.sval : aConst.val.String.str;
+}
+
+describe('Query input encoding', () => {
+  before(async () => {
+    await query.parse('SELECT 1');
+  });
+
+  for (const count of [1, 2, 4, 8, 16, 64]) {
+    it(`parses every statement after ${count} unpaired high surrogates`, () => {
+      const result = query.parseSync(`SELECT '${LONE_HIGH.repeat(count)}'; SELECT 2`);
+      assert.equal(result.stmts.length, 2);
+    });
+  }
+
+  it('parses every statement after unpaired low surrogates', async () => {
+    const result = await query.parse(`SELECT '${LONE_LOW.repeat(8)}'; SELECT 2`);
+    assert.equal(result.stmts.length, 2);
+  });
+
+  it('reads an unpaired surrogate as U+FFFD, as a Postgres driver sends it', () => {
+    const result = query.parseSync(`SELECT '${LONE_HIGH}x${LONE_LOW}'`);
+    assert.equal(firstLiteral(result), '\uFFFDx\uFFFD');
+  });
+
+  it('keeps valid surrogate pairs intact', () => {
+    const result = query.parseSync("SELECT '\u{1F600}'; SELECT 2");
+    assert.equal(result.stmts.length, 2);
+    assert.equal(firstLiteral(result), '\u{1F600}');
+  });
+
+  it('rejects an embedded NUL instead of dropping the rest of the query', async () => {
+    assert.throws(() => query.parseSync('SELECT 1\u0000; SELECT 2'), /NUL/);
+    await assert.rejects(query.parse('SELECT 1\u0000; SELECT 2'), /NUL/);
+  });
+});

versions/15/package.json

@@ -28,7 +28,7 @@
     "wasm:rebuild": "pnpm wasm:make rebuild",
     "wasm:clean": "pnpm wasm:make clean",
     "wasm:clean-cache": "pnpm wasm:make clean-cache",
-    "test": "node --test test/parsing.test.js test/errors.test.js"
+    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js"
   },
   "author": "Constructive <developers@constructive.io>",
   "license": "MIT",

versions/15/src/index.ts

@@ -152,20 +152,28 @@ function awaitInit<T extends (...args: any[]) => any>(fn: T): T {
   }) as T;
 }
 
+const utf8Encoder = new TextEncoder();
+
 function stringToPtr(str: string): number {
   ensureLoaded();
   if (typeof str !== 'string') {
     throw new TypeError(`Expected a string, got ${typeof str}`);
   }
-  const len = wasmModule.lengthBytesUTF8(str) + 1;
-  const ptr = wasmModule._malloc(len);
-  try {
-    wasmModule.stringToUTF8(str, ptr, len);
-    return ptr;
-  } catch (error) {
-    wasmModule._free(ptr);
-    throw error;
+  // libpg_query takes a NUL-terminated C string: an embedded NUL would end the
+  // parse early and silently drop everything after it.
+  if (str.includes('\0')) {
+    throw new Error('Query must not contain a NUL (U+0000) character');
+  }
+  // Size the buffer from the exact bytes written. TextEncoder writes each
+  // unpaired surrogate as U+FFFD — the same bytes a Postgres driver sends.
+  const bytes = utf8Encoder.encode(str);
+  const ptr = wasmModule._malloc(bytes.length + 1);
+  if (!ptr) {
+    throw new Error('Failed to allocate memory for query');
   }
+  wasmModule.HEAPU8.set(bytes, ptr);
+  wasmModule.HEAPU8[ptr + bytes.length] = 0;
+  return ptr;
 }
 
 function ptrToString(ptr: number): string {

versions/15/test/input-encoding.test.js

@@ -0,0 +1,45 @@
+const { describe, it, before } = require('node:test');
+const assert = require('node:assert/strict');
+const query = require('../');
+
+const LONE_HIGH = '\uD800';
+const LONE_LOW = '\uDC00';
+
+function firstLiteral(result) {
+  const aConst = result.stmts[0].stmt.SelectStmt.targetList[0].ResTarget.val.A_Const;
+  return aConst.sval ? aConst.sval.sval : aConst.val.String.str;
+}
+
+describe('Query input encoding', () => {
+  before(async () => {
+    await query.parse('SELECT 1');
+  });
+
+  for (const count of [1, 2, 4, 8, 16, 64]) {
+    it(`parses every statement after ${count} unpaired high surrogates`, () => {
+      const result = query.parseSync(`SELECT '${LONE_HIGH.repeat(count)}'; SELECT 2`);
+      assert.equal(result.stmts.length, 2);
+    });
+  }
+
+  it('parses every statement after unpaired low surrogates', async () => {
+    const result = await query.parse(`SELECT '${LONE_LOW.repeat(8)}'; SELECT 2`);
+    assert.equal(result.stmts.length, 2);
+  });
+
+  it('reads an unpaired surrogate as U+FFFD, as a Postgres driver sends it', () => {
+    const result = query.parseSync(`SELECT '${LONE_HIGH}x${LONE_LOW}'`);
+    assert.equal(firstLiteral(result), '\uFFFDx\uFFFD');
+  });
+
+  it('keeps valid surrogate pairs intact', () => {
+    const result = query.parseSync("SELECT '\u{1F600}'; SELECT 2");
+    assert.equal(result.stmts.length, 2);
+    assert.equal(firstLiteral(result), '\u{1F600}');
+  });
+
+  it('rejects an embedded NUL instead of dropping the rest of the query', async () => {
+    assert.throws(() => query.parseSync('SELECT 1\u0000; SELECT 2'), /NUL/);
+    await assert.rejects(query.parse('SELECT 1\u0000; SELECT 2'), /NUL/);
+  });
+});

versions/16/package.json

@@ -28,7 +28,7 @@
     "wasm:rebuild": "pnpm wasm:make rebuild",
     "wasm:clean": "pnpm wasm:make clean",
     "wasm:clean-cache": "pnpm wasm:make clean-cache",
-    "test": "node --test test/parsing.test.js test/errors.test.js"
+    "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js"
   },
   "author": "Constructive <developers@constructive.io>",
   "license": "MIT",