From 6942283da2bf04ce53e0eededf922765c1653c7c Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Tue, 21 Jul 2026 08:55:47 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(full):=20version=20@libpg-query/parser?= =?UTF-8?q?=20per=20PG=20major=20=E2=80=94=20full/17=20(pg17)=20+=20full/1?= =?UTF-8?q?8=20(pg18)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-wasm-no-docker.yaml | 6 +- .github/workflows/build-wasm.yml | 21 +- .github/workflows/ci.yml | 6 +- PUBLISH.md | 21 +- README.md | 4 +- full/{ => 17}/CHANGELOG.md | 0 full/{ => 17}/Makefile | 0 full/{ => 17}/README.md | 2 +- full/{ => 17}/SCAN.md | 0 full/{ => 17}/libpg_query.md | 0 full/{ => 17}/libpg_query/protobuf/.gitkeep | 0 full/{ => 17}/package.json | 9 +- full/{ => 17}/scripts/build.js | 0 full/{ => 17}/src/index.ts | 0 full/{ => 17}/src/libpg-query.d.ts | 0 full/{ => 17}/src/proto.d.ts | 0 full/{ => 17}/src/wasm_wrapper.c | 0 full/{ => 17}/test/errors.test.js | 0 full/{ => 17}/test/fingerprint.test.js | 0 full/{ => 17}/test/normalize.test.js | 0 full/{ => 17}/test/parsing.test.js | 0 full/{ => 17}/test/plpgsql.test.js | 0 full/{ => 17}/test/scan.test.js | 0 full/{ => 17}/tsconfig.esm.json | 0 full/{ => 17}/tsconfig.json | 0 full/18/CHANGELOG.md | 44 ++ full/18/Makefile | 96 ++++ full/18/README.md | 358 +++++++++++++ full/18/SCAN.md | 439 +++++++++++++++ full/18/libpg_query.md | 558 ++++++++++++++++++++ full/18/libpg_query/protobuf/.gitkeep | 0 full/18/package.json | 55 ++ full/18/scripts/build.js | 38 ++ full/18/src/index.ts | 484 +++++++++++++++++ full/18/src/libpg-query.d.ts | 17 + full/18/src/proto.d.ts | 20 + full/18/src/wasm_wrapper.c | 395 ++++++++++++++ full/18/test/errors.test.js | 325 ++++++++++++ full/18/test/fingerprint.test.js | 67 +++ full/18/test/normalize.test.js | 69 +++ full/18/test/parsing.test.js | 84 +++ full/18/test/pg18.test.js | 69 +++ full/18/test/plpgsql.test.js | 75 +++ full/18/test/scan.test.js | 279 ++++++++++ full/18/tsconfig.esm.json | 9 + full/18/tsconfig.json | 18 + pnpm-lock.yaml | 12 +- pnpm-workspace.yaml | 3 +- scripts/analyze-sizes.js | 3 +- scripts/publish-versions.js | 62 ++- 50 files changed, 3596 insertions(+), 52 deletions(-) rename full/{ => 17}/CHANGELOG.md (100%) rename full/{ => 17}/Makefile (100%) rename full/{ => 17}/README.md (99%) rename full/{ => 17}/SCAN.md (100%) rename full/{ => 17}/libpg_query.md (100%) rename full/{ => 17}/libpg_query/protobuf/.gitkeep (100%) rename full/{ => 17}/package.json (84%) rename full/{ => 17}/scripts/build.js (100%) rename full/{ => 17}/src/index.ts (100%) rename full/{ => 17}/src/libpg-query.d.ts (100%) rename full/{ => 17}/src/proto.d.ts (100%) rename full/{ => 17}/src/wasm_wrapper.c (100%) rename full/{ => 17}/test/errors.test.js (100%) rename full/{ => 17}/test/fingerprint.test.js (100%) rename full/{ => 17}/test/normalize.test.js (100%) rename full/{ => 17}/test/parsing.test.js (100%) rename full/{ => 17}/test/plpgsql.test.js (100%) rename full/{ => 17}/test/scan.test.js (100%) rename full/{ => 17}/tsconfig.esm.json (100%) rename full/{ => 17}/tsconfig.json (100%) create mode 100644 full/18/CHANGELOG.md create mode 100644 full/18/Makefile create mode 100644 full/18/README.md create mode 100644 full/18/SCAN.md create mode 100644 full/18/libpg_query.md create mode 100644 full/18/libpg_query/protobuf/.gitkeep create mode 100644 full/18/package.json create mode 100644 full/18/scripts/build.js create mode 100644 full/18/src/index.ts create mode 100644 full/18/src/libpg-query.d.ts create mode 100644 full/18/src/proto.d.ts create mode 100644 full/18/src/wasm_wrapper.c create mode 100644 full/18/test/errors.test.js create mode 100644 full/18/test/fingerprint.test.js create mode 100644 full/18/test/normalize.test.js create mode 100644 full/18/test/parsing.test.js create mode 100644 full/18/test/pg18.test.js create mode 100644 full/18/test/plpgsql.test.js create mode 100644 full/18/test/scan.test.js create mode 100644 full/18/tsconfig.esm.json create mode 100644 full/18/tsconfig.json diff --git a/.github/workflows/build-wasm-no-docker.yaml b/.github/workflows/build-wasm-no-docker.yaml index 71d20294..0e6b1c3f 100644 --- a/.github/workflows/build-wasm-no-docker.yaml +++ b/.github/workflows/build-wasm-no-docker.yaml @@ -42,15 +42,15 @@ jobs: ./emsdk install 3.1.59 ./emsdk activate 3.1.59 source ./emsdk_env.sh - working-directory: full + working-directory: full/18 - name: Build with Emscripten πŸ— run: | source ./emsdk/emsdk_env.sh emmake make emmake make build - working-directory: full + working-directory: full/18 - name: Archive production artifacts πŸ› uses: actions/upload-artifact@v4 with: name: wasm-artifacts - path: full/wasm + path: full/18/wasm diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index d5200325..962873e1 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -36,13 +36,24 @@ jobs: - name: Install Dependencies 🧢 run: pnpm install - - name: Build WASM πŸ— + - name: Build WASM (PG17) πŸ— run: pnpm run build - working-directory: full + working-directory: full/17 - - name: Archive production artifacts πŸ› + - name: Build WASM (PG18) πŸ— + run: pnpm run build + working-directory: full/18 + + - name: Archive PG17 artifacts πŸ› + uses: actions/upload-artifact@v4 + with: + name: wasm-artifacts-pg17 + path: full/17/wasm/ + retention-days: 7 + + - name: Archive PG18 artifacts πŸ› uses: actions/upload-artifact@v4 with: - name: wasm-artifacts - path: full/wasm/ + name: wasm-artifacts-pg18 + path: full/18/wasm/ retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dac525b4..eacc9905 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,8 @@ jobs: strategy: matrix: package: - - { name: 'full', path: 'full', version: '18' } + - { name: 'full17', path: 'full/17', version: '17' } + - { name: 'full18', path: 'full/18', version: '18' } - { name: 'v13', path: 'versions/13', version: '13' } - { name: 'v14', path: 'versions/14', version: '14' } - { name: 'v15', path: 'versions/15', version: '15' } @@ -78,7 +79,8 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] package: - - { name: 'full', path: 'full', version: '18' } + - { name: 'full17', path: 'full/17', version: '17' } + - { name: 'full18', path: 'full/18', version: '18' } - { name: 'v13', path: 'versions/13', version: '13' } - { name: 'v14', path: 'versions/14', version: '14' } - { name: 'v15', path: 'versions/15', version: '15' } diff --git a/PUBLISH.md b/PUBLISH.md index 956384ae..5faa7de6 100644 --- a/PUBLISH.md +++ b/PUBLISH.md @@ -10,7 +10,7 @@ pnpm run publish:versions This interactive script will: - Check for uncommitted changes (will error if any exist) - Let you select which versions to publish (or all) -- Also includes the full package (@libpg-query/parser) +- Also includes the full package versions (@libpg-query/parser, full/17 and full/18) - Ask for version bump type (patch or minor only) - Ask if you want to skip the build step (useful if already built) - Always run tests (even if build is skipped) @@ -165,28 +165,35 @@ npm install libpg-query # Latest/default version ## Full Package (@libpg-query/parser) -### Quick Publish +The full package is versioned per PostgreSQL major version, mirroring `versions/`: +- `full/17` β€” built from `17-constructive`, published as `@libpg-query/parser` with tag `pg17` +- `full/18` β€” built from `18-constructive`, published as `@libpg-query/parser` with tag `pg18` + +### Quick Publish (per version) ```bash -cd full +cd full/17 # or full/18 pnpm version patch git add . && git commit -m "release: bump @libpg-query/parser version" pnpm build pnpm test -pnpm publish --tag pg17 +pnpm run publish:pkg ``` +`publish:pkg` uses `x-publish.publishName` (`@libpg-query/parser`) and `x-publish.distTag` +(`pg17` / `pg18`) from each package.json, temporarily renaming the package during publish. + ### Promote to latest (optional) ```bash npm dist-tag add @libpg-query/parser@pg17 latest ``` ### What it does -- Publishes `@libpg-query/parser` with tag `pg17` -- Currently based on PostgreSQL 17 -- Includes full parser with all features +- Publishes `@libpg-query/parser` with tag `pg17` (from `full/17`) or `pg18` (from `full/18`) +- Includes full parser with all features (parse, parsePlPgSQL, scan, fingerprint, normalize + sync variants) ### Install published package ```bash +npm install @libpg-query/parser@pg18 # PostgreSQL 18 specific npm install @libpg-query/parser@pg17 # PostgreSQL 17 specific npm install @libpg-query/parser # Latest version ``` diff --git a/README.md b/README.md index 21f928f4..5fed1572 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ This repository contains multiple packages to support different PostgreSQL versi | **[@pgsql/parser](https://github.com/constructive-io/libpg-query-node/tree/main/parser)** | Multi-version parser (runtime selection) | 15, 16, 17, 18 | [`@pgsql/parser`](https://www.npmjs.com/package/@pgsql/parser) | | **[@pgsql/types](https://github.com/constructive-io/libpg-query-node/tree/main/types)** | TypeScript type definitions | 13, 14, 15, 16, 17, 18 | [`@pgsql/types`](https://www.npmjs.com/package/@pgsql/types) | | **[@pgsql/enums](https://github.com/constructive-io/libpg-query-node/tree/main/enums)** | TypeScript enum definitions | 13, 14, 15, 16, 17, 18 | [`@pgsql/enums`](https://www.npmjs.com/package/@pgsql/enums) | -| **[@libpg-query/parser](https://github.com/constructive-io/libpg-query-node/tree/main/full)** | Full parser with all features | 17 only | [`@libpg-query/parser`](https://www.npmjs.com/package/@libpg-query/parser) | +| **[@libpg-query/parser](https://github.com/constructive-io/libpg-query-node/tree/main/full)** | Full parser with all features | 17, 18 | [`@libpg-query/parser`](https://www.npmjs.com/package/@libpg-query/parser) | ### Version Tags @@ -102,7 +102,7 @@ npm install @pgsql/enums - **Just need to parse SQL?** β†’ Use `libpg-query` (lightweight, all PG versions) - **Need multiple versions at runtime?** β†’ Use `@pgsql/parser` (dynamic version selection) - **Need TypeScript types?** β†’ Add `@pgsql/types` and/or `@pgsql/enums` -- **Need fingerprint, normalize, scan, or PL/pgSQL parsing?** β†’ Use `@libpg-query/parser` (PG 18) +- **Need fingerprint, normalize, scan, or PL/pgSQL parsing?** β†’ Use `@libpg-query/parser` (PG 17 & 18 via `pg17`/`pg18` dist-tags) ## API Documentation diff --git a/full/CHANGELOG.md b/full/17/CHANGELOG.md similarity index 100% rename from full/CHANGELOG.md rename to full/17/CHANGELOG.md diff --git a/full/Makefile b/full/17/Makefile similarity index 100% rename from full/Makefile rename to full/17/Makefile diff --git a/full/README.md b/full/17/README.md similarity index 99% rename from full/README.md rename to full/17/README.md index 2091d08a..040a55aa 100644 --- a/full/README.md +++ b/full/17/README.md @@ -8,7 +8,7 @@ -
+
diff --git a/full/SCAN.md b/full/17/SCAN.md similarity index 100% rename from full/SCAN.md rename to full/17/SCAN.md diff --git a/full/libpg_query.md b/full/17/libpg_query.md similarity index 100% rename from full/libpg_query.md rename to full/17/libpg_query.md diff --git a/full/libpg_query/protobuf/.gitkeep b/full/17/libpg_query/protobuf/.gitkeep similarity index 100% rename from full/libpg_query/protobuf/.gitkeep rename to full/17/libpg_query/protobuf/.gitkeep diff --git a/full/package.json b/full/17/package.json similarity index 84% rename from full/package.json rename to full/17/package.json index 91baefef..b7444a64 100644 --- a/full/package.json +++ b/full/17/package.json @@ -1,5 +1,5 @@ { - "name": "@libpg-query/parser", + "name": "@libpg-query/full17", "version": "17.8.0", "description": "The real PostgreSQL query parser", "homepage": "https://github.com/constructive-io/libpg-query-node", @@ -9,6 +9,12 @@ "publishConfig": { "access": "public" }, + "x-publish": { + "publishName": "@libpg-query/parser", + "pgVersion": "17", + "distTag": "pg17", + "libpgQueryTag": "17-constructive" + }, "files": [ "wasm/*" ], @@ -16,6 +22,7 @@ "clean": "pnpm wasm:clean && rimraf cjs esm", "build:js": "node scripts/build.js", "build": "pnpm clean; pnpm wasm:build; pnpm build:js", + "publish:pkg": "node ../../scripts/publish-single-version.js", "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", "wasm:build": "pnpm wasm:make build", "wasm:rebuild": "pnpm wasm:make rebuild", diff --git a/full/scripts/build.js b/full/17/scripts/build.js similarity index 100% rename from full/scripts/build.js rename to full/17/scripts/build.js diff --git a/full/src/index.ts b/full/17/src/index.ts similarity index 100% rename from full/src/index.ts rename to full/17/src/index.ts diff --git a/full/src/libpg-query.d.ts b/full/17/src/libpg-query.d.ts similarity index 100% rename from full/src/libpg-query.d.ts rename to full/17/src/libpg-query.d.ts diff --git a/full/src/proto.d.ts b/full/17/src/proto.d.ts similarity index 100% rename from full/src/proto.d.ts rename to full/17/src/proto.d.ts diff --git a/full/src/wasm_wrapper.c b/full/17/src/wasm_wrapper.c similarity index 100% rename from full/src/wasm_wrapper.c rename to full/17/src/wasm_wrapper.c diff --git a/full/test/errors.test.js b/full/17/test/errors.test.js similarity index 100% rename from full/test/errors.test.js rename to full/17/test/errors.test.js diff --git a/full/test/fingerprint.test.js b/full/17/test/fingerprint.test.js similarity index 100% rename from full/test/fingerprint.test.js rename to full/17/test/fingerprint.test.js diff --git a/full/test/normalize.test.js b/full/17/test/normalize.test.js similarity index 100% rename from full/test/normalize.test.js rename to full/17/test/normalize.test.js diff --git a/full/test/parsing.test.js b/full/17/test/parsing.test.js similarity index 100% rename from full/test/parsing.test.js rename to full/17/test/parsing.test.js diff --git a/full/test/plpgsql.test.js b/full/17/test/plpgsql.test.js similarity index 100% rename from full/test/plpgsql.test.js rename to full/17/test/plpgsql.test.js diff --git a/full/test/scan.test.js b/full/17/test/scan.test.js similarity index 100% rename from full/test/scan.test.js rename to full/17/test/scan.test.js diff --git a/full/tsconfig.esm.json b/full/17/tsconfig.esm.json similarity index 100% rename from full/tsconfig.esm.json rename to full/17/tsconfig.esm.json diff --git a/full/tsconfig.json b/full/17/tsconfig.json similarity index 100% rename from full/tsconfig.json rename to full/17/tsconfig.json diff --git a/full/18/CHANGELOG.md b/full/18/CHANGELOG.md new file mode 100644 index 00000000..d40399b2 --- /dev/null +++ b/full/18/CHANGELOG.md @@ -0,0 +1,44 @@ +# 18.0.0 +* PostgreSQL 18 full build, compiled from the `18-constructive` branch of constructive-io/libpg_query +* Full API: parse, parsePlPgSQL, scan, fingerprint, normalize + sync variants +* Uses @pgsql/types ^18.0.0 +* Published as @libpg-query/parser under the `pg18` dist-tag + +# 17.2.1 +* Add normalize() and normalizeSync() functions for SQL normalization +* Add fingerprint() and fingerprintSync() functions for query fingerprinting +* Add isReady() function to check WASM module initialization status +* Improve memory management and error handling in WASM wrapper +* Remove unused dependencies (lodash, deasync, @launchql/protobufjs) +* Split test suite into separate files for better organization +* Update documentation with comprehensive function examples + +# 1.2.1 +* Rename package + +# 1.2.0 +* Add TS typings + +# 1.1.2 (07/23/19) +* Fix build script + +# 1.1.0 (Republish, 07/23/19) +* Rewrite to use Node-Addons-API (C++ wrapper for the stable N-API) +* Support PL/PGSql Parsing +* Support async parsing + +## 0.0.5 (November 19, 2015) +* Update libpg_query to include latest A_CONST fixes + +## 0.0.4 (November 16, 2015) +* Update libpg_query to include fix for empty string constants +* Warning fixed in 0.0.3 is back for now until libpg_query has support for custom CFLAGS + +## 0.0.3 (November 14, 2015) +* Update OSX static library to include `-mmacosx-version-min=10.5` to fix warnings on installation + +## 0.0.2 (November 14, 2015) +* Rename repo to pg-query-native + +## 0.0.1 (November 14, 2015) +* First version diff --git a/full/18/Makefile b/full/18/Makefile new file mode 100644 index 00000000..c9685261 --- /dev/null +++ b/full/18/Makefile @@ -0,0 +1,96 @@ +WASM_OUT_DIR := wasm +WASM_OUT_NAME := libpg-query +WASM_MODULE_NAME := PgQueryModule +# LIBPG_QUERY_REPO := https://github.com/pganalyze/libpg_query.git +# LIBPG_QUERY_TAG := 18.0.0 +LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git +LIBPG_QUERY_TAG := 18-constructive + +CACHE_DIR := .cache + +OS ?= $(shell uname -s) +ARCH ?= $(shell uname -m) + +ifdef EMSCRIPTEN +PLATFORM := emscripten +else ifeq ($(OS),Darwin) +PLATFORM := darwin +else ifeq ($(OS),Linux) +PLATFORM := linux +else +$(error Unsupported platform: $(OS)) +endif + +ifdef EMSCRIPTEN +ARCH := wasm +endif + +PLATFORM_ARCH := $(PLATFORM)-$(ARCH) +SRC_FILES := src/wasm_wrapper.c +LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_TAG) +LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a +LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h +CXXFLAGS := -O3 + +ifdef EMSCRIPTEN +OUT_FILES := $(foreach EXT,.js .wasm,$(WASM_OUT_DIR)/$(WASM_OUT_NAME)$(EXT)) +else +$(error Native builds are no longer supported. Use EMSCRIPTEN=1 for WASM builds only.) +endif + +# Clone libpg_query source (lives in CACHE_DIR) +$(LIBPG_QUERY_DIR): + mkdir -p $(CACHE_DIR) + git clone -b $(LIBPG_QUERY_TAG) --single-branch $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) + +$(LIBPG_QUERY_HEADER): $(LIBPG_QUERY_DIR) + +# Build libpg_query +$(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR) + cd $(LIBPG_QUERY_DIR); $(MAKE) build + +# Build libpg-query-node WASM module +$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) +ifdef EMSCRIPTEN + mkdir -p $(WASM_OUT_DIR) + $(CC) \ + -v \ + $(CXXFLAGS) \ + -I$(LIBPG_QUERY_DIR) \ + -I$(LIBPG_QUERY_DIR)/vendor \ + -L$(LIBPG_QUERY_DIR) \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string','_wasm_parse_query_raw','_wasm_free_parse_result']" \ + -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','UTF8ToString','getValue','HEAPU8','HEAPU32']" \ + -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ + -sENVIRONMENT="web,node,worker" \ + -sASSERTIONS=0 \ + -sSINGLE_FILE=0 \ + -sMODULARIZE=1 \ + -sEXPORT_ES6=0 \ + -sALLOW_MEMORY_GROWTH=1 \ + -sINITIAL_MEMORY=134217728 \ + -sMAXIMUM_MEMORY=1073741824 \ + -sSTACK_SIZE=33554432 \ + -lpg_query \ + -o $@ \ + $(SRC_FILES) +else +$(error Native builds are no longer supported. Use EMSCRIPTEN=1 for WASM builds only.) +endif + +# Commands +build: $(OUT_FILES) + +build-cache: $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) + +rebuild: clean build + +rebuild-cache: clean-cache build-cache + +clean: + -@ rm -r $(OUT_FILES) > /dev/null 2>&1 + +clean-cache: + -@ rm -rf $(LIBPG_QUERY_DIR) + +.PHONY: build build-cache rebuild rebuild-cache clean clean-cache diff --git a/full/18/README.md b/full/18/README.md new file mode 100644 index 00000000..42d174f5 --- /dev/null +++ b/full/18/README.md @@ -0,0 +1,358 @@ +# @libpg-query/parser + +

+ constructive.io +

+ +

+ + + +
+ + + + +

+ +# The Real PostgreSQL Parser for JavaScript + +### Bring the power of PostgreSQL’s native parser to your JavaScript projects β€” no native builds, no platform headaches. + +This is the official PostgreSQL parser, compiled to WebAssembly (WASM) for seamless, cross-platform compatibility. Use it in Node.js or the browser, on Linux, Windows, or anywhere JavaScript runs. + +Built to power [pgsql-parser](https://github.com/constructive-io/pgsql-parser), this library delivers full fidelity with the Postgres C codebase β€” no rewrites, no shortcuts. + +### Features + +* πŸ”§ **Powered by PostgreSQL** – Uses the official Postgres C parser compiled to WebAssembly +* πŸ–₯️ **Cross-Platform** – Runs smoothly on macOS, Linux, and Windows +* 🌐 **Node.js & Browser Support** – Consistent behavior in any JS environment +* πŸ“¦ **No Native Builds Required** – No compilation, no system-specific dependencies +* 🧠 **Spec-Accurate Parsing** – Produces faithful, standards-compliant ASTs +* πŸš€ **Production-Grade** – Millions of downloads and trusted by countless projects and top teams + +## πŸš€ For Round-trip Codegen + +> 🎯 **Want to parse + deparse (full round trip)?** +> We highly recommend using [`pgsql-parser`](https://github.com/constructive-io/pgsql-parser) which leverages a pure TypeScript deparser that has been battle-tested against 23,000+ SQL statements and is built on top of libpg-query. + +## Installation + +```sh +npm install @libpg-query/parser +``` + +## Usage + +### `parse(query: string): Promise` + +Parses the SQL and returns a Promise for the parse tree. May reject with a parse error. + +```typescript +import { parse } from '@libpg-query/parser'; + +const result = await parse('SELECT * FROM users WHERE active = true'); +// Returns: ParseResult - parsed query object +``` + +### `parseSync(query: string): ParseResult` + +Synchronous version that returns the parse tree directly. May throw a parse error. + +```typescript +import { parseSync } from '@libpg-query/parser'; + +const result = parseSync('SELECT * FROM users WHERE active = true'); +// Returns: ParseResult - parsed query object +``` + +### `parsePlPgSQL(funcsSql: string): Promise` + +Parses the contents of a PL/pgSQL function from a `CREATE FUNCTION` declaration. Returns a Promise for the parse tree. + +```typescript +import { parsePlPgSQL } from '@libpg-query/parser'; + +const functionSql = ` +CREATE FUNCTION get_user_count() RETURNS integer AS $$ +BEGIN + RETURN (SELECT COUNT(*) FROM users); +END; +$$ LANGUAGE plpgsql; +`; + +const result = await parsePlPgSQL(functionSql); +``` + +### `parsePlPgSQLSync(funcsSql: string): ParseResult` + +Synchronous version of PL/pgSQL parsing. + +```typescript +import { parsePlPgSQLSync } from '@libpg-query/parser'; + +const result = parsePlPgSQLSync(functionSql); +``` + +### `fingerprint(sql: string): Promise` + +Generates a unique fingerprint for a SQL query that can be used for query identification and caching. Returns a Promise for a 16-character fingerprint string. + +```typescript +import { fingerprint } from '@libpg-query/parser'; + +const fp = await fingerprint('SELECT * FROM users WHERE active = $1'); +// Returns: string - unique 16-character fingerprint (e.g., "50fde20626009aba") +``` + +### `fingerprintSync(sql: string): string` + +Synchronous version that generates a unique fingerprint for a SQL query directly. + +```typescript +import { fingerprintSync } from '@libpg-query/parser'; + +const fp = fingerprintSync('SELECT * FROM users WHERE active = $1'); +// Returns: string - unique 16-character fingerprint +``` + +### `normalize(sql: string): Promise` + +Normalizes a SQL query by removing comments, standardizing whitespace, and converting to a canonical form. Returns a Promise for the normalized SQL string. + +```typescript +import { normalize } from '@libpg-query/parser'; + +const normalized = await normalize('SELECT * FROM users WHERE active = true'); +// Returns: string - normalized SQL query +``` + +### `normalizeSync(sql: string): string` + +Synchronous version that normalizes a SQL query directly. + +```typescript +import { normalizeSync } from '@libpg-query/parser'; + +const normalized = normalizeSync('SELECT * FROM users WHERE active = true'); +// Returns: string - normalized SQL query +``` + +### `scan(sql: string): Promise` + +Scans (tokenizes) a SQL query and returns detailed information about each token. Returns a Promise for a ScanResult containing all tokens with their positions, types, and classifications. + +```typescript +import { scan } from '@libpg-query/parser'; + +const result = await scan('SELECT * FROM users WHERE id = $1'); +// Returns: ScanResult - detailed tokenization information +console.log(result.tokens[0]); // { start: 0, end: 6, text: "SELECT", tokenType: 651, tokenName: "UNKNOWN", keywordKind: 4, keywordName: "RESERVED_KEYWORD" } +``` + +### `scanSync(sql: string): ScanResult` + +Synchronous version that scans (tokenizes) a SQL query directly. + +```typescript +import { scanSync } from '@libpg-query/parser'; + +const result = scanSync('SELECT * FROM users WHERE id = $1'); +// Returns: ScanResult - detailed tokenization information +``` + +### Initialization + +The library provides both async and sync methods. Async methods handle initialization automatically, while sync methods require explicit initialization. + +#### Async Methods (Recommended) + +Async methods handle initialization automatically and are always safe to use: + +```typescript +import { parse, scan } from '@libpg-query/parser'; + +// These handle initialization automatically +const result = await parse('SELECT * FROM users'); +const tokens = await scan('SELECT * FROM users'); +``` + +#### Sync Methods + +Sync methods require explicit initialization using `loadModule()`: + +```typescript +import { loadModule, parseSync, scanSync } from '@libpg-query/parser'; + +// Initialize first +await loadModule(); + +// Now safe to use sync methods +const result = parseSync('SELECT * FROM users'); +const tokens = scanSync('SELECT * FROM users'); +``` + +### `loadModule(): Promise` + +Explicitly initializes the WASM module. Required before using any sync methods. + +```typescript +import { loadModule, parseSync, scanSync } from '@libpg-query/parser'; + +// Initialize before using sync methods +await loadModule(); +const result = parseSync('SELECT * FROM users'); +const tokens = scanSync('SELECT * FROM users'); +``` + +Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. + +### Type Definitions + +```typescript +interface ParseResult { + version: number; + stmts: Statement[]; +} + +interface Statement { + stmt_type: string; + stmt_len: number; + stmt_location: number; + query: string; +} + +interface ScanResult { + version: number; + tokens: ScanToken[]; +} + +interface ScanToken { + start: number; // Starting position in the SQL string + end: number; // Ending position in the SQL string + text: string; // The actual token text + tokenType: number; // Numeric token type identifier + tokenName: string; // Human-readable token type name + keywordKind: number; // Numeric keyword classification + keywordName: string; // Human-readable keyword classification +} +``` + +**Note:** The return value is an array, as multiple queries may be provided in a single string (semicolon-delimited, as PostgreSQL expects). + +## Build Instructions + +This package uses a **WASM-only build system** for true cross-platform compatibility without native compilation dependencies. + +### Prerequisites + +- Node.js (version 16 or higher recommended) +- [pnpm](https://pnpm.io/) (v8+ recommended) + +### Building WASM Artifacts + +1. **Install dependencies:** + ```bash + pnpm install + ``` + +2. **Build WASM artifacts:** + ```bash + pnpm run build + ``` + +3. **Clean WASM build (if needed):** + ```bash + pnpm run clean + ``` + +4. **Rebuild WASM artifacts from scratch:** + ```bash + pnpm run clean && pnpm run build + ``` + +### Build Process Details + +The WASM build process: +- Uses Emscripten SDK for compilation +- Compiles C wrapper code to WebAssembly +- Generates `wasm/libpg-query.js` and `wasm/libpg-query.wasm` files +- No native compilation or node-gyp dependencies required + +## Testing + +### Running Tests + +```bash +pnpm run test +``` + +### Test Requirements + +- WASM artifacts must be built before running tests +- If tests fail with "fetch failed" errors, rebuild WASM artifacts: + ```bash + pnpm run clean && pnpm run build && pnpm run test + ``` + + + +## Versions + +Built from the `18-constructive` branch of [constructive-io/libpg_query](https://github.com/constructive-io/libpg_query) (PostgreSQL 18, with PL/pgSQL serializer fixes). + + +## Troubleshooting + +### Common Issues + +**"fetch failed" errors during tests:** +- This indicates stale or missing WASM artifacts +- Solution: `pnpm run clean && pnpm run build` + +**"WASM module not initialized" errors:** +- Ensure you call an async method first to initialize the WASM module +- Or use the async versions of methods which handle initialization automatically + +**Build environment issues:** +- Ensure Emscripten SDK is properly installed and configured +- Check that all required build dependencies are available + +### Build Artifacts + +The build process generates these files: +- `wasm/libpg-query.js` - Emscripten-generated JavaScript loader +- `wasm/libpg-query.wasm` - WebAssembly binary +- `wasm/index.js` - ES module exports +- `wasm/index.cjs` - CommonJS exports with sync wrappers + +## Credits + +Built on the excellent work of several contributors: + +* **[Dan Lynch](https://github.com/pyramation)** β€” official maintainer since 2018 and architect of the current implementation +* **[Lukas Fittl](https://github.com/lfittl)** for [libpg_query](https://github.com/pganalyze/libpg_query) β€” the core PostgreSQL parser that powers this project +* **[Greg Richardson](https://github.com/gregnr)** for AST guidance and pushing the transition to WASM and multiple PG runtimes for better interoperability +* **[Ethan Resnick](https://github.com/ethanresnick)** for the original Node.js N-API bindings +* **[Zac McCormick](https://github.com/zhm)** for the foundational [node-pg-query-native](https://github.com/zhm/node-pg-query-native) parser + +**πŸ›  Built by the [Constructive](https://constructive.io) team β€” creators of modular Postgres tooling for secure, composable backends. If you like our work, contribute on [GitHub](https://github.com/constructive-io).** + + +## Related + +* [pgsql-parser](https://www.npmjs.com/package/pgsql-parser): The real PostgreSQL parser for Node.js, providing symmetric parsing and deparsing of SQL statements with actual PostgreSQL parser integration. +* [pgsql-deparser](https://www.npmjs.com/package/pgsql-deparser): A streamlined tool designed for converting PostgreSQL ASTs back into SQL queries, focusing solely on deparser functionality to complement `pgsql-parser`. +* [@pgsql/parser](https://www.npmjs.com/package/@pgsql/parser): Multi-version PostgreSQL parser with dynamic version selection at runtime, supporting PostgreSQL 15, 16, and 17 in a single package. +* [@pgsql/types](https://www.npmjs.com/package/@pgsql/types): Offers TypeScript type definitions for PostgreSQL AST nodes, facilitating type-safe construction, analysis, and manipulation of ASTs. +* [@pgsql/enums](https://www.npmjs.com/package/@pgsql/enums): Provides TypeScript enum definitions for PostgreSQL constants, enabling type-safe usage of PostgreSQL enums and constants in your applications. +* [@pgsql/utils](https://www.npmjs.com/package/@pgsql/utils): A comprehensive utility library for PostgreSQL, offering type-safe AST node creation and enum value conversions, simplifying the construction and manipulation of PostgreSQL ASTs. +* [pg-proto-parser](https://www.npmjs.com/package/pg-proto-parser): A TypeScript tool that parses PostgreSQL Protocol Buffers definitions to generate TypeScript interfaces, utility functions, and JSON mappings for enums. +* [libpg-query](https://github.com/constructive-io/libpg-query-node): The real PostgreSQL parser exposed for Node.js, used primarily in `pgsql-parser` for parsing and deparsing SQL queries. + +## Disclaimer + +AS DESCRIBED IN THE LICENSES, THE SOFTWARE IS PROVIDED "AS IS", AT YOUR OWN RISK, AND WITHOUT WARRANTIES OF ANY KIND. + +No developer or entity involved in creating Software will be liable for any claims or damages whatsoever associated with your use, inability to use, or your interaction with other users of the Software code or Software CLI, including any direct, indirect, incidental, special, exemplary, punitive or consequential damages, or loss of profits, cryptocurrencies, tokens, or anything else of value. \ No newline at end of file diff --git a/full/18/SCAN.md b/full/18/SCAN.md new file mode 100644 index 00000000..aafbc4fe --- /dev/null +++ b/full/18/SCAN.md @@ -0,0 +1,439 @@ +# Scan API Documentation + +## Overview + +The scan API provides detailed tokenization of PostgreSQL SQL queries, returning information about each token including its position, type, and keyword classification. This document explains how to use the scan functionality and how it relates to the parsing API. + +## Basic Usage + +### Async Scanning +```typescript +import { scan } from 'libpg-query'; + +const result = await scan('SELECT id, name FROM users WHERE active = true'); +console.log(result.tokens); +``` + +### Sync Scanning +```typescript +import { scanSync, loadModule } from 'libpg-query'; + +// Initialize WASM module first +await loadModule(); + +const result = scanSync('SELECT id, name FROM users WHERE active = true'); +console.log(result.tokens); +``` + +## Token Information + +Each token in the scan result contains: + +```typescript +interface ScanToken { + start: number; // Starting position in the SQL string (0-based) + end: number; // Ending position in the SQL string (exclusive) + text: string; // The actual token text extracted from SQL + tokenType: number; // Numeric token type identifier + tokenName: string; // Human-readable token type name + keywordKind: number; // Numeric keyword classification + keywordName: string; // Human-readable keyword classification +} +``` + +## Token Types + +### Common Token Types + +| tokenName | Description | Example | +|-----------|-------------|---------| +| `IDENT` | Regular identifier | `users`, `id`, `column_name` | +| `SCONST` | String constant | `'hello world'`, `'value'` | +| `ICONST` | Integer constant | `42`, `123`, `0` | +| `FCONST` | Float constant | `3.14`, `2.718` | +| `PARAM` | Parameter marker | `$1`, `$2`, `$3` | +| `ASCII_40` | Left parenthesis | `(` | +| `ASCII_41` | Right parenthesis | `)` | +| `ASCII_42` | Asterisk | `*` | +| `ASCII_44` | Comma | `,` | +| `ASCII_59` | Semicolon | `;` | +| `ASCII_61` | Equals sign | `=` | +| `TYPECAST` | Type casting operator | `::` | +| `GREATER_EQUALS` | Greater than or equal | `>=` | +| `LESS_EQUALS` | Less than or equal | `<=` | +| `NOT_EQUALS` | Not equal operator | `<>`, `!=` | +| `SQL_COMMENT` | SQL comment | `-- comment` | +| `C_COMMENT` | C-style comment | `/* comment */` | +| `UNKNOWN` | Keywords and other tokens | `SELECT`, `FROM`, `WHERE` | + +### Keyword Classifications + +| keywordName | Description | Examples | +|-------------|-------------|----------| +| `NO_KEYWORD` | Not a keyword | identifiers, operators, literals | +| `UNRESERVED_KEYWORD` | Unreserved keyword | `INSERT`, `UPDATE`, `name` | +| `COL_NAME_KEYWORD` | Column name keyword | `VALUES`, `BETWEEN` | +| `TYPE_FUNC_NAME_KEYWORD` | Type/function name keyword | `INNER`, `JOIN`, `IS` | +| `RESERVED_KEYWORD` | Reserved keyword | `SELECT`, `FROM`, `WHERE` | + +## Relationship with Parse Tree + +The scan token positions directly correspond to `location` fields in the Abstract Syntax Tree (AST) produced by the parse API. This allows you to map between tokens and AST nodes. + +### Example Mapping + +Given this SQL: +```sql +SELECT id, name FROM users WHERE active = true +``` + +**Scan tokens:** +``` +[0] "SELECT" (0-6) +[1] "id" (7-9) +[2] "," (9-10) +[3] "name" (11-15) +[4] "FROM" (16-20) +[5] "users" (21-26) +[6] "WHERE" (27-32) +[7] "active" (33-39) +[8] "=" (40-41) +[9] "true" (42-46) +``` + +**Corresponding AST locations:** +```json +{ + "SelectStmt": { + "targetList": [ + { + "ResTarget": { + "val": { + "ColumnRef": { + "fields": [{"String": {"sval": "id"}}], + "location": 7 // matches "id" token at position 7-9 + } + }, + "location": 7 + } + }, + { + "ResTarget": { + "val": { + "ColumnRef": { + "fields": [{"String": {"sval": "name"}}], + "location": 11 // matches "name" token at position 11-15 + } + }, + "location": 11 + } + } + ], + "fromClause": [ + { + "RangeVar": { + "relname": "users", + "location": 21 // matches "users" token at position 21-26 + } + } + ], + "whereClause": { + "A_Expr": { + "lexpr": { + "ColumnRef": { + "fields": [{"String": {"sval": "active"}}], + "location": 33 // matches "active" token at position 33-39 + } + }, + "rexpr": { + "A_Const": { + "boolval": {"boolval": true}, + "location": 42 // matches "true" token at position 42-46 + } + }, + "location": 40 // matches "=" token at position 40-41 + } + } + } +} +``` + +### Token-to-AST Mapping Function + +Here's a utility function to map between scan tokens and AST nodes: + +```typescript +import { scan, parse } from 'libpg-query'; + +interface TokenASTMapping { + token: ScanToken; + astNodes: Array<{path: string; node: any}>; +} + +async function mapTokensToAST(sql: string): Promise { + const [scanResult, parseResult] = await Promise.all([ + scan(sql), + parse(sql) + ]); + + // Collect all AST nodes with locations + const astNodes: Array<{path: string; location: number; node: any}> = []; + + function traverse(obj: any, path: string = '') { + if (obj && typeof obj === 'object') { + if (typeof obj.location === 'number') { + astNodes.push({path, location: obj.location, node: obj}); + } + + for (const [key, value] of Object.entries(obj)) { + const newPath = path ? `${path}.${key}` : key; + traverse(value, newPath); + } + } + } + + traverse(parseResult); + + // Map tokens to AST nodes + return scanResult.tokens.map(token => { + const matchingNodes = astNodes.filter(astNode => { + // AST location points to start of token + return astNode.location >= token.start && astNode.location < token.end; + }); + + return { + token, + astNodes: matchingNodes.map(n => ({path: n.path, node: n.node})) + }; + }); +} + +// Usage +const mappings = await mapTokensToAST('SELECT id FROM users WHERE active = true'); +mappings.forEach(({token, astNodes}) => { + console.log(`Token "${token.text}" (${token.start}-${token.end}):`); + astNodes.forEach(({path, node}) => { + console.log(` AST: ${path} at location ${node.location}`); + }); +}); +``` + +## Use Cases + +### 1. Syntax Highlighting + +Use scan results to apply syntax highlighting in code editors: + +```typescript +function applySyntaxHighlighting(sql: string, tokens: ScanToken[]): string { + let highlighted = ''; + let lastEnd = 0; + + for (const token of tokens) { + // Add any whitespace between tokens + highlighted += sql.substring(lastEnd, token.start); + + // Apply highlighting based on token type + const cssClass = getHighlightClass(token); + highlighted += `${token.text}`; + + lastEnd = token.end; + } + + // Add remaining SQL + highlighted += sql.substring(lastEnd); + + return highlighted; +} + +function getHighlightClass(token: ScanToken): string { + if (token.keywordName === 'RESERVED_KEYWORD') return 'sql-keyword'; + if (token.tokenName === 'SCONST') return 'sql-string'; + if (token.tokenName === 'ICONST' || token.tokenName === 'FCONST') return 'sql-number'; + if (token.tokenName === 'PARAM') return 'sql-parameter'; + if (token.tokenName === 'SQL_COMMENT' || token.tokenName === 'C_COMMENT') return 'sql-comment'; + return 'sql-default'; +} +``` + +### 2. Parameter Extraction + +Extract all parameters from a query: + +```typescript +function extractParameters(sql: string): Array<{param: string; position: number}> { + const result = scanSync(sql); + return result.tokens + .filter(token => token.tokenName === 'PARAM') + .map(token => ({ + param: token.text, + position: token.start + })); +} + +const params = extractParameters('SELECT * FROM users WHERE id = $1 AND name = $2'); +// Returns: [{param: '$1', position: 38}, {param: '$2', position: 52}] +``` + +### 3. Query Complexity Analysis + +Analyze query complexity based on token types: + +```typescript +function analyzeQueryComplexity(sql: string): { + joins: number; + subqueries: number; + parameters: number; + aggregates: string[]; + totalTokens: number; +} { + const result = scanSync(sql); + const tokens = result.tokens; + + const joins = tokens.filter(t => + t.text.toUpperCase() === 'JOIN' || + t.text.toUpperCase() === 'INNER' || + t.text.toUpperCase() === 'LEFT' || + t.text.toUpperCase() === 'RIGHT' + ).length; + + const subqueries = tokens.filter(t => t.text.toUpperCase() === 'SELECT').length - 1; + + const parameters = tokens.filter(t => t.tokenName === 'PARAM').length; + + const aggregates = tokens + .filter(t => ['COUNT', 'SUM', 'AVG', 'MIN', 'MAX'].includes(t.text.toUpperCase())) + .map(t => t.text.toUpperCase()); + + return { + joins, + subqueries: Math.max(0, subqueries), + parameters, + aggregates, + totalTokens: tokens.length + }; +} +``` + +### 4. SQL Formatting + +Use token positions for intelligent SQL formatting: + +```typescript +function formatSQL(sql: string): string { + const result = scanSync(sql); + let formatted = ''; + let indentLevel = 0; + let lastEnd = 0; + + for (const token of result.tokens) { + // Add whitespace between tokens + const gap = sql.substring(lastEnd, token.start); + + // Format based on token type + if (token.text.toUpperCase() === 'SELECT') { + formatted += '\\n' + ' '.repeat(indentLevel) + token.text; + } else if (token.text.toUpperCase() === 'FROM') { + formatted += '\\n' + ' '.repeat(indentLevel) + token.text; + } else if (token.text.toUpperCase() === 'WHERE') { + formatted += '\\n' + ' '.repeat(indentLevel) + token.text; + } else if (token.text === '(') { + formatted += token.text; + indentLevel++; + } else if (token.text === ')') { + indentLevel--; + formatted += token.text; + } else { + formatted += (gap.trim() ? ' ' : '') + token.text; + } + + lastEnd = token.end; + } + + return formatted.trim(); +} +``` + +## Performance Considerations + +- Scanning is generally faster than full parsing +- For large SQL strings, consider streaming or chunked processing +- Token positions are 0-based and use exclusive end positions +- The scan operation is stateless and thread-safe + +## Error Handling + +The scan API is more permissive than the parse API and will attempt to tokenize even malformed SQL: + +```typescript +try { + const result = scanSync('SELECT * FROM invalid$$$'); + // May still return tokens for recognizable parts + console.log(result.tokens); +} catch (error) { + console.error('Scan error:', error.message); +} +``` + +## Integration with Other Tools + +### ESLint Rules +Create custom ESLint rules for SQL: + +```typescript +function createSQLLintRule() { + return { + meta: { + type: 'problem', + docs: { description: 'Detect SQL injection risks' } + }, + create(context) { + return { + TemplateLiteral(node) { + if (node.quasiSConst) { + const sql = node.quasiSConst[0].value.raw; + const tokens = scanSync(sql); + + // Check for unparameterized string concatenation + const hasStringLiterals = tokens.some(t => t.tokenName === 'SCONST'); + const hasParameters = tokens.some(t => t.tokenName === 'PARAM'); + + if (hasStringLiterals && !hasParameters) { + context.report({ + node, + message: 'Potential SQL injection: use parameterized queries' + }); + } + } + } + }; + } + }; +} +``` + +### Database Migration Tools +Analyze schema changes: + +```typescript +function detectSchemaChanges(oldSQL: string, newSQL: string): string[] { + const oldTokens = scanSync(oldSQL); + const newTokens = scanSync(newSQL); + + const changes: string[] = []; + + // Detect table name changes + const oldTables = extractTableNames(oldTokens); + const newTables = extractTableNames(newTokens); + + const addedTables = newTables.filter(t => !oldTables.includes(t)); + const removedTables = oldTables.filter(t => !newTables.includes(t)); + + changes.push(...addedTables.map(t => `Added table: ${t}`)); + changes.push(...removedTables.map(t => `Removed table: ${t}`)); + + return changes; +} +``` + +This comprehensive relationship between scan tokens and AST locations enables powerful SQL analysis, transformation, and tooling capabilities. \ No newline at end of file diff --git a/full/18/libpg_query.md b/full/18/libpg_query.md new file mode 100644 index 00000000..2b4e4e8b --- /dev/null +++ b/full/18/libpg_query.md @@ -0,0 +1,558 @@ +# libpg_query API Documentation + +This document provides comprehensive documentation for the libpg_query API, focusing on the core parsing, scanning, and deparsing functionality. + +## Overview + +libpg_query is a C library that provides PostgreSQL SQL parsing functionality. It exposes the PostgreSQL parser as a standalone library, allowing you to parse SQL statements into parse trees, scan for tokens, and deparse parse trees back into SQL. + +## Core Data Structures + +### PgQueryError +```c +typedef struct { + char* message; // exception message + char* funcname; // source function of exception (e.g. SearchSysCache) + char* filename; // source of exception (e.g. parse.l) + int lineno; // source of exception (e.g. 104) + int cursorpos; // char in query at which exception occurred + char* context; // additional context (optional, can be NULL) +} PgQueryError; +``` + +### PgQueryProtobuf +```c +typedef struct { + size_t len; + char* data; +} PgQueryProtobuf; +``` + +### Parser Options +```c +typedef enum { + PG_QUERY_PARSE_DEFAULT = 0, + PG_QUERY_PARSE_TYPE_NAME, + PG_QUERY_PARSE_PLPGSQL_EXPR, + PG_QUERY_PARSE_PLPGSQL_ASSIGN1, + PG_QUERY_PARSE_PLPGSQL_ASSIGN2, + PG_QUERY_PARSE_PLPGSQL_ASSIGN3 +} PgQueryParseMode; +``` + +### Parser Option Flags +- `PG_QUERY_DISABLE_BACKSLASH_QUOTE` (16) - backslash_quote = off +- `PG_QUERY_DISABLE_STANDARD_CONFORMING_STRINGS` (32) - standard_conforming_strings = off +- `PG_QUERY_DISABLE_ESCAPE_STRING_WARNING` (64) - escape_string_warning = off + +## Core API Functions + +### Scanning Functions + +#### pg_query_scan +```c +PgQueryScanResult pg_query_scan(const char* input); +``` +**Description**: Scans SQL input and returns tokens in protobuf format. + +**Parameters**: +- `input`: SQL string to scan + +**Returns**: `PgQueryScanResult` containing: +- `pbuf`: Protobuf data with scan results +- `stderr_buffer`: Any stderr output during scanning +- `error`: Error information if scanning failed + +**Usage**: Use this when you need to tokenize SQL without full parsing. + +## Scanning and Token Processing + +### Working with Scan Results + +The `pg_query_scan` function returns tokens in protobuf format that need to be unpacked to access individual tokens. Here's the complete workflow: + +### Step 1: Scan SQL +```c +const char* sql = "SELECT * FROM users WHERE id = $1"; +PgQueryScanResult result = pg_query_scan(sql); + +if (result.error) { + printf("Scan error: %s at position %d\n", + result.error->message, result.error->cursorpos); + pg_query_free_scan_result(result); + return; +} +``` + +### Step 2: Unpack Protobuf Data +```c +#include "protobuf/pg_query.pb-c.h" + +PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( + NULL, // Use default allocator + result.pbuf.len, // Length of protobuf data + (void *) result.pbuf.data // Protobuf data +); + +printf("Found %zu tokens\n", scan_result->n_tokens); +``` + +### Step 3: Process Individual Tokens +```c +for (size_t i = 0; i < scan_result->n_tokens; i++) { + PgQuery__ScanToken *token = scan_result->tokens[i]; + + // Extract token text from original SQL + int token_length = token->end - token->start; + char token_text[token_length + 1]; + strncpy(token_text, &sql[token->start], token_length); + token_text[token_length] = '\0'; + + // Get token type name + const ProtobufCEnumValue *token_kind = + protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, token->token); + + // Get keyword classification + const ProtobufCEnumValue *keyword_kind = + protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, token->keyword_kind); + + printf("Token %zu: \"%s\" [%d-%d] Type: %s, Keyword: %s\n", + i, token_text, token->start, token->end, + token_kind->name, keyword_kind->name); +} +``` + +### Step 4: Clean Up Memory +```c +// Free the unpacked protobuf data +pg_query__scan_result__free_unpacked(scan_result, NULL); + +// Free the original scan result +pg_query_free_scan_result(result); +``` + +## Token Structure Details + +### PgQuery__ScanResult Structure +```c +struct PgQuery__ScanResult { + ProtobufCMessage base; + int32_t version; // Protocol version + size_t n_tokens; // Number of tokens + PgQuery__ScanToken **tokens; // Array of token pointers +}; +``` + +### PgQuery__ScanToken Structure +```c +struct PgQuery__ScanToken { + ProtobufCMessage base; + int32_t start; // Starting position in SQL string + int32_t end; // Ending position in SQL string + PgQuery__Token token; // Token type enum + PgQuery__KeywordKind keyword_kind; // Keyword classification +}; +``` + +## Token Types and Classifications + +### Keyword Classifications (PgQuery__KeywordKind) +- `PG_QUERY__KEYWORD_KIND__NO_KEYWORD` (0) - Not a keyword +- `PG_QUERY__KEYWORD_KIND__UNRESERVED_KEYWORD` (1) - Unreserved keyword +- `PG_QUERY__KEYWORD_KIND__COL_NAME_KEYWORD` (2) - Column name keyword +- `PG_QUERY__KEYWORD_KIND__TYPE_FUNC_NAME_KEYWORD` (3) - Type/function name keyword +- `PG_QUERY__KEYWORD_KIND__RESERVED_KEYWORD` (4) - Reserved keyword + +### Common Token Types (PgQuery__Token) +**Special/Control Tokens:** +- `PG_QUERY__TOKEN__NUL` (0) - Null token + +**Single-Character Operators:** +- `PG_QUERY__TOKEN__ASCII_40` (40) - "(" +- `PG_QUERY__TOKEN__ASCII_41` (41) - ")" +- `PG_QUERY__TOKEN__ASCII_42` (42) - "*" +- `PG_QUERY__TOKEN__ASCII_44` (44) - "," +- `PG_QUERY__TOKEN__ASCII_59` (59) - ";" +- `PG_QUERY__TOKEN__ASCII_61` (61) - "=" + +**Named Lexical Tokens:** +- `PG_QUERY__TOKEN__IDENT` (258) - Regular identifier +- `PG_QUERY__TOKEN__SCONST` (261) - String constant +- `PG_QUERY__TOKEN__ICONST` (266) - Integer constant +- `PG_QUERY__TOKEN__FCONST` (260) - Float constant +- `PG_QUERY__TOKEN__PARAM` (267) - Parameter marker ($1, $2, etc.) + +**Multi-Character Operators:** +- `PG_QUERY__TOKEN__TYPECAST` (268) - "::" +- `PG_QUERY__TOKEN__DOT_DOT` (269) - ".." +- `PG_QUERY__TOKEN__LESS_EQUALS` (272) - "<=" +- `PG_QUERY__TOKEN__GREATER_EQUALS` (273) - ">=" +- `PG_QUERY__TOKEN__NOT_EQUALS` (274) - "!=" or "<>" + +**Common SQL Keywords:** +- `PG_QUERY__TOKEN__SELECT` - SELECT keyword +- `PG_QUERY__TOKEN__FROM` - FROM keyword +- `PG_QUERY__TOKEN__WHERE` - WHERE keyword +- `PG_QUERY__TOKEN__INSERT` - INSERT keyword +- `PG_QUERY__TOKEN__UPDATE` - UPDATE keyword +- `PG_QUERY__TOKEN__DELETE` - DELETE keyword + +**Comments:** +- `PG_QUERY__TOKEN__SQL_COMMENT` (275) - SQL-style comment (-- comment) +- `PG_QUERY__TOKEN__C_COMMENT` (276) - C-style comment (/* comment */) + +## Protobuf Helper Functions + +### Unpacking Functions +```c +// Unpack scan result +PgQuery__ScanResult *pg_query__scan_result__unpack( + ProtobufCAllocator *allocator, // NULL for default + size_t len, // Length of data + const uint8_t *data // Protobuf data +); + +// Free unpacked scan result +void pg_query__scan_result__free_unpacked( + PgQuery__ScanResult *message, // Message to free + ProtobufCAllocator *allocator // NULL for default +); +``` + +### Enum Value Lookup Functions +```c +// Get token type name +const ProtobufCEnumValue *protobuf_c_enum_descriptor_get_value( + const ProtobufCEnumDescriptor *desc, // &pg_query__token__descriptor + int value // token->token +); + +// Get keyword kind name +const ProtobufCEnumValue *protobuf_c_enum_descriptor_get_value( + const ProtobufCEnumDescriptor *desc, // &pg_query__keyword_kind__descriptor + int value // token->keyword_kind +); +``` + +## Complete Example: SQL Tokenizer + +```c +#include +#include +#include +#include +#include "protobuf/pg_query.pb-c.h" + +void print_tokens(const char* sql) { + PgQueryScanResult result = pg_query_scan(sql); + + if (result.error) { + printf("Error: %s at position %d\n", + result.error->message, result.error->cursorpos); + pg_query_free_scan_result(result); + return; + } + + PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( + NULL, result.pbuf.len, (void *) result.pbuf.data); + + printf("SQL: %s\n", sql); + printf("Tokens (%zu):\n", scan_result->n_tokens); + + for (size_t i = 0; i < scan_result->n_tokens; i++) { + PgQuery__ScanToken *token = scan_result->tokens[i]; + + // Extract token text + int len = token->end - token->start; + printf(" [%zu] \"%.*s\" (%d-%d) ", + i, len, &sql[token->start], token->start, token->end); + + // Get token type + const ProtobufCEnumValue *token_kind = + protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, token->token); + printf("Type: %s", token_kind->name); + + // Get keyword classification if applicable + if (token->keyword_kind != PG_QUERY__KEYWORD_KIND__NO_KEYWORD) { + const ProtobufCEnumValue *keyword_kind = + protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, token->keyword_kind); + printf(", Keyword: %s", keyword_kind->name); + } + printf("\n"); + } + + pg_query__scan_result__free_unpacked(scan_result, NULL); + pg_query_free_scan_result(result); +} + +int main() { + print_tokens("SELECT * FROM users WHERE id = $1"); + print_tokens("INSERT INTO table VALUES (1, 'text', 3.14)"); + print_tokens("-- Comment\nUPDATE table SET col = col + 1"); + + pg_query_exit(); + return 0; +} +``` + +## Build Requirements + +To use scanning functionality, compile with: +```bash +gcc -I. -I./protobuf your_program.c -lpg_query -lprotobuf-c +``` + +Make sure to include: +- `pg_query.h` - Main API header +- `protobuf/pg_query.pb-c.h` - Protobuf definitions + +### Parsing Functions + +#### pg_query_parse +```c +PgQueryParseResult pg_query_parse(const char* input); +``` +**Description**: Parses SQL input into a JSON parse tree. + +**Parameters**: +- `input`: SQL string to parse + +**Returns**: `PgQueryParseResult` containing: +- `parse_tree`: JSON representation of the parse tree +- `stderr_buffer`: Any stderr output during parsing +- `error`: Error information if parsing failed + +#### pg_query_parse_opts +```c +PgQueryParseResult pg_query_parse_opts(const char* input, int parser_options); +``` +**Description**: Parses SQL input with custom parser options. + +**Parameters**: +- `input`: SQL string to parse +- `parser_options`: Bitwise OR of parser options and flags + +**Returns**: Same as `pg_query_parse` + +#### pg_query_parse_protobuf +```c +PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input); +``` +**Description**: Parses SQL input into protobuf format parse tree. + +**Parameters**: +- `input`: SQL string to parse + +**Returns**: `PgQueryProtobufParseResult` containing: +- `parse_tree`: Protobuf representation of the parse tree +- `stderr_buffer`: Any stderr output during parsing +- `error`: Error information if parsing failed + +#### pg_query_parse_protobuf_opts +```c +PgQueryProtobufParseResult pg_query_parse_protobuf_opts(const char* input, int parser_options); +``` +**Description**: Parses SQL input into protobuf format with custom options. + +**Parameters**: +- `input`: SQL string to parse +- `parser_options`: Bitwise OR of parser options and flags + +**Returns**: Same as `pg_query_parse_protobuf` + +#### pg_query_parse_plpgsql +```c +PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); +``` +**Description**: Parses PL/pgSQL code. + +**Parameters**: +- `input`: PL/pgSQL code to parse + +**Returns**: `PgQueryPlpgsqlParseResult` containing: +- `plpgsql_funcs`: JSON representation of PL/pgSQL functions +- `error`: Error information if parsing failed + +### Deparsing Functions + +#### pg_query_deparse_protobuf +```c +PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree); +``` +**Description**: Converts a protobuf parse tree back into SQL. + +**Parameters**: +- `parse_tree`: Protobuf parse tree to deparse + +**Returns**: `PgQueryDeparseResult` containing: +- `query`: Deparsed SQL string +- `error`: Error information if deparsing failed + +**Usage**: Use this to convert parse trees back to SQL, useful for query transformation. + +### Utility Functions + +#### pg_query_normalize +```c +PgQueryNormalizeResult pg_query_normalize(const char* input); +``` +**Description**: Normalizes a SQL query by removing comments and standardizing formatting. + +**Parameters**: +- `input`: SQL string to normalize + +**Returns**: `PgQueryNormalizeResult` containing: +- `normalized_query`: Normalized SQL string +- `error`: Error information if normalization failed + +#### pg_query_normalize_utility +```c +PgQueryNormalizeResult pg_query_normalize_utility(const char* input); +``` +**Description**: Normalizes utility statements (DDL, etc.). + +**Parameters**: +- `input`: SQL string to normalize + +**Returns**: Same as `pg_query_normalize` + +#### pg_query_fingerprint +```c +PgQueryFingerprintResult pg_query_fingerprint(const char* input); +``` +**Description**: Generates a fingerprint for a SQL query. + +**Parameters**: +- `input`: SQL string to fingerprint + +**Returns**: `PgQueryFingerprintResult` containing: +- `fingerprint`: 64-bit fingerprint hash +- `fingerprint_str`: String representation of fingerprint +- `stderr_buffer`: Any stderr output +- `error`: Error information if fingerprinting failed + +#### pg_query_fingerprint_opts +```c +PgQueryFingerprintResult pg_query_fingerprint_opts(const char* input, int parser_options); +``` +**Description**: Generates a fingerprint with custom parser options. + +**Parameters**: +- `input`: SQL string to fingerprint +- `parser_options`: Bitwise OR of parser options and flags + +**Returns**: Same as `pg_query_fingerprint` + +### Statement Splitting Functions + +#### pg_query_split_with_scanner +```c +PgQuerySplitResult pg_query_split_with_scanner(const char *input); +``` +**Description**: Splits multi-statement SQL using the scanner. Use when statements may contain parse errors. + +**Parameters**: +- `input`: SQL string containing multiple statements + +**Returns**: `PgQuerySplitResult` containing: +- `stmts`: Array of statement locations and lengths +- `n_stmts`: Number of statements found +- `stderr_buffer`: Any stderr output +- `error`: Error information if splitting failed + +#### pg_query_split_with_parser +```c +PgQuerySplitResult pg_query_split_with_parser(const char *input); +``` +**Description**: Splits multi-statement SQL using the parser (recommended for better accuracy). + +**Parameters**: +- `input`: SQL string containing multiple statements + +**Returns**: Same as `pg_query_split_with_scanner` + +## Memory Management + +### Cleanup Functions +All result structures must be freed using their corresponding cleanup functions: + +```c +void pg_query_free_normalize_result(PgQueryNormalizeResult result); +void pg_query_free_scan_result(PgQueryScanResult result); +void pg_query_free_parse_result(PgQueryParseResult result); +void pg_query_free_split_result(PgQuerySplitResult result); +void pg_query_free_deparse_result(PgQueryDeparseResult result); +void pg_query_free_protobuf_parse_result(PgQueryProtobufParseResult result); +void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result); +void pg_query_free_fingerprint_result(PgQueryFingerprintResult result); +``` + +### Global Cleanup +```c +void pg_query_exit(void); +``` +**Description**: Optional cleanup of top-level memory context. Automatically done for threads that exit. + +## Error Handling + +All functions return result structures that include an `error` field. Always check this field before using other result data: + +```c +PgQueryParseResult result = pg_query_parse(sql); +if (result.error) { + printf("Parse error: %s\n", result.error->message); + printf("Location: %s:%d\n", result.error->filename, result.error->lineno); + if (result.error->cursorpos > 0) { + printf("Position: %d\n", result.error->cursorpos); + } +} else { + // Use result.parse_tree +} +pg_query_free_parse_result(result); +``` + +## Example Usage + +### Basic Parsing +```c +#include "pg_query.h" + +const char* sql = "SELECT * FROM users WHERE id = $1"; +PgQueryParseResult result = pg_query_parse(sql); + +if (result.error) { + printf("Error: %s\n", result.error->message); +} else { + printf("Parse tree: %s\n", result.parse_tree); +} + +pg_query_free_parse_result(result); +``` + +### Parse and Deparse Cycle +```c +// Parse to protobuf +PgQueryProtobufParseResult parse_result = pg_query_parse_protobuf(sql); +if (!parse_result.error) { + // Deparse back to SQL + PgQueryDeparseResult deparse_result = pg_query_deparse_protobuf(parse_result.parse_tree); + if (!deparse_result.error) { + printf("Deparsed: %s\n", deparse_result.query); + } + pg_query_free_deparse_result(deparse_result); +} +pg_query_free_protobuf_parse_result(parse_result); +``` + +## Version Information + +- PostgreSQL Version: 17.4 (PG_VERSION_NUM: 170004) +- Major Version: 17 + +## Notes + +- The library is thread-safe +- Always free result structures to avoid memory leaks +- Use protobuf format for better performance when doing parse/deparse cycles +- Scanner-based splitting is more robust for malformed SQL +- Parser-based splitting is more accurate for well-formed SQL \ No newline at end of file diff --git a/full/18/libpg_query/protobuf/.gitkeep b/full/18/libpg_query/protobuf/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/full/18/package.json b/full/18/package.json new file mode 100644 index 00000000..a06bb5a2 --- /dev/null +++ b/full/18/package.json @@ -0,0 +1,55 @@ +{ + "name": "@libpg-query/full18", + "version": "18.0.0", + "description": "The real PostgreSQL query parser", + "homepage": "https://github.com/constructive-io/libpg-query-node", + "main": "./wasm/index.cjs", + "module": "./wasm/index.js", + "typings": "./wasm/index.d.ts", + "publishConfig": { + "access": "public" + }, + "x-publish": { + "publishName": "@libpg-query/parser", + "pgVersion": "18", + "distTag": "pg18", + "libpgQueryTag": "18-constructive" + }, + "files": [ + "wasm/*" + ], + "scripts": { + "clean": "pnpm wasm:clean && rimraf cjs esm", + "build:js": "node scripts/build.js", + "build": "pnpm clean; pnpm wasm:build; pnpm build:js", + "publish:pkg": "node ../../scripts/publish-single-version.js", + "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", + "wasm:build": "pnpm wasm:make build", + "wasm:rebuild": "pnpm wasm:make rebuild", + "wasm:clean": "pnpm wasm:make clean", + "wasm:clean-cache": "pnpm wasm:make clean-cache", + "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js", + "yamlize": "node ./scripts/yamlize.js" + }, + "author": "Constructive ", + "license": "MIT", + "repository": { + "type": "git", + "url": "git://github.com/constructive-io/libpg-query-node.git" + }, + "devDependencies": { + "@yamlize/cli": "^0.8.0" + }, + "dependencies": { + "@pgsql/types": "^18.0.0" + }, + "keywords": [ + "sql", + "postgres", + "postgresql", + "pg", + "query", + "plpgsql", + "database" + ] +} diff --git a/full/18/scripts/build.js b/full/18/scripts/build.js new file mode 100644 index 00000000..55d83240 --- /dev/null +++ b/full/18/scripts/build.js @@ -0,0 +1,38 @@ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +// Run TypeScript compilation +console.log('Compiling TypeScript...'); +execSync('tsc', { stdio: 'inherit' }); +execSync('tsc -p tsconfig.esm.json', { stdio: 'inherit' }); + +// Rename files to have correct extensions +const wasmDir = path.join(__dirname, '../wasm'); +const cjsDir = path.join(__dirname, '../cjs'); +const esmDir = path.join(__dirname, '../esm'); + +// Ensure wasm directory exists +if (!fs.existsSync(wasmDir)) { + fs.mkdirSync(wasmDir, { recursive: true }); +} + +// Rename CommonJS files +fs.renameSync( + path.join(cjsDir, 'index.js'), + path.join(wasmDir, 'index.cjs') +); + +// Rename ESM files +fs.renameSync( + path.join(esmDir, 'index.js'), + path.join(wasmDir, 'index.js') +); + +// Rename declaration files +fs.renameSync( + path.join(cjsDir, 'index.d.ts'), + path.join(wasmDir, 'index.d.ts') +); + +console.log('Build completed successfully!'); \ No newline at end of file diff --git a/full/18/src/index.ts b/full/18/src/index.ts new file mode 100644 index 00000000..b616d923 --- /dev/null +++ b/full/18/src/index.ts @@ -0,0 +1,484 @@ +import { ParseResult } from "@pgsql/types"; +export * from "@pgsql/types"; + +export interface ScanToken { + start: number; + end: number; + text: string; + tokenType: number; + tokenName: string; + keywordKind: number; + keywordName: string; +} + +export interface ScanResult { + version: number; + tokens: ScanToken[]; +} + +export interface SqlErrorDetails { + message: string; + cursorPosition: number; + fileName?: string; + functionName?: string; + lineNumber?: number; + context?: string; +} + +export class SqlError extends Error { + sqlDetails?: SqlErrorDetails; + + constructor(message: string, details?: SqlErrorDetails) { + super(message); + this.name = 'SqlError'; + this.sqlDetails = details; + } +} + +export function hasSqlDetails(error: unknown): error is SqlError { + return error instanceof SqlError && error.sqlDetails !== undefined; +} + +export function formatSqlError( + error: SqlError, + query: string, + options: { + showPosition?: boolean; + showQuery?: boolean; + color?: boolean; + maxQueryLength?: number; + } = {} +): string { + const { + showPosition = true, + showQuery = true, + color = false, + maxQueryLength + } = options; + + const lines: string[] = []; + + // ANSI color codes + const red = color ? '\x1b[31m' : ''; + const yellow = color ? '\x1b[33m' : ''; + const reset = color ? '\x1b[0m' : ''; + + // Add error message + lines.push(`${red}Error: ${error.message}${reset}`); + + // Add SQL details if available + if (error.sqlDetails) { + const { cursorPosition, fileName, functionName, lineNumber } = error.sqlDetails; + + if (cursorPosition !== undefined && cursorPosition >= 0) { + lines.push(`Position: ${cursorPosition}`); + } + + if (fileName || functionName || lineNumber) { + const details = []; + if (fileName) details.push(`file: ${fileName}`); + if (functionName) details.push(`function: ${functionName}`); + if (lineNumber) details.push(`line: ${lineNumber}`); + lines.push(`Source: ${details.join(', ')}`); + } + + // Show query with position marker + if (showQuery && showPosition && cursorPosition !== undefined && cursorPosition >= 0) { + let displayQuery = query; + let adjustedPosition = cursorPosition; + + // Truncate if needed + if (maxQueryLength && query.length > maxQueryLength) { + const start = Math.max(0, cursorPosition - Math.floor(maxQueryLength / 2)); + const end = Math.min(query.length, start + maxQueryLength); + displayQuery = (start > 0 ? '...' : '') + + query.substring(start, end) + + (end < query.length ? '...' : ''); + // Adjust cursor position for truncation + adjustedPosition = cursorPosition - start + (start > 0 ? 3 : 0); + } + + lines.push(displayQuery); + lines.push(' '.repeat(adjustedPosition) + `${yellow}^${reset}`); + } + } else if (showQuery) { + // No SQL details, just show the query if requested + let displayQuery = query; + if (maxQueryLength && query.length > maxQueryLength) { + displayQuery = query.substring(0, maxQueryLength) + '...'; + } + lines.push(`Query: ${displayQuery}`); + } + + return lines.join('\n'); +} + +// @ts-ignore +import PgQueryModule from './libpg-query.js'; + +interface WasmModule { + _malloc: (size: number) => number; + _free: (ptr: number) => void; + _wasm_free_string: (ptr: number) => void; + _wasm_parse_query: (queryPtr: number) => number; + _wasm_parse_query_raw: (queryPtr: number) => number; + _wasm_free_parse_result: (ptr: number) => void; + _wasm_parse_plpgsql: (queryPtr: number) => number; + _wasm_fingerprint: (queryPtr: number) => number; + _wasm_normalize_query: (queryPtr: number) => number; + _wasm_scan: (queryPtr: number) => number; + lengthBytesUTF8: (str: string) => number; + stringToUTF8: (str: string, ptr: number, len: number) => void; + UTF8ToString: (ptr: number) => string; + getValue: (ptr: number, type: string) => number; +} + +let wasmModule: WasmModule; + +const initPromise = PgQueryModule().then((module: WasmModule) => { + wasmModule = module; +}); + +function ensureLoaded() { + if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); +} + +export async function loadModule(): Promise { + if (!wasmModule) { + await initPromise; + } +} + +function awaitInit Promise>(fn: T): T { + return (async (...args: Parameters) => { + await initPromise; + return fn(...args); + }) as T; +} + +function stringToPtr(str: string): number { + ensureLoaded(); + if (typeof str !== 'string') { + throw new TypeError(`Expected a string, got ${typeof str}`); + } + const len = wasmModule.lengthBytesUTF8(str) + 1; + const ptr = wasmModule._malloc(len); + try { + wasmModule.stringToUTF8(str, ptr, len); + return ptr; + } catch (error) { + wasmModule._free(ptr); + throw error; + } +} + +function ptrToString(ptr: number): string { + ensureLoaded(); + if (typeof ptr !== 'number') { + throw new TypeError(`Expected a number, got ${typeof ptr}`); + } + return wasmModule.UTF8ToString(ptr); +} + +export const parse = awaitInit(async (query: string): Promise => { + // Input validation + if (query === null || query === undefined) { + throw new Error('Query cannot be null or undefined'); + } + + if (query === '') { + throw new Error('Query cannot be empty'); + } + + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_parse_query_raw(queryPtr); + if (!resultPtr) { + throw new Error('Failed to parse query: memory allocation failed'); + } + + // Read the PgQueryParseResult struct + const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); + const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); + const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); + + if (errorPtr) { + // Read PgQueryError struct + const messagePtr = wasmModule.getValue(errorPtr, 'i32'); + const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); + const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); + const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); + const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); + + const message = messagePtr ? wasmModule.UTF8ToString(messagePtr) : 'Unknown error'; + const funcname = funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined; + const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : undefined; + + throw new SqlError(message, { + message, + cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, // Convert to 0-based + fileName: filename, + functionName: funcname, + lineNumber: lineno > 0 ? lineno : undefined + }); + } + + if (!parseTreePtr) { + throw new Error('No parse tree generated'); + } + + const parseTreeStr = wasmModule.UTF8ToString(parseTreePtr); + return JSON.parse(parseTreeStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_parse_result(resultPtr); + } + } +}); + +export const parsePlPgSQL = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +// Sync versions +export function parseSync(query: string): ParseResult { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + + // Input validation + if (query === null || query === undefined) { + throw new Error('Query cannot be null or undefined'); + } + + if (query === '') { + throw new Error('Query cannot be empty'); + } + + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_parse_query_raw(queryPtr); + if (!resultPtr) { + throw new Error('Failed to parse query: memory allocation failed'); + } + + // Read the PgQueryParseResult struct + const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); + const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); + const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); + + if (errorPtr) { + // Read PgQueryError struct + const messagePtr = wasmModule.getValue(errorPtr, 'i32'); + const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); + const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); + const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); + const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); + + const message = messagePtr ? wasmModule.UTF8ToString(messagePtr) : 'Unknown error'; + const funcname = funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined; + const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : undefined; + + throw new SqlError(message, { + message, + cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, // Convert to 0-based + fileName: filename, + functionName: funcname, + lineNumber: lineno > 0 ? lineno : undefined + }); + } + + if (!parseTreePtr) { + throw new Error('No parse tree generated'); + } + + const parseTreeStr = wasmModule.UTF8ToString(parseTreePtr); + return JSON.parse(parseTreeStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_parse_result(resultPtr); + } + } +} + +export function parsePlPgSQLSync(query: string): ParseResult { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query: string): string { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query: string): string { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export const scan = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_scan(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function scanSync(query: string): ScanResult { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_scan(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} \ No newline at end of file diff --git a/full/18/src/libpg-query.d.ts b/full/18/src/libpg-query.d.ts new file mode 100644 index 00000000..a7c80962 --- /dev/null +++ b/full/18/src/libpg-query.d.ts @@ -0,0 +1,17 @@ +declare module './libpg-query.js' { + interface WasmModule { + _malloc: (size: number) => number; + _free: (ptr: number) => void; + _wasm_free_string: (ptr: number) => void; + _wasm_parse_query: (queryPtr: number) => number; + _wasm_parse_plpgsql: (queryPtr: number) => number; + _wasm_fingerprint: (queryPtr: number) => number; + _wasm_normalize_query: (queryPtr: number) => number; + lengthBytesUTF8: (str: string) => number; + stringToUTF8: (str: string, ptr: number, len: number) => void; + UTF8ToString: (ptr: number) => string; + } + + const PgQueryModule: () => Promise; + export default PgQueryModule; +} \ No newline at end of file diff --git a/full/18/src/proto.d.ts b/full/18/src/proto.d.ts new file mode 100644 index 00000000..55a504f3 --- /dev/null +++ b/full/18/src/proto.d.ts @@ -0,0 +1,20 @@ +declare module '../proto.js' { + export namespace pg_query { + interface ParseResult { + version: number; + stmts: Statement[]; + } + + interface Statement { + stmt_type: string; + stmt_len: number; + stmt_location: number; + query: string; + } + + class ParseResult { + static fromObject(obj: ParseResult): ParseResult; + static encode(msg: ParseResult): { finish(): Uint8Array }; + } + } +} \ No newline at end of file diff --git a/full/18/src/wasm_wrapper.c b/full/18/src/wasm_wrapper.c new file mode 100644 index 00000000..3844d960 --- /dev/null +++ b/full/18/src/wasm_wrapper.c @@ -0,0 +1,395 @@ +#include "pg_query.h" +#include "protobuf/pg_query.pb-c.h" +#include +#include +#include +#include + +static int validate_input(const char* input) { + return input != NULL && strlen(input) > 0; +} + +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + +static void* safe_malloc(size_t size) { + void* ptr = malloc(size); + if (!ptr && size > 0) { + return NULL; + } + return ptr; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +PgQueryParseResult* wasm_parse_query_raw(const char* input) { + PgQueryParseResult* result = (PgQueryParseResult*)malloc(sizeof(PgQueryParseResult)); + if (!result) { + return NULL; + } + + *result = pg_query_parse(input); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_parse_result(PgQueryParseResult* result) { + if (result) { + pg_query_free_parse_result(*result); + free(result); + } +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + + + + + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +static const char* get_token_name(PgQuery__Token token_type) { + // Map some common token types to readable names + // Note: This is a simplified mapping - full enum lookup would require more complexity + switch(token_type) { + case 258: return "IDENT"; + case 261: return "SCONST"; + case 266: return "ICONST"; + case 260: return "FCONST"; + case 267: return "PARAM"; + case 40: return "ASCII_40"; // ( + case 41: return "ASCII_41"; // ) + case 42: return "ASCII_42"; // * + case 44: return "ASCII_44"; // , + case 59: return "ASCII_59"; // ; + case 61: return "ASCII_61"; // = + case 268: return "TYPECAST"; + case 272: return "LESS_EQUALS"; + case 273: return "GREATER_EQUALS"; + case 274: return "NOT_EQUALS"; + case 275: return "SQL_COMMENT"; + case 276: return "C_COMMENT"; + default: return "UNKNOWN"; + } +} + +static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { + switch(keyword_kind) { + case 0: return "NO_KEYWORD"; + case 1: return "UNRESERVED_KEYWORD"; + case 2: return "COL_NAME_KEYWORD"; + case 3: return "TYPE_FUNC_NAME_KEYWORD"; + case 4: return "RESERVED_KEYWORD"; + default: return "UNKNOWN_KEYWORD"; + } +} + +static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { + if (!scan_result || !original_sql) { + return safe_strdup("{\"version\":0,\"tokens\":[]}"); + } + + // Calculate rough JSON size estimate + size_t estimated_size = 1024 + (scan_result->n_tokens * 200); + char* json = safe_malloc(estimated_size); + if (!json) { + return safe_strdup("{\"version\":0,\"tokens\":[]}"); + } + + // Start building JSON + int pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version); + + for (size_t i = 0; i < scan_result->n_tokens; i++) { + PgQuery__ScanToken *token = scan_result->tokens[i]; + + // Extract token text from original SQL + int token_length = token->end - token->start; + if (token_length < 0) token_length = 0; // Safety check + + char* token_text = safe_malloc(token_length + 1); + if (!token_text) continue; + + if (token_length > 0) { + strncpy(token_text, &original_sql[token->start], token_length); + } + token_text[token_length] = '\0'; + + // Escape token text for JSON + char* escaped_text = safe_malloc(token_length * 2 + 1); + if (!escaped_text) { + free(token_text); + continue; + } + + int escaped_pos = 0; + for (int j = 0; j < token_length; j++) { + char c = token_text[j]; + if (c == '"' || c == '\\') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = c; + } else if (c == '\n') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = 'n'; + } else if (c == '\r') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = 'r'; + } else if (c == '\t') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = 't'; + } else { + escaped_text[escaped_pos++] = c; + } + } + escaped_text[escaped_pos] = '\0'; + + // Get token type name and keyword kind name + const char* token_name = get_token_name(token->token); + const char* keyword_name = get_keyword_name(token->keyword_kind); + + // Add comma if not first token + if (i > 0) { + pos += snprintf(json + pos, estimated_size - pos, ","); + } + + // Add token object to JSON + pos += snprintf(json + pos, estimated_size - pos, + "{\"start\":%d,\"end\":%d,\"text\":\"%s\",\"tokenType\":%d,\"tokenName\":\"%s\",\"keywordKind\":%d,\"keywordName\":\"%s\"}", + token->start, token->end, escaped_text, token->token, token_name, token->keyword_kind, keyword_name); + + free(token_text); + free(escaped_text); + + // Check if we're running out of space + if (pos >= estimated_size - 200) { + char* new_json = realloc(json, estimated_size * 2); + if (!new_json) break; + json = new_json; + estimated_size *= 2; + } + } + + // Close JSON + snprintf(json + pos, estimated_size - pos, "]}"); + + return json; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_scan(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryScanResult result = pg_query_scan(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_scan_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + // Unpack protobuf data + PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( + NULL, result.pbuf.len, (void *) result.pbuf.data); + + if (!scan_result) { + pg_query_free_scan_result(result); + return safe_strdup("Failed to unpack scan result"); + } + + // Convert to JSON + char* json_result = build_scan_json(scan_result, input); + + // Clean up + pg_query__scan_result__free_unpacked(scan_result, NULL); + pg_query_free_scan_result(result); + + return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} diff --git a/full/18/test/errors.test.js b/full/18/test/errors.test.js new file mode 100644 index 00000000..f6c0fd61 --- /dev/null +++ b/full/18/test/errors.test.js @@ -0,0 +1,325 @@ +const { describe, it, before } = require('node:test'); +const assert = require('node:assert/strict'); +const { parseSync, loadModule, formatSqlError, hasSqlDetails } = require('../wasm/index.cjs'); + +describe('Enhanced Error Handling', () => { + before(async () => { + await loadModule(); + }); + + describe('Error Details Structure', () => { + it('should include sqlDetails property on parse errors', () => { + assert.throws(() => { + parseSync('SELECT * FROM users WHERE id = @'); + }); + + try { + parseSync('SELECT * FROM users WHERE id = @'); + } catch (error) { + assert.ok('sqlDetails' in error); + assert.ok('message' in error.sqlDetails); + assert.ok('cursorPosition' in error.sqlDetails); + assert.ok('fileName' in error.sqlDetails); + assert.ok('functionName' in error.sqlDetails); + assert.ok('lineNumber' in error.sqlDetails); + } + }); + + it('should have correct cursor position (0-based)', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 32); + } + }); + + it('should identify error source file', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.fileName, 'scan.l'); + assert.equal(error.sqlDetails.functionName, 'scanner_yyerror'); + } + }); + }); + + describe('Error Position Accuracy', () => { + const positionTests = [ + { query: '@ SELECT * FROM users', expectedPos: 0, desc: 'error at start' }, + { query: 'SELECT @ FROM users', expectedPos: 9, desc: 'error after SELECT' }, + { query: 'SELECT * FROM users WHERE @ = 1', expectedPos: 28, desc: 'error after WHERE' }, + { query: 'SELECT * FROM users WHERE id = @', expectedPos: 32, desc: 'error at end' }, + { query: 'INSERT INTO users (id, name) VALUES (1, @)', expectedPos: 41, desc: 'error in VALUES' }, + { query: 'UPDATE users SET name = @ WHERE id = 1', expectedPos: 26, desc: 'error in SET' }, + { query: 'CREATE TABLE test (id INT, name @)', expectedPos: 32, desc: 'error in CREATE TABLE' }, + ]; + + positionTests.forEach(({ query, expectedPos, desc }) => { + it(`should correctly identify position for ${desc}`, () => { + try { + parseSync(query); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, expectedPos); + } + }); + }); + }); + + describe('Error Types', () => { + it('should handle unterminated string literals', () => { + try { + parseSync("SELECT * FROM users WHERE name = 'unclosed"); + assert.fail('Expected error'); + } catch (error) { + assert.ok(error.message.includes('unterminated quoted string')); + assert.equal(error.sqlDetails.cursorPosition, 33); + } + }); + + it('should handle unterminated quoted identifiers', () => { + try { + parseSync('SELECT * FROM users WHERE name = "unclosed'); + assert.fail('Expected error'); + } catch (error) { + assert.ok(error.message.includes('unterminated quoted identifier')); + assert.equal(error.sqlDetails.cursorPosition, 33); + } + }); + + it('should handle invalid tokens', () => { + try { + parseSync('SELECT * FROM users WHERE id = $'); + assert.fail('Expected error'); + } catch (error) { + assert.ok(error.message.includes('syntax error at or near "$"')); + assert.equal(error.sqlDetails.cursorPosition, 31); + } + }); + + it('should handle reserved keywords', () => { + try { + parseSync('SELECT * FROM table'); + assert.fail('Expected error'); + } catch (error) { + assert.ok(error.message.includes('syntax error at or near "table"')); + assert.equal(error.sqlDetails.cursorPosition, 14); + } + }); + + it('should handle syntax error in WHERE clause', () => { + try { + parseSync('SELECT * FROM users WHERE'); + assert.fail('Expected error'); + } catch (error) { + assert.ok(error.message.includes('syntax error at end of input')); + assert.equal(error.sqlDetails.cursorPosition, 25); + } + }); + }); + + describe('formatSqlError Helper', () => { + it('should format error with position indicator', () => { + try { + parseSync("SELECT * FROM users WHERE id = 'unclosed"); + assert.fail('Expected error'); + } catch (error) { + const formatted = formatSqlError(error, "SELECT * FROM users WHERE id = 'unclosed"); + assert.ok(formatted.includes('Error: unterminated quoted string')); + assert.ok(formatted.includes('Position: 31')); + assert.ok(formatted.includes("SELECT * FROM users WHERE id = 'unclosed")); + assert.ok(formatted.includes(' ^')); + } + }); + + it('should respect showPosition option', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { + showPosition: false + }); + assert.ok(!formatted.includes('^')); + assert.ok(formatted.includes('Position: 32')); + } + }); + + it('should respect showQuery option', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { + showQuery: false + }); + assert.ok(!formatted.includes('SELECT * FROM users')); + assert.ok(formatted.includes('Error:')); + assert.ok(formatted.includes('Position:')); + } + }); + + it('should truncate long queries', () => { + const longQuery = 'SELECT ' + 'a, '.repeat(50) + 'z FROM users WHERE id = @'; + try { + parseSync(longQuery); + assert.fail('Expected error'); + } catch (error) { + const formatted = formatSqlError(error, longQuery, { maxQueryLength: 50 }); + assert.ok(formatted.includes('...')); + const lines = formatted.split('\n'); + const queryLine = lines.find(line => line.includes('...')); + assert.ok(queryLine.length <= 56); // 50 + 2*3 for ellipsis + } + }); + + it('should handle color option without breaking output', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { + color: true + }); + assert.ok(formatted.includes('Error:')); + assert.ok(formatted.includes('Position:')); + // Should contain ANSI codes but still be readable + const cleanFormatted = formatted.replace(/\x1b\[[0-9;]*m/g, ''); + assert.ok(cleanFormatted.includes('syntax error')); + } + }); + }); + + describe('hasSqlDetails Type Guard', () => { + it('should return true for SQL parse errors', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(hasSqlDetails(error), true); + } + }); + + it('should return false for regular errors', () => { + const regularError = new Error('Regular error'); + assert.equal(hasSqlDetails(regularError), false); + }); + + it('should return false for non-Error objects', () => { + assert.equal(hasSqlDetails('string'), false); + assert.equal(hasSqlDetails(123), false); + assert.equal(hasSqlDetails(null), false); + assert.equal(hasSqlDetails(undefined), false); + assert.equal(hasSqlDetails({}), false); + }); + + it('should return false for Error with incomplete sqlDetails', () => { + const error = new Error('Test'); + error.sqlDetails = { message: 'test' }; // Missing cursorPosition + assert.equal(hasSqlDetails(error), false); + }); + }); + + describe('Edge Cases', () => { + it('should handle empty query', () => { + assert.throws(() => parseSync(''), { + message: 'Query cannot be empty' + }); + }); + + it('should handle null query', () => { + assert.throws(() => parseSync(null), { + message: 'Query cannot be null or undefined' + }); + }); + + it('should handle undefined query', () => { + assert.throws(() => parseSync(undefined), { + message: 'Query cannot be null or undefined' + }); + }); + + it('should handle @ in comments', () => { + const query = 'SELECT * FROM users /* @ in comment */ WHERE id = 1'; + assert.doesNotThrow(() => parseSync(query)); + }); + + it('should handle @ in strings', () => { + const query = 'SELECT * FROM users WHERE email = \'user@example.com\''; + assert.doesNotThrow(() => parseSync(query)); + }); + }); + + describe('Complex Error Scenarios', () => { + it('should handle errors in CASE statements', () => { + try { + parseSync('SELECT CASE WHEN id = 1 THEN "one" WHEN id = 2 THEN @ ELSE "other" END FROM users'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 54); + } + }); + + it('should handle errors in subqueries', () => { + try { + parseSync('SELECT * FROM users WHERE id IN (SELECT @ FROM orders)'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 42); + } + }); + + it('should handle errors in function calls', () => { + try { + parseSync('SELECT COUNT(@) FROM users'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 14); + } + }); + + it('should handle errors in second statement', () => { + try { + parseSync('SELECT * FROM users; SELECT * FROM orders WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 54); + } + }); + + it('should handle errors in CTE', () => { + try { + parseSync('WITH cte AS (SELECT * FROM users WHERE id = @) SELECT * FROM cte'); + assert.fail('Expected error'); + } catch (error) { + assert.equal(error.sqlDetails.cursorPosition, 45); + } + }); + }); + + describe('Backward Compatibility', () => { + it('should maintain Error instance', () => { + try { + parseSync('SELECT * FROM users WHERE id = @'); + assert.fail('Expected error'); + } catch (error) { + assert.ok(error instanceof Error); + assert.ok(error.message); + assert.ok(error.stack); + } + }); + + it('should work with standard error handling', () => { + let caught = false; + try { + parseSync('SELECT * FROM users WHERE id = @'); + } catch (e) { + caught = true; + assert.ok(e.message.includes('syntax error')); + } + assert.equal(caught, true); + }); + }); +}); \ No newline at end of file diff --git a/full/18/test/fingerprint.test.js b/full/18/test/fingerprint.test.js new file mode 100644 index 00000000..07171c82 --- /dev/null +++ b/full/18/test/fingerprint.test.js @@ -0,0 +1,67 @@ +const query = require("../"); +const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); + +describe("Query Fingerprinting", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Fingerprinting", () => { + it("should return a fingerprint for a simple query", () => { + const fingerprint = query.fingerprintSync("select 1"); + assert.equal(typeof fingerprint, "string"); + assert.equal(fingerprint.length, 16); + }); + + it("should return same fingerprint for equivalent queries", () => { + const fp1 = query.fingerprintSync("select 1"); + const fp2 = query.fingerprintSync("SELECT 1"); + const fp3 = query.fingerprintSync("select 1"); + + assert.equal(fp1, fp2); + assert.equal(fp1, fp3); + }); + + it("should return different fingerprints for different queries", () => { + const fp1 = query.fingerprintSync("select name from users"); + const fp2 = query.fingerprintSync("select id from customers"); + + assert.notEqual(fp1, fp2); + }); + + it("should normalize parameter values", () => { + const fp1 = query.fingerprintSync("select * from users where id = 123"); + const fp2 = query.fingerprintSync("select * from users where id = 456"); + + assert.equal(fp1, fp2); + }); + + it("should fail on invalid queries", () => { + assert.throws(() => query.fingerprintSync("NOT A QUERY"), Error); + }); + }); + + describe("Async Fingerprinting", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from users"; + const fpPromise = query.fingerprint(testQuery); + const fp = await fpPromise; + + assert.ok(fpPromise instanceof Promise); + assert.equal(fp, query.fingerprintSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.fingerprint("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/full/18/test/normalize.test.js b/full/18/test/normalize.test.js new file mode 100644 index 00000000..14f0a431 --- /dev/null +++ b/full/18/test/normalize.test.js @@ -0,0 +1,69 @@ +const query = require("../"); +const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); + +describe("Query Normalization", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Normalization", () => { + it("should normalize a simple query", () => { + const normalized = query.normalizeSync("select 1"); + assert.equal(typeof normalized, "string"); + assert.ok(normalized.includes("$1")); + }); + + it("should normalize parameter values", () => { + const normalized1 = query.normalizeSync("select * from users where id = 123"); + const normalized2 = query.normalizeSync("select * from users where id = 456"); + + assert.equal(normalized1, normalized2); + assert.ok(normalized1.includes("$1")); + }); + + it("should normalize string literals", () => { + const normalized1 = query.normalizeSync("select * from users where name = 'john'"); + const normalized2 = query.normalizeSync("select * from users where name = 'jane'"); + + assert.equal(normalized1, normalized2); + assert.ok(normalized1.includes("$1")); + }); + + it("should preserve query structure", () => { + const normalized = query.normalizeSync("SELECT id, name FROM users WHERE active = true ORDER BY name"); + + assert.ok(normalized.includes("SELECT")); + assert.ok(normalized.includes("FROM")); + assert.ok(normalized.includes("WHERE")); + assert.ok(normalized.includes("ORDER BY")); + }); + + it("should fail on invalid queries", () => { + assert.throws(() => query.normalizeSync("NOT A QUERY"), Error); + }); + }); + + describe("Async Normalization", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from users where id = 123"; + const normalizedPromise = query.normalize(testQuery); + const normalized = await normalizedPromise; + + assert.ok(normalizedPromise instanceof Promise); + assert.equal(normalized, query.normalizeSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.normalize("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/full/18/test/parsing.test.js b/full/18/test/parsing.test.js new file mode 100644 index 00000000..5477c6a0 --- /dev/null +++ b/full/18/test/parsing.test.js @@ -0,0 +1,84 @@ +const query = require("../"); +const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); + +function removeLocationProperties(obj) { + if (typeof obj !== 'object' || obj === null) { + return obj; + } + + if (Array.isArray(obj)) { + return obj.map(item => removeLocationProperties(item)); + } + + const result = {}; + for (const key in obj) { + if (obj.hasOwnProperty(key)) { + if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { + continue; // Skip location-related properties + } + result[key] = removeLocationProperties(obj[key]); + } + } + return result; +} + +describe("Query Parsing", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Parsing", () => { + it("should return a single-item parse result for common queries", () => { + const queries = ["select 1", "select null", "select ''", "select a, b"]; + const results = queries.map(query.parseSync); + results.forEach((res) => { + assert.equal(res.stmts.length, 1); + }); + + const selectedDatas = results.map( + (it) => it.stmts[0].stmt.SelectStmt.targetList + ); + + assert.equal(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival, 1); + assert.equal(selectedDatas[1][0].ResTarget.val.A_Const.isnull, true); + assert.equal(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval, ""); + assert.equal(selectedDatas[3].length, 2); + }); + + it("should support parsing multiple queries", () => { + const res = query.parseSync("select 1; select null;"); + assert.deepEqual(res.stmts.map(removeLocationProperties), [ + ...query.parseSync("select 1;").stmts.map(removeLocationProperties), + ...query.parseSync("select null;").stmts.map(removeLocationProperties), + ]); + }); + + it("should not parse a bogus query", () => { + assert.throws(() => query.parseSync("NOT A QUERY"), Error); + }); + }); + + describe("Async parsing", () => { + it("should return a promise resolving to same result", async () => { + const testQuery = "select * from john;"; + const resPromise = query.parse(testQuery); + const res = await resPromise; + + assert.ok(resPromise instanceof Promise); + assert.deepEqual(res, query.parseSync(testQuery)); + }); + + it("should reject on bogus queries", async () => { + return query.parse("NOT A QUERY").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/full/18/test/pg18.test.js b/full/18/test/pg18.test.js new file mode 100644 index 00000000..fb49bae0 --- /dev/null +++ b/full/18/test/pg18.test.js @@ -0,0 +1,69 @@ +const { describe, it, before } = require("node:test"); +const assert = require("node:assert"); +const { parse, parseSync, parsePlPgSQL, scan, loadModule } = require("../"); + +describe("PostgreSQL 18 syntax", () => { + before(async () => { + await loadModule(); + }); + + it("should report parser version 180004", async () => { + const result = await parse("SELECT 1"); + assert.equal(result.version, 180004); + }); + + it("should parse RETURNING OLD/NEW aliases", async () => { + const result = await parse( + "UPDATE t SET a = 1 RETURNING OLD.a AS oa, NEW.a AS na" + ); + const stmt = result.stmts[0].stmt.UpdateStmt; + const targets = stmt.returningClause.exprs.map( + (rt) => rt.ResTarget.name + ); + assert.deepEqual(targets, ["oa", "na"]); + }); + + it("should parse VIRTUAL generated columns", () => { + const result = parseSync( + "CREATE TABLE tt (a int, b int GENERATED ALWAYS AS (a * 2) VIRTUAL)" + ); + const columns = result.stmts[0].stmt.CreateStmt.tableElts; + const generated = columns[1].ColumnDef.constraints.find( + (c) => c.Constraint.contype === "CONSTR_GENERATED" + ); + assert.ok(generated); + assert.equal(generated.Constraint.generated_kind, "v"); + }); + + it("should parse ALTER CONSTRAINT ... NOT ENFORCED", async () => { + const result = await parse( + "ALTER TABLE t ALTER CONSTRAINT c NOT ENFORCED" + ); + const cmd = result.stmts[0].stmt.AlterTableStmt.cmds[0].AlterTableCmd; + assert.equal(cmd.subtype, "AT_AlterConstraint"); + assert.equal(cmd.def.ATAlterConstraint.conname, "c"); + assert.equal(cmd.def.ATAlterConstraint.alterEnforceability, true); + }); + + it("should parse PL/pgSQL on the 18 build", async () => { + const result = await parsePlPgSQL(` + CREATE FUNCTION f() RETURNS int AS $$ + BEGIN + RETURN 1; + END; + $$ LANGUAGE plpgsql; + `); + assert.ok(Array.isArray(result.plpgsql_funcs)); + assert.equal(result.plpgsql_funcs.length, 1); + }); + + it("should scan PG18-specific SQL", async () => { + const result = await scan( + "UPDATE t SET a = 1 RETURNING OLD.a AS oa, NEW.a AS na" + ); + assert.equal(result.version, 180004); + const texts = result.tokens.map((t) => t.text); + assert.ok(texts.includes("OLD")); + assert.ok(texts.includes("NEW")); + }); +}); diff --git a/full/18/test/plpgsql.test.js b/full/18/test/plpgsql.test.js new file mode 100644 index 00000000..e3235c83 --- /dev/null +++ b/full/18/test/plpgsql.test.js @@ -0,0 +1,75 @@ +const query = require("../"); +const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); + +describe("PL/pgSQL Parsing", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync PL/pgSQL Parsing", () => { + it("should parse a simple PL/pgSQL function", () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION test_func() + RETURNS INTEGER AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + `; + + const result = query.parsePlPgSQLSync(funcSql); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.plpgsql_funcs)); + }); + + it("should parse function with parameters", () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION add_numbers(a INTEGER, b INTEGER) + RETURNS INTEGER AS $$ + BEGIN + RETURN a + b; + END; + $$ LANGUAGE plpgsql; + `; + + const result = query.parsePlPgSQLSync(funcSql); + assert.ok(result.plpgsql_funcs.length > 0); + }); + + it("should fail on invalid PL/pgSQL", () => { + assert.throws(() => query.parsePlPgSQLSync("NOT A FUNCTION"), Error); + }); + }); + + describe("Async PL/pgSQL Parsing", () => { + it("should return a promise resolving to same result", async () => { + const funcSql = ` + CREATE OR REPLACE FUNCTION test_func() + RETURNS INTEGER AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + `; + + const resultPromise = query.parsePlPgSQL(funcSql); + const result = await resultPromise; + + assert.ok(resultPromise instanceof Promise); + assert.deepEqual(result, query.parsePlPgSQLSync(funcSql)); + }); + + it("should reject on invalid PL/pgSQL", async () => { + return query.parsePlPgSQL("NOT A FUNCTION").then( + () => { + throw new Error("should have rejected"); + }, + (e) => { + assert.ok(e instanceof Error); + assert.match(e.message, /NOT/); + } + ); + }); + }); +}); diff --git a/full/18/test/scan.test.js b/full/18/test/scan.test.js new file mode 100644 index 00000000..24d7b4cf --- /dev/null +++ b/full/18/test/scan.test.js @@ -0,0 +1,279 @@ +const query = require("../"); +const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); +const assert = require('node:assert/strict'); + +describe("Query Scanning", () => { + before(async () => { + await query.parse("SELECT 1"); + }); + + describe("Sync Scanning", () => { + it("should return a scan result with version and tokens", () => { + const result = query.scanSync("SELECT 1"); + + assert.equal(typeof result, "object"); + assert.ok("version" in result); + assert.ok("tokens" in result); + assert.equal(typeof result.version, "number"); + assert.ok(Array.isArray(result.tokens)); + }); + + it("should scan a simple SELECT query correctly", () => { + const result = query.scanSync("SELECT 1"); + + assert.equal(result.tokens.length, 2); + + // First token should be SELECT + const selectToken = result.tokens[0]; + assert.equal(selectToken.text, "SELECT"); + assert.equal(selectToken.start, 0); + assert.equal(selectToken.end, 6); + assert.equal(selectToken.tokenName, "UNKNOWN"); // SELECT is mapped as UNKNOWN in our simplified mapping + assert.equal(selectToken.keywordName, "RESERVED_KEYWORD"); + + // Second token should be 1 + const numberToken = result.tokens[1]; + assert.equal(numberToken.text, "1"); + assert.equal(numberToken.start, 7); + assert.equal(numberToken.end, 8); + assert.equal(numberToken.tokenName, "ICONST"); + assert.equal(numberToken.keywordName, "NO_KEYWORD"); + }); + + it("should scan tokens with correct positions", () => { + const sql = "SELECT * FROM users"; + const result = query.scanSync(sql); + + assert.equal(result.tokens.length, 4); + + // Verify each token position matches the original SQL + result.tokens.forEach(token => { + const actualText = sql.substring(token.start, token.end); + assert.equal(token.text, actualText); + }); + }); + + it("should identify different token types", () => { + const result = query.scanSync("SELECT 'string', 123, 3.14, $1 FROM users"); + + const tokenTypes = result.tokens.map(t => t.tokenName); + assert.ok(tokenTypes.includes("SCONST")); // String constant + assert.ok(tokenTypes.includes("ICONST")); // Integer constant + assert.ok(tokenTypes.includes("FCONST")); // Float constant + assert.ok(tokenTypes.includes("PARAM")); // Parameter marker + // Note: keywords like FROM may be tokenized as UNKNOWN in our simplified mapping + assert.ok(tokenTypes.includes("UNKNOWN")); // Keywords and identifiers + }); + + it("should identify operators and punctuation", () => { + const result = query.scanSync("SELECT * FROM users WHERE id = 1"); + + const operators = result.tokens.filter(t => + t.tokenName.startsWith("ASCII_") || t.text === "=" + ); + + assert.ok(operators.length > 0); + assert.equal(operators.some(t => t.text === "*"), true); + assert.equal(operators.some(t => t.text === "="), true); + }); + + it("should classify keyword types correctly", () => { + const result = query.scanSync("SELECT COUNT(*) FROM users WHERE active = true"); + + const reservedKeywords = result.tokens.filter(t => + t.keywordName === "RESERVED_KEYWORD" + ); + const unreservedKeywords = result.tokens.filter(t => + t.keywordName === "UNRESERVED_KEYWORD" + ); + + assert.ok(reservedKeywords.length > 0); + // SELECT, FROM, WHERE should be reserved keywords + assert.equal(reservedKeywords.some(t => t.text === "SELECT"), true); + assert.equal(reservedKeywords.some(t => t.text === "FROM"), true); + assert.equal(reservedKeywords.some(t => t.text === "WHERE"), true); + }); + + it("should handle complex queries with parameters", () => { + const result = query.scanSync("SELECT * FROM users WHERE id = $1 AND name = $2"); + + const params = result.tokens.filter(t => t.tokenName === "PARAM"); + assert.equal(params.length, 2); + assert.equal(params[0].text, "$1"); + assert.equal(params[1].text, "$2"); + }); + + it("should handle string escaping in JSON output", () => { + const result = query.scanSync("SELECT 'text with \"quotes\" and \\backslash'"); + + const stringToken = result.tokens.find(t => t.tokenName === "SCONST"); + assert.ok(stringToken); + assert.ok(stringToken.text.includes('"')); + assert.ok(stringToken.text.includes('\\')); + }); + + it("should scan INSERT statements", () => { + const result = query.scanSync("INSERT INTO table VALUES (1, 'text', 3.14)"); + + assert.equal(result.tokens.some(t => t.text === "INSERT"), true); + assert.equal(result.tokens.some(t => t.text === "INTO"), true); + assert.equal(result.tokens.some(t => t.text === "VALUES"), true); + assert.equal(result.tokens.some(t => t.tokenName === "ICONST"), true); + assert.equal(result.tokens.some(t => t.tokenName === "SCONST"), true); + assert.equal(result.tokens.some(t => t.tokenName === "FCONST"), true); + }); + + it("should scan UPDATE statements", () => { + const result = query.scanSync("UPDATE users SET name = 'John' WHERE id = 1"); + + assert.equal(result.tokens.some(t => t.text === "UPDATE"), true); + assert.equal(result.tokens.some(t => t.text === "SET"), true); + assert.equal(result.tokens.some(t => t.text === "="), true); + }); + + it("should scan DELETE statements", () => { + const result = query.scanSync("DELETE FROM users WHERE active = false"); + + assert.equal(result.tokens.some(t => t.text === "DELETE"), true); + assert.equal(result.tokens.some(t => t.text === "FROM"), true); + assert.equal(result.tokens.some(t => t.text === "WHERE"), true); + }); + + it("should handle empty or whitespace-only input", () => { + const result = query.scanSync(" "); + assert.equal(result.tokens.length, 0); + }); + + it("should handle unusual input gracefully", () => { + // The scanner is more permissive than the parser and may tokenize unusual input + const result = query.scanSync("$$$INVALID$$$"); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.tokens)); + // Scanner may still produce tokens even for unusual input + }); + + it("should preserve original token order", () => { + const sql = "SELECT id, name FROM users ORDER BY name"; + const result = query.scanSync(sql); + + // Tokens should be in order of appearance + for (let i = 1; i < result.tokens.length; i++) { + assert.ok(result.tokens[i].start >= result.tokens[i-1].end); + } + }); + }); + + describe("Async Scanning", () => { + it("should return a promise resolving to same result as sync", async () => { + const testQuery = "SELECT * FROM users WHERE id = $1"; + const resultPromise = query.scan(testQuery); + const result = await resultPromise; + + assert.ok(resultPromise instanceof Promise); + assert.deepEqual(result, query.scanSync(testQuery)); + }); + + it("should handle complex queries asynchronously", async () => { + const testQuery = "SELECT COUNT(*) as total FROM orders WHERE status = 'completed' AND created_at > '2023-01-01'"; + const result = await query.scan(testQuery); + + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.tokens)); + assert.ok(result.tokens.length > 10); + }); + + it("should handle unusual input asynchronously", async () => { + // Scanner is more permissive than parser + const result = await query.scan("$$$INVALID$$$"); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.tokens)); + }); + }); + + describe("Edge Cases", () => { + it("should handle queries with comments", () => { + const result = query.scanSync("SELECT 1 -- this is a comment"); + + // Should have at least SELECT and 1 tokens + assert.ok(result.tokens.length >= 2); + assert.equal(result.tokens.some(t => t.text === "SELECT"), true); + assert.equal(result.tokens.some(t => t.text === "1"), true); + }); + + it("should handle very long identifiers", () => { + const longIdentifier = "a".repeat(100); + const result = query.scanSync(`SELECT ${longIdentifier} FROM table`); + + const identToken = result.tokens.find(t => t.text === longIdentifier); + assert.ok(identToken); + assert.equal(identToken.tokenName, "IDENT"); + }); + + it("should handle special PostgreSQL operators", () => { + const result = query.scanSync("SELECT id::text FROM users"); + + assert.equal(result.tokens.some(t => t.text === "::"), true); + const typecastToken = result.tokens.find(t => t.text === "::"); + assert.equal(typecastToken?.tokenName, "TYPECAST"); + }); + + it("should provide consistent version information", () => { + const result1 = query.scanSync("SELECT 1"); + const result2 = query.scanSync("INSERT INTO table VALUES (1)"); + + assert.equal(result1.version, result2.version); + assert.equal(typeof result1.version, "number"); + assert.ok(result1.version > 0); + }); + + it("should handle multi-line dollar-quoted strings without JSON errors", () => { + // Without the fix, scanSync throws: + // "Bad control character in string literal" + // because build_scan_json() doesn't escape \n in the token text. + const sql = `CREATE FUNCTION test() RETURNS void AS $$ +BEGIN + RAISE NOTICE 'hello'; +END; +$$ LANGUAGE plpgsql`; + + const result = query.scanSync(sql); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.tokens)); + assert.ok(result.tokens.length > 0); + + // The dollar-quoted body spans multiple lines + const dollarToken = result.tokens.find(t => t.text.includes('BEGIN')); + assert.ok(dollarToken, "should have a token containing the function body"); + assert.ok(dollarToken.text.includes('\n'), "token text should preserve newlines"); + }); + + it("should handle dollar-quoted tokens with tabs", () => { + // Tab characters also break JSON.parse when unescaped. + const sql = `SELECT $$line1 + indented +line3$$`; + + const result = query.scanSync(sql); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.tokens)); + + const dollarToken = result.tokens.find(t => t.text.includes('indented')); + assert.ok(dollarToken, "should have a token containing the tabbed content"); + }); + + it("should handle multi-line block comments", () => { + // C-style block comments spanning multiple lines hit the same bug. + const sql = `SELECT 1; /* multi +line +comment */ SELECT 2`; + + const result = query.scanSync(sql); + assert.equal(typeof result, "object"); + assert.ok(Array.isArray(result.tokens)); + + const commentToken = result.tokens.find(t => t.tokenName === "C_COMMENT"); + assert.ok(commentToken, "should have a C_COMMENT token"); + assert.ok(commentToken.text.includes('\n'), "comment text should preserve newlines"); + }); + }); +}); diff --git a/full/18/tsconfig.esm.json b/full/18/tsconfig.esm.json new file mode 100644 index 00000000..63012c17 --- /dev/null +++ b/full/18/tsconfig.esm.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "module": "es2022", + "rootDir": "src/", + "declaration": false, + "outDir": "esm/" + } + } \ No newline at end of file diff --git a/full/18/tsconfig.json b/full/18/tsconfig.json new file mode 100644 index 00000000..d7da207b --- /dev/null +++ b/full/18/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "commonjs", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "strictNullChecks": false, + "skipLibCheck": true, + "sourceMap": false, + "declaration": true, + "resolveJsonModule": true, + "moduleResolution": "node", + "outDir": "cjs/", + "rootDir": "src" + }, + "exclude": ["cjs", "esm", "wasm", "node_modules", "**/*.test.ts"] +} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 712b71b5..f2564356 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,7 +72,7 @@ importers: version: 1.28.2 publishDirectory: dist - full: + full/17: dependencies: '@pgsql/types': specifier: ^17.6.2 @@ -82,6 +82,16 @@ importers: specifier: ^0.8.0 version: 0.8.0 + full/18: + dependencies: + '@pgsql/types': + specifier: ^18.0.0 + version: 18.0.0 + devDependencies: + '@yamlize/cli': + specifier: ^0.8.0 + version: 0.8.0 + parser: devDependencies: cross-env: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 71353548..0a36f3f1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,7 @@ packages: - 'parser' - - 'full' + - 'full/18' + - 'full/17' - 'versions/18' - 'versions/17' - 'versions/16' diff --git a/scripts/analyze-sizes.js b/scripts/analyze-sizes.js index 674d8642..959cb7b5 100644 --- a/scripts/analyze-sizes.js +++ b/scripts/analyze-sizes.js @@ -75,7 +75,8 @@ function analyzePackage(packagePath, packageName) { function getVersionPackages() { const versionsDir = './versions'; const packages = [ - { path: './full', name: 'full (Full)', version: 'original' } + { path: './full/17', name: 'full/17 (Full)', version: 'original' }, + { path: './full/18', name: 'full/18 (Full)', version: 'original' } ]; if (fs.existsSync(versionsDir)) { diff --git a/scripts/publish-versions.js b/scripts/publish-versions.js index ed9d8e0b..1db54967 100755 --- a/scripts/publish-versions.js +++ b/scripts/publish-versions.js @@ -29,25 +29,27 @@ async function main() { .filter(dir => /^\d+$/.test(dir)) .sort((a, b) => parseInt(b) - parseInt(a)); // Sort descending - // Also check for full package - const fullPackagePath = path.join(__dirname, '..', 'full', 'package.json'); - const hasFullPackage = fs.existsSync(fullPackagePath); + // Also check for full package versions (full/17, full/18, ...) + const fullDir = path.join(__dirname, '..', 'full'); + const fullVersionDirs = fs.existsSync(fullDir) + ? fs.readdirSync(fullDir) + .filter(dir => /^\d+$/.test(dir) && fs.existsSync(path.join(fullDir, dir, 'package.json'))) + .sort((a, b) => parseInt(b) - parseInt(a)) // Sort descending + : []; console.log('πŸ“¦ Available packages:'); versionDirs.forEach(v => console.log(` - PostgreSQL ${v} (versions/${v})`)); - if (hasFullPackage) { - console.log(` - Full package (./full) - PostgreSQL 17`); - } + fullVersionDirs.forEach(v => console.log(` - Full package PostgreSQL ${v} (full/${v})`)); console.log(); // Ask which versions to publish const publishAll = await question('Publish all packages? (y/N): '); let selectedVersions = []; - let includeFullPackage = false; + let selectedFullVersions = []; if (publishAll.toLowerCase() === 'y') { selectedVersions = versionDirs; - includeFullPackage = hasFullPackage; + selectedFullVersions = fullVersionDirs; } else { // Let user select versions for (const version of versionDirs) { @@ -57,13 +59,15 @@ async function main() { } } - if (hasFullPackage) { - const publishFull = await question(`Publish full package (PostgreSQL 17)? (y/N): `); - includeFullPackage = publishFull.toLowerCase() === 'y'; + for (const version of fullVersionDirs) { + const publishFull = await question(`Publish full package PostgreSQL ${version} (full/${version})? (y/N): `); + if (publishFull.toLowerCase() === 'y') { + selectedFullVersions.push(version); + } } } - if (selectedVersions.length === 0 && !includeFullPackage) { + if (selectedVersions.length === 0 && selectedFullVersions.length === 0) { console.log('\n❌ No packages selected for publishing.'); rl.close(); return; @@ -78,9 +82,7 @@ async function main() { console.log(`\nπŸ“‹ Will publish:`); selectedVersions.forEach(v => console.log(` - PostgreSQL ${v} (${bump} bump)`)); - if (includeFullPackage) { - console.log(` - Full package (${bump} bump)`); - } + selectedFullVersions.forEach(v => console.log(` - Full package PostgreSQL ${v} (${bump} bump)`)); // Ask about building const skipBuild = await question('\nSkip build step? (y/N): '); @@ -141,10 +143,10 @@ async function main() { } } - // Process full package if selected - if (includeFullPackage) { - console.log(`\nπŸ“¦ Publishing full package...`); - const fullPath = path.join(__dirname, '..', 'full'); + // Process full package versions if selected + for (const version of selectedFullVersions) { + console.log(`\nπŸ“¦ Publishing full package PostgreSQL ${version}...`); + const fullPath = path.join(fullDir, version); try { // Version bump @@ -154,7 +156,7 @@ async function main() { // Commit console.log(` πŸ’Ύ Committing version bump...`); execSync(`git add package.json`, { cwd: fullPath }); - execSync(`git commit -m "release: bump @libpg-query/parser version"`, { stdio: 'inherit' }); + execSync(`git commit -m "release: bump @libpg-query/parser v${version} version"`, { stdio: 'inherit' }); // Build (if not skipped) if (shouldBuild) { @@ -168,19 +170,23 @@ async function main() { console.log(` πŸ§ͺ Running tests...`); execSync('pnpm test', { cwd: fullPath, stdio: 'inherit' }); - // Publish with pg17 tag - console.log(` πŸ“€ Publishing to npm with pg17 tag...`); - // use npm so staged changes are OK - execSync('npm publish --tag pg17', { cwd: fullPath, stdio: 'inherit' }); + // Publish with the pg tag (uses x-publish metadata) + console.log(` πŸ“€ Publishing to npm with pg${version} tag...`); + execSync('pnpm run publish:pkg', { cwd: fullPath, stdio: 'inherit' }); - console.log(` βœ… Full package published successfully with pg17 tag!`); + console.log(` βœ… Full package published successfully with pg${version} tag!`); } catch (error) { - console.error(` ❌ Failed to publish full package:`, error.message); + console.error(` ❌ Failed to publish full package PostgreSQL ${version}:`, error.message); + const continuePublish = await question('Continue with other versions? (y/N): '); + if (continuePublish.toLowerCase() !== 'y') { + rl.close(); + process.exit(1); + } } } // Ask about promoting to latest - if (selectedVersions.includes('17') || includeFullPackage) { + if (selectedVersions.includes('17') || selectedFullVersions.includes('17')) { console.log('\n🏷️ Tag Management'); if (selectedVersions.includes('17')) { @@ -195,7 +201,7 @@ async function main() { } } - if (includeFullPackage) { + if (selectedFullVersions.includes('17')) { const promoteFullPackage = await question('Promote @libpg-query/parser@pg17 to latest? (y/N): '); if (promoteFullPackage.toLowerCase() === 'y') { try { From c86b84ed83606de1b948b08de5de7e076f83600e Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Tue, 21 Jul 2026 09:15:00 +0000 Subject: [PATCH 2/2] =?UTF-8?q?feat(v18)!:=20retire=20full/=20=E2=80=94=20?= =?UTF-8?q?libpg-query=20pg18+=20ships=20the=20full=20API=20(13-17=20stay?= =?UTF-8?q?=20slim)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-wasm-no-docker.yaml | 6 +- .github/workflows/build-wasm.yml | 21 +- .github/workflows/ci.yml | 4 - PUBLISH.md | 38 +- README.md | 8 +- REPO_NOTES.md | 8 + full/.npmignore | 18 - full/17/CHANGELOG.md | 38 -- full/17/Makefile | 96 --- full/17/README.md | 358 ----------- full/17/libpg_query/protobuf/.gitkeep | 0 full/17/package.json | 55 -- full/17/scripts/build.js | 38 -- full/17/src/proto.d.ts | 20 - full/17/test/errors.test.js | 325 ---------- full/17/test/parsing.test.js | 84 --- full/17/tsconfig.esm.json | 9 - full/17/tsconfig.json | 18 - full/18/CHANGELOG.md | 44 -- full/18/Makefile | 96 --- full/18/README.md | 358 ----------- full/18/SCAN.md | 439 -------------- full/18/libpg_query.md | 558 ------------------ full/18/libpg_query/protobuf/.gitkeep | 0 full/18/package.json | 55 -- full/18/scripts/build.js | 38 -- full/18/src/index.ts | 484 --------------- full/18/src/libpg-query.d.ts | 17 - full/18/src/proto.d.ts | 20 - full/18/src/wasm_wrapper.c | 395 ------------- full/18/test/errors.test.js | 325 ---------- full/18/test/fingerprint.test.js | 67 --- full/18/test/normalize.test.js | 69 --- full/18/test/parsing.test.js | 84 --- full/18/test/plpgsql.test.js | 75 --- full/18/test/scan.test.js | 279 --------- full/18/tsconfig.esm.json | 9 - full/18/tsconfig.json | 18 - pnpm-lock.yaml | 97 --- pnpm-workspace.yaml | 2 - scripts/analyze-sizes.js | 5 +- scripts/copy-templates.js | 28 +- scripts/publish-versions.js | 94 +-- templates/Makefile.template | 2 +- {full/17/src => templates/full}/index.ts | 0 .../src => templates/full}/libpg-query.d.ts | 0 .../17/src => templates/full}/wasm_wrapper.c | 0 versions/18/Makefile | 2 +- versions/18/README.md | 55 +- {full/17 => versions/18}/SCAN.md | 0 {full/17 => versions/18}/libpg_query.md | 0 versions/18/package.json | 5 +- versions/18/src/index.ts | 442 +++++++++----- versions/18/src/libpg-query.d.ts | 4 +- versions/18/src/wasm_wrapper.c | 367 +++++++++++- .../18}/test/fingerprint.test.js | 0 .../17 => versions/18}/test/normalize.test.js | 0 {full => versions}/18/test/pg18.test.js | 0 {full/17 => versions/18}/test/plpgsql.test.js | 0 {full/17 => versions/18}/test/scan.test.js | 0 60 files changed, 788 insertions(+), 4889 deletions(-) delete mode 100644 full/.npmignore delete mode 100644 full/17/CHANGELOG.md delete mode 100644 full/17/Makefile delete mode 100644 full/17/README.md delete mode 100644 full/17/libpg_query/protobuf/.gitkeep delete mode 100644 full/17/package.json delete mode 100644 full/17/scripts/build.js delete mode 100644 full/17/src/proto.d.ts delete mode 100644 full/17/test/errors.test.js delete mode 100644 full/17/test/parsing.test.js delete mode 100644 full/17/tsconfig.esm.json delete mode 100644 full/17/tsconfig.json delete mode 100644 full/18/CHANGELOG.md delete mode 100644 full/18/Makefile delete mode 100644 full/18/README.md delete mode 100644 full/18/SCAN.md delete mode 100644 full/18/libpg_query.md delete mode 100644 full/18/libpg_query/protobuf/.gitkeep delete mode 100644 full/18/package.json delete mode 100644 full/18/scripts/build.js delete mode 100644 full/18/src/index.ts delete mode 100644 full/18/src/libpg-query.d.ts delete mode 100644 full/18/src/proto.d.ts delete mode 100644 full/18/src/wasm_wrapper.c delete mode 100644 full/18/test/errors.test.js delete mode 100644 full/18/test/fingerprint.test.js delete mode 100644 full/18/test/normalize.test.js delete mode 100644 full/18/test/parsing.test.js delete mode 100644 full/18/test/plpgsql.test.js delete mode 100644 full/18/test/scan.test.js delete mode 100644 full/18/tsconfig.esm.json delete mode 100644 full/18/tsconfig.json rename {full/17/src => templates/full}/index.ts (100%) rename {full/17/src => templates/full}/libpg-query.d.ts (100%) rename {full/17/src => templates/full}/wasm_wrapper.c (100%) rename {full/17 => versions/18}/SCAN.md (100%) rename {full/17 => versions/18}/libpg_query.md (100%) rename {full/17 => versions/18}/test/fingerprint.test.js (100%) rename {full/17 => versions/18}/test/normalize.test.js (100%) rename {full => versions}/18/test/pg18.test.js (100%) rename {full/17 => versions/18}/test/plpgsql.test.js (100%) rename {full/17 => versions/18}/test/scan.test.js (100%) diff --git a/.github/workflows/build-wasm-no-docker.yaml b/.github/workflows/build-wasm-no-docker.yaml index 0e6b1c3f..4cbceebb 100644 --- a/.github/workflows/build-wasm-no-docker.yaml +++ b/.github/workflows/build-wasm-no-docker.yaml @@ -42,15 +42,15 @@ jobs: ./emsdk install 3.1.59 ./emsdk activate 3.1.59 source ./emsdk_env.sh - working-directory: full/18 + working-directory: versions/18 - name: Build with Emscripten πŸ— run: | source ./emsdk/emsdk_env.sh emmake make emmake make build - working-directory: full/18 + working-directory: versions/18 - name: Archive production artifacts πŸ› uses: actions/upload-artifact@v4 with: name: wasm-artifacts - path: full/18/wasm + path: versions/18/wasm diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 962873e1..54d3bd4d 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -36,24 +36,13 @@ jobs: - name: Install Dependencies 🧢 run: pnpm install - - name: Build WASM (PG17) πŸ— + - name: Build WASM πŸ— run: pnpm run build - working-directory: full/17 + working-directory: versions/18 - - name: Build WASM (PG18) πŸ— - run: pnpm run build - working-directory: full/18 - - - name: Archive PG17 artifacts πŸ› - uses: actions/upload-artifact@v4 - with: - name: wasm-artifacts-pg17 - path: full/17/wasm/ - retention-days: 7 - - - name: Archive PG18 artifacts πŸ› + - name: Archive production artifacts πŸ› uses: actions/upload-artifact@v4 with: - name: wasm-artifacts-pg18 - path: full/18/wasm/ + name: wasm-artifacts + path: versions/18/wasm/ retention-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eacc9905..67b79d5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,6 @@ jobs: strategy: matrix: package: - - { name: 'full17', path: 'full/17', version: '17' } - - { name: 'full18', path: 'full/18', version: '18' } - { name: 'v13', path: 'versions/13', version: '13' } - { name: 'v14', path: 'versions/14', version: '14' } - { name: 'v15', path: 'versions/15', version: '15' } @@ -79,8 +77,6 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] package: - - { name: 'full17', path: 'full/17', version: '17' } - - { name: 'full18', path: 'full/18', version: '18' } - { name: 'v13', path: 'versions/13', version: '13' } - { name: 'v14', path: 'versions/14', version: '14' } - { name: 'v15', path: 'versions/15', version: '15' } diff --git a/PUBLISH.md b/PUBLISH.md index 5faa7de6..e86e4703 100644 --- a/PUBLISH.md +++ b/PUBLISH.md @@ -10,7 +10,6 @@ pnpm run publish:versions This interactive script will: - Check for uncommitted changes (will error if any exist) - Let you select which versions to publish (or all) -- Also includes the full package versions (@libpg-query/parser, full/17 and full/18) - Ask for version bump type (patch or minor only) - Ask if you want to skip the build step (useful if already built) - Always run tests (even if build is skipped) @@ -163,39 +162,16 @@ npm install libpg-query@pg16 # PostgreSQL 16 specific npm install libpg-query # Latest/default version ``` -## Full Package (@libpg-query/parser) +## Full API on PG 18+ -The full package is versioned per PostgreSQL major version, mirroring `versions/`: -- `full/17` β€” built from `17-constructive`, published as `@libpg-query/parser` with tag `pg17` -- `full/18` β€” built from `18-constructive`, published as `@libpg-query/parser` with tag `pg18` +The former `full/` package (`@libpg-query/parser`) has been retired. Starting with +PostgreSQL 18, the regular `libpg-query` package (`versions/18`) ships the full API: +`parse`, `parsePlPgSQL`, `scan`, `fingerprint`, `normalize` + sync variants. +Versions 13–17 remain slim (parse only). -### Quick Publish (per version) ```bash -cd full/17 # or full/18 -pnpm version patch -git add . && git commit -m "release: bump @libpg-query/parser version" -pnpm build -pnpm test -pnpm run publish:pkg -``` - -`publish:pkg` uses `x-publish.publishName` (`@libpg-query/parser`) and `x-publish.distTag` -(`pg17` / `pg18`) from each package.json, temporarily renaming the package during publish. - -### Promote to latest (optional) -```bash -npm dist-tag add @libpg-query/parser@pg17 latest -``` - -### What it does -- Publishes `@libpg-query/parser` with tag `pg17` (from `full/17`) or `pg18` (from `full/18`) -- Includes full parser with all features (parse, parsePlPgSQL, scan, fingerprint, normalize + sync variants) - -### Install published package -```bash -npm install @libpg-query/parser@pg18 # PostgreSQL 18 specific -npm install @libpg-query/parser@pg17 # PostgreSQL 17 specific -npm install @libpg-query/parser # Latest version +npm install libpg-query@pg18 # full API +npm install libpg-query@pg17 # parse only ``` ## Parser Package (@pgsql/parser) diff --git a/README.md b/README.md index 5fed1572..072f72b8 100644 --- a/README.md +++ b/README.md @@ -73,11 +73,10 @@ This repository contains multiple packages to support different PostgreSQL versi | Package | Description | PostgreSQL Versions | npm Package | |---------|-------------|---------------------|-------------| -| **[libpg-query](https://github.com/constructive-io/libpg-query-node/tree/main/versions)** | Lightweight parser (parse only) | 13, 14, 15, 16, 17, 18 | [`libpg-query`](https://www.npmjs.com/package/libpg-query) | +| **[libpg-query](https://github.com/constructive-io/libpg-query-node/tree/main/versions)** | PostgreSQL parser (full API on PG 18+, parse-only on 13–17) | 13, 14, 15, 16, 17, 18 | [`libpg-query`](https://www.npmjs.com/package/libpg-query) | | **[@pgsql/parser](https://github.com/constructive-io/libpg-query-node/tree/main/parser)** | Multi-version parser (runtime selection) | 15, 16, 17, 18 | [`@pgsql/parser`](https://www.npmjs.com/package/@pgsql/parser) | | **[@pgsql/types](https://github.com/constructive-io/libpg-query-node/tree/main/types)** | TypeScript type definitions | 13, 14, 15, 16, 17, 18 | [`@pgsql/types`](https://www.npmjs.com/package/@pgsql/types) | | **[@pgsql/enums](https://github.com/constructive-io/libpg-query-node/tree/main/enums)** | TypeScript enum definitions | 13, 14, 15, 16, 17, 18 | [`@pgsql/enums`](https://www.npmjs.com/package/@pgsql/enums) | -| **[@libpg-query/parser](https://github.com/constructive-io/libpg-query-node/tree/main/full)** | Full parser with all features | 17, 18 | [`@libpg-query/parser`](https://www.npmjs.com/package/@libpg-query/parser) | ### Version Tags @@ -99,10 +98,10 @@ npm install @pgsql/enums ### Which Package Should I Use? -- **Just need to parse SQL?** β†’ Use `libpg-query` (lightweight, all PG versions) +- **Just need to parse SQL?** β†’ Use `libpg-query` (all PG versions) - **Need multiple versions at runtime?** β†’ Use `@pgsql/parser` (dynamic version selection) - **Need TypeScript types?** β†’ Add `@pgsql/types` and/or `@pgsql/enums` -- **Need fingerprint, normalize, scan, or PL/pgSQL parsing?** β†’ Use `@libpg-query/parser` (PG 17 & 18 via `pg17`/`pg18` dist-tags) +- **Need fingerprint, normalize, scan, or PL/pgSQL parsing?** β†’ Use `libpg-query@pg18` β€” the full API ships on PG 18+ (13–17 remain parse-only) ## API Documentation @@ -113,7 +112,6 @@ For detailed API documentation and usage examples, see the package-specific READ - **@pgsql/parser** - [Multi-Version Parser Documentation](https://github.com/constructive-io/libpg-query-node/tree/main/parser) - **@pgsql/types** - [Types Documentation](https://github.com/constructive-io/libpg-query-node/tree/main/types/18) - **@pgsql/enums** - [Enums Documentation](https://github.com/constructive-io/libpg-query-node/tree/main/enums/18) -- **@libpg-query/parser** - [Full Parser Documentation](https://github.com/constructive-io/libpg-query-node/tree/main/full) ## Build Instructions diff --git a/REPO_NOTES.md b/REPO_NOTES.md index dd3fa3a2..4ba54e9e 100644 --- a/REPO_NOTES.md +++ b/REPO_NOTES.md @@ -2,6 +2,14 @@ There is a templates/ dir to solve some of this. +Versions with `x-publish.fullApi: true` (currently 18) are generated from the +full-API templates in `templates/full/` and export the full API +(parse, parsePlPgSQL, scan, fingerprint, normalize + sync variants); +the other versions use the slim (parse-only) templates. +⚠️ Note: the Makefiles for 15/17/18 have been hand-edited to point at +constructive-io/libpg_query branches; `copy:templates` regenerates Makefiles +with the pganalyze repo URL, so re-apply the repo override if you regenerate. + ## Code Duplication πŸ“‹ ### 1. Identical Test Files diff --git a/full/.npmignore b/full/.npmignore deleted file mode 100644 index 5099528c..00000000 --- a/full/.npmignore +++ /dev/null @@ -1,18 +0,0 @@ -# project files -test -.gitignore -package.json -build - -*.log -npm-debug.log* - -# Dependency directories -node_modules - -libpg_query/* - -# npm package lock -package-lock.json -yarn.lock - diff --git a/full/17/CHANGELOG.md b/full/17/CHANGELOG.md deleted file mode 100644 index 1e26c64c..00000000 --- a/full/17/CHANGELOG.md +++ /dev/null @@ -1,38 +0,0 @@ -# 17.2.1 -* Add normalize() and normalizeSync() functions for SQL normalization -* Add fingerprint() and fingerprintSync() functions for query fingerprinting -* Add isReady() function to check WASM module initialization status -* Improve memory management and error handling in WASM wrapper -* Remove unused dependencies (lodash, deasync, @launchql/protobufjs) -* Split test suite into separate files for better organization -* Update documentation with comprehensive function examples - -# 1.2.1 -* Rename package - -# 1.2.0 -* Add TS typings - -# 1.1.2 (07/23/19) -* Fix build script - -# 1.1.0 (Republish, 07/23/19) -* Rewrite to use Node-Addons-API (C++ wrapper for the stable N-API) -* Support PL/PGSql Parsing -* Support async parsing - -## 0.0.5 (November 19, 2015) -* Update libpg_query to include latest A_CONST fixes - -## 0.0.4 (November 16, 2015) -* Update libpg_query to include fix for empty string constants -* Warning fixed in 0.0.3 is back for now until libpg_query has support for custom CFLAGS - -## 0.0.3 (November 14, 2015) -* Update OSX static library to include `-mmacosx-version-min=10.5` to fix warnings on installation - -## 0.0.2 (November 14, 2015) -* Rename repo to pg-query-native - -## 0.0.1 (November 14, 2015) -* First version diff --git a/full/17/Makefile b/full/17/Makefile deleted file mode 100644 index e3284679..00000000 --- a/full/17/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -WASM_OUT_DIR := wasm -WASM_OUT_NAME := libpg-query -WASM_MODULE_NAME := PgQueryModule -# LIBPG_QUERY_REPO := https://github.com/pganalyze/libpg_query.git -# LIBPG_QUERY_TAG := 17-6.1.0 -LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git -LIBPG_QUERY_TAG := 17-constructive - -CACHE_DIR := .cache - -OS ?= $(shell uname -s) -ARCH ?= $(shell uname -m) - -ifdef EMSCRIPTEN -PLATFORM := emscripten -else ifeq ($(OS),Darwin) -PLATFORM := darwin -else ifeq ($(OS),Linux) -PLATFORM := linux -else -$(error Unsupported platform: $(OS)) -endif - -ifdef EMSCRIPTEN -ARCH := wasm -endif - -PLATFORM_ARCH := $(PLATFORM)-$(ARCH) -SRC_FILES := src/wasm_wrapper.c -LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_TAG) -LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a -LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h -CXXFLAGS := -O3 - -ifdef EMSCRIPTEN -OUT_FILES := $(foreach EXT,.js .wasm,$(WASM_OUT_DIR)/$(WASM_OUT_NAME)$(EXT)) -else -$(error Native builds are no longer supported. Use EMSCRIPTEN=1 for WASM builds only.) -endif - -# Clone libpg_query source (lives in CACHE_DIR) -$(LIBPG_QUERY_DIR): - mkdir -p $(CACHE_DIR) - git clone -b $(LIBPG_QUERY_TAG) --single-branch $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) - -$(LIBPG_QUERY_HEADER): $(LIBPG_QUERY_DIR) - -# Build libpg_query -$(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR) - cd $(LIBPG_QUERY_DIR); $(MAKE) build - -# Build libpg-query-node WASM module -$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) -ifdef EMSCRIPTEN - mkdir -p $(WASM_OUT_DIR) - $(CC) \ - -v \ - $(CXXFLAGS) \ - -I$(LIBPG_QUERY_DIR) \ - -I$(LIBPG_QUERY_DIR)/vendor \ - -L$(LIBPG_QUERY_DIR) \ - -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','UTF8ToString','getValue','HEAPU8','HEAPU32']" \ - -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ - -sENVIRONMENT="web,node,worker" \ - -sASSERTIONS=0 \ - -sSINGLE_FILE=0 \ - -sMODULARIZE=1 \ - -sEXPORT_ES6=0 \ - -sALLOW_MEMORY_GROWTH=1 \ - -sINITIAL_MEMORY=134217728 \ - -sMAXIMUM_MEMORY=1073741824 \ - -sSTACK_SIZE=33554432 \ - -lpg_query \ - -o $@ \ - $(SRC_FILES) -else -$(error Native builds are no longer supported. Use EMSCRIPTEN=1 for WASM builds only.) -endif - -# Commands -build: $(OUT_FILES) - -build-cache: $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) - -rebuild: clean build - -rebuild-cache: clean-cache build-cache - -clean: - -@ rm -r $(OUT_FILES) > /dev/null 2>&1 - -clean-cache: - -@ rm -rf $(LIBPG_QUERY_DIR) - -.PHONY: build build-cache rebuild rebuild-cache clean clean-cache diff --git a/full/17/README.md b/full/17/README.md deleted file mode 100644 index 040a55aa..00000000 --- a/full/17/README.md +++ /dev/null @@ -1,358 +0,0 @@ -# @libpg-query/parser - -

- constructive.io -

- -

- - - -
- - - - -

- -# The Real PostgreSQL Parser for JavaScript - -### Bring the power of PostgreSQL’s native parser to your JavaScript projects β€” no native builds, no platform headaches. - -This is the official PostgreSQL parser, compiled to WebAssembly (WASM) for seamless, cross-platform compatibility. Use it in Node.js or the browser, on Linux, Windows, or anywhere JavaScript runs. - -Built to power [pgsql-parser](https://github.com/constructive-io/pgsql-parser), this library delivers full fidelity with the Postgres C codebase β€” no rewrites, no shortcuts. - -### Features - -* πŸ”§ **Powered by PostgreSQL** – Uses the official Postgres C parser compiled to WebAssembly -* πŸ–₯️ **Cross-Platform** – Runs smoothly on macOS, Linux, and Windows -* 🌐 **Node.js & Browser Support** – Consistent behavior in any JS environment -* πŸ“¦ **No Native Builds Required** – No compilation, no system-specific dependencies -* 🧠 **Spec-Accurate Parsing** – Produces faithful, standards-compliant ASTs -* πŸš€ **Production-Grade** – Millions of downloads and trusted by countless projects and top teams - -## πŸš€ For Round-trip Codegen - -> 🎯 **Want to parse + deparse (full round trip)?** -> We highly recommend using [`pgsql-parser`](https://github.com/constructive-io/pgsql-parser) which leverages a pure TypeScript deparser that has been battle-tested against 23,000+ SQL statements and is built on top of libpg-query. - -## Installation - -```sh -npm install @libpg-query/parser -``` - -## Usage - -### `parse(query: string): Promise` - -Parses the SQL and returns a Promise for the parse tree. May reject with a parse error. - -```typescript -import { parse } from '@libpg-query/parser'; - -const result = await parse('SELECT * FROM users WHERE active = true'); -// Returns: ParseResult - parsed query object -``` - -### `parseSync(query: string): ParseResult` - -Synchronous version that returns the parse tree directly. May throw a parse error. - -```typescript -import { parseSync } from '@libpg-query/parser'; - -const result = parseSync('SELECT * FROM users WHERE active = true'); -// Returns: ParseResult - parsed query object -``` - -### `parsePlPgSQL(funcsSql: string): Promise` - -Parses the contents of a PL/pgSQL function from a `CREATE FUNCTION` declaration. Returns a Promise for the parse tree. - -```typescript -import { parsePlPgSQL } from '@libpg-query/parser'; - -const functionSql = ` -CREATE FUNCTION get_user_count() RETURNS integer AS $$ -BEGIN - RETURN (SELECT COUNT(*) FROM users); -END; -$$ LANGUAGE plpgsql; -`; - -const result = await parsePlPgSQL(functionSql); -``` - -### `parsePlPgSQLSync(funcsSql: string): ParseResult` - -Synchronous version of PL/pgSQL parsing. - -```typescript -import { parsePlPgSQLSync } from '@libpg-query/parser'; - -const result = parsePlPgSQLSync(functionSql); -``` - -### `fingerprint(sql: string): Promise` - -Generates a unique fingerprint for a SQL query that can be used for query identification and caching. Returns a Promise for a 16-character fingerprint string. - -```typescript -import { fingerprint } from '@libpg-query/parser'; - -const fp = await fingerprint('SELECT * FROM users WHERE active = $1'); -// Returns: string - unique 16-character fingerprint (e.g., "50fde20626009aba") -``` - -### `fingerprintSync(sql: string): string` - -Synchronous version that generates a unique fingerprint for a SQL query directly. - -```typescript -import { fingerprintSync } from '@libpg-query/parser'; - -const fp = fingerprintSync('SELECT * FROM users WHERE active = $1'); -// Returns: string - unique 16-character fingerprint -``` - -### `normalize(sql: string): Promise` - -Normalizes a SQL query by removing comments, standardizing whitespace, and converting to a canonical form. Returns a Promise for the normalized SQL string. - -```typescript -import { normalize } from '@libpg-query/parser'; - -const normalized = await normalize('SELECT * FROM users WHERE active = true'); -// Returns: string - normalized SQL query -``` - -### `normalizeSync(sql: string): string` - -Synchronous version that normalizes a SQL query directly. - -```typescript -import { normalizeSync } from '@libpg-query/parser'; - -const normalized = normalizeSync('SELECT * FROM users WHERE active = true'); -// Returns: string - normalized SQL query -``` - -### `scan(sql: string): Promise` - -Scans (tokenizes) a SQL query and returns detailed information about each token. Returns a Promise for a ScanResult containing all tokens with their positions, types, and classifications. - -```typescript -import { scan } from '@libpg-query/parser'; - -const result = await scan('SELECT * FROM users WHERE id = $1'); -// Returns: ScanResult - detailed tokenization information -console.log(result.tokens[0]); // { start: 0, end: 6, text: "SELECT", tokenType: 651, tokenName: "UNKNOWN", keywordKind: 4, keywordName: "RESERVED_KEYWORD" } -``` - -### `scanSync(sql: string): ScanResult` - -Synchronous version that scans (tokenizes) a SQL query directly. - -```typescript -import { scanSync } from '@libpg-query/parser'; - -const result = scanSync('SELECT * FROM users WHERE id = $1'); -// Returns: ScanResult - detailed tokenization information -``` - -### Initialization - -The library provides both async and sync methods. Async methods handle initialization automatically, while sync methods require explicit initialization. - -#### Async Methods (Recommended) - -Async methods handle initialization automatically and are always safe to use: - -```typescript -import { parse, scan } from '@libpg-query/parser'; - -// These handle initialization automatically -const result = await parse('SELECT * FROM users'); -const tokens = await scan('SELECT * FROM users'); -``` - -#### Sync Methods - -Sync methods require explicit initialization using `loadModule()`: - -```typescript -import { loadModule, parseSync, scanSync } from '@libpg-query/parser'; - -// Initialize first -await loadModule(); - -// Now safe to use sync methods -const result = parseSync('SELECT * FROM users'); -const tokens = scanSync('SELECT * FROM users'); -``` - -### `loadModule(): Promise` - -Explicitly initializes the WASM module. Required before using any sync methods. - -```typescript -import { loadModule, parseSync, scanSync } from '@libpg-query/parser'; - -// Initialize before using sync methods -await loadModule(); -const result = parseSync('SELECT * FROM users'); -const tokens = scanSync('SELECT * FROM users'); -``` - -Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. - -### Type Definitions - -```typescript -interface ParseResult { - version: number; - stmts: Statement[]; -} - -interface Statement { - stmt_type: string; - stmt_len: number; - stmt_location: number; - query: string; -} - -interface ScanResult { - version: number; - tokens: ScanToken[]; -} - -interface ScanToken { - start: number; // Starting position in the SQL string - end: number; // Ending position in the SQL string - text: string; // The actual token text - tokenType: number; // Numeric token type identifier - tokenName: string; // Human-readable token type name - keywordKind: number; // Numeric keyword classification - keywordName: string; // Human-readable keyword classification -} -``` - -**Note:** The return value is an array, as multiple queries may be provided in a single string (semicolon-delimited, as PostgreSQL expects). - -## Build Instructions - -This package uses a **WASM-only build system** for true cross-platform compatibility without native compilation dependencies. - -### Prerequisites - -- Node.js (version 16 or higher recommended) -- [pnpm](https://pnpm.io/) (v8+ recommended) - -### Building WASM Artifacts - -1. **Install dependencies:** - ```bash - pnpm install - ``` - -2. **Build WASM artifacts:** - ```bash - pnpm run build - ``` - -3. **Clean WASM build (if needed):** - ```bash - pnpm run clean - ``` - -4. **Rebuild WASM artifacts from scratch:** - ```bash - pnpm run clean && pnpm run build - ``` - -### Build Process Details - -The WASM build process: -- Uses Emscripten SDK for compilation -- Compiles C wrapper code to WebAssembly -- Generates `wasm/libpg-query.js` and `wasm/libpg-query.wasm` files -- No native compilation or node-gyp dependencies required - -## Testing - -### Running Tests - -```bash -pnpm run test -``` - -### Test Requirements - -- WASM artifacts must be built before running tests -- If tests fail with "fetch failed" errors, rebuild WASM artifacts: - ```bash - pnpm run clean && pnpm run build && pnpm run test - ``` - - - -## Versions - -Built from the `17-constructive` branch of [constructive-io/libpg_query](https://github.com/constructive-io/libpg_query) (PostgreSQL 17, with PL/pgSQL serializer fixes). - - -## Troubleshooting - -### Common Issues - -**"fetch failed" errors during tests:** -- This indicates stale or missing WASM artifacts -- Solution: `pnpm run clean && pnpm run build` - -**"WASM module not initialized" errors:** -- Ensure you call an async method first to initialize the WASM module -- Or use the async versions of methods which handle initialization automatically - -**Build environment issues:** -- Ensure Emscripten SDK is properly installed and configured -- Check that all required build dependencies are available - -### Build Artifacts - -The build process generates these files: -- `wasm/libpg-query.js` - Emscripten-generated JavaScript loader -- `wasm/libpg-query.wasm` - WebAssembly binary -- `wasm/index.js` - ES module exports -- `wasm/index.cjs` - CommonJS exports with sync wrappers - -## Credits - -Built on the excellent work of several contributors: - -* **[Dan Lynch](https://github.com/pyramation)** β€” official maintainer since 2018 and architect of the current implementation -* **[Lukas Fittl](https://github.com/lfittl)** for [libpg_query](https://github.com/pganalyze/libpg_query) β€” the core PostgreSQL parser that powers this project -* **[Greg Richardson](https://github.com/gregnr)** for AST guidance and pushing the transition to WASM and multiple PG runtimes for better interoperability -* **[Ethan Resnick](https://github.com/ethanresnick)** for the original Node.js N-API bindings -* **[Zac McCormick](https://github.com/zhm)** for the foundational [node-pg-query-native](https://github.com/zhm/node-pg-query-native) parser - -**πŸ›  Built by the [Constructive](https://constructive.io) team β€” creators of modular Postgres tooling for secure, composable backends. If you like our work, contribute on [GitHub](https://github.com/constructive-io).** - - -## Related - -* [pgsql-parser](https://www.npmjs.com/package/pgsql-parser): The real PostgreSQL parser for Node.js, providing symmetric parsing and deparsing of SQL statements with actual PostgreSQL parser integration. -* [pgsql-deparser](https://www.npmjs.com/package/pgsql-deparser): A streamlined tool designed for converting PostgreSQL ASTs back into SQL queries, focusing solely on deparser functionality to complement `pgsql-parser`. -* [@pgsql/parser](https://www.npmjs.com/package/@pgsql/parser): Multi-version PostgreSQL parser with dynamic version selection at runtime, supporting PostgreSQL 15, 16, and 17 in a single package. -* [@pgsql/types](https://www.npmjs.com/package/@pgsql/types): Offers TypeScript type definitions for PostgreSQL AST nodes, facilitating type-safe construction, analysis, and manipulation of ASTs. -* [@pgsql/enums](https://www.npmjs.com/package/@pgsql/enums): Provides TypeScript enum definitions for PostgreSQL constants, enabling type-safe usage of PostgreSQL enums and constants in your applications. -* [@pgsql/utils](https://www.npmjs.com/package/@pgsql/utils): A comprehensive utility library for PostgreSQL, offering type-safe AST node creation and enum value conversions, simplifying the construction and manipulation of PostgreSQL ASTs. -* [pg-proto-parser](https://www.npmjs.com/package/pg-proto-parser): A TypeScript tool that parses PostgreSQL Protocol Buffers definitions to generate TypeScript interfaces, utility functions, and JSON mappings for enums. -* [libpg-query](https://github.com/constructive-io/libpg-query-node): The real PostgreSQL parser exposed for Node.js, used primarily in `pgsql-parser` for parsing and deparsing SQL queries. - -## Disclaimer - -AS DESCRIBED IN THE LICENSES, THE SOFTWARE IS PROVIDED "AS IS", AT YOUR OWN RISK, AND WITHOUT WARRANTIES OF ANY KIND. - -No developer or entity involved in creating Software will be liable for any claims or damages whatsoever associated with your use, inability to use, or your interaction with other users of the Software code or Software CLI, including any direct, indirect, incidental, special, exemplary, punitive or consequential damages, or loss of profits, cryptocurrencies, tokens, or anything else of value. \ No newline at end of file diff --git a/full/17/libpg_query/protobuf/.gitkeep b/full/17/libpg_query/protobuf/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/full/17/package.json b/full/17/package.json deleted file mode 100644 index b7444a64..00000000 --- a/full/17/package.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "@libpg-query/full17", - "version": "17.8.0", - "description": "The real PostgreSQL query parser", - "homepage": "https://github.com/constructive-io/libpg-query-node", - "main": "./wasm/index.cjs", - "module": "./wasm/index.js", - "typings": "./wasm/index.d.ts", - "publishConfig": { - "access": "public" - }, - "x-publish": { - "publishName": "@libpg-query/parser", - "pgVersion": "17", - "distTag": "pg17", - "libpgQueryTag": "17-constructive" - }, - "files": [ - "wasm/*" - ], - "scripts": { - "clean": "pnpm wasm:clean && rimraf cjs esm", - "build:js": "node scripts/build.js", - "build": "pnpm clean; pnpm wasm:build; pnpm build:js", - "publish:pkg": "node ../../scripts/publish-single-version.js", - "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", - "wasm:build": "pnpm wasm:make build", - "wasm:rebuild": "pnpm wasm:make rebuild", - "wasm:clean": "pnpm wasm:make clean", - "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js", - "yamlize": "node ./scripts/yamlize.js" - }, - "author": "Constructive ", - "license": "MIT", - "repository": { - "type": "git", - "url": "git://github.com/constructive-io/libpg-query-node.git" - }, - "devDependencies": { - "@yamlize/cli": "^0.8.0" - }, - "dependencies": { - "@pgsql/types": "^17.6.2" - }, - "keywords": [ - "sql", - "postgres", - "postgresql", - "pg", - "query", - "plpgsql", - "database" - ] -} diff --git a/full/17/scripts/build.js b/full/17/scripts/build.js deleted file mode 100644 index 55d83240..00000000 --- a/full/17/scripts/build.js +++ /dev/null @@ -1,38 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Run TypeScript compilation -console.log('Compiling TypeScript...'); -execSync('tsc', { stdio: 'inherit' }); -execSync('tsc -p tsconfig.esm.json', { stdio: 'inherit' }); - -// Rename files to have correct extensions -const wasmDir = path.join(__dirname, '../wasm'); -const cjsDir = path.join(__dirname, '../cjs'); -const esmDir = path.join(__dirname, '../esm'); - -// Ensure wasm directory exists -if (!fs.existsSync(wasmDir)) { - fs.mkdirSync(wasmDir, { recursive: true }); -} - -// Rename CommonJS files -fs.renameSync( - path.join(cjsDir, 'index.js'), - path.join(wasmDir, 'index.cjs') -); - -// Rename ESM files -fs.renameSync( - path.join(esmDir, 'index.js'), - path.join(wasmDir, 'index.js') -); - -// Rename declaration files -fs.renameSync( - path.join(cjsDir, 'index.d.ts'), - path.join(wasmDir, 'index.d.ts') -); - -console.log('Build completed successfully!'); \ No newline at end of file diff --git a/full/17/src/proto.d.ts b/full/17/src/proto.d.ts deleted file mode 100644 index 55a504f3..00000000 --- a/full/17/src/proto.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -declare module '../proto.js' { - export namespace pg_query { - interface ParseResult { - version: number; - stmts: Statement[]; - } - - interface Statement { - stmt_type: string; - stmt_len: number; - stmt_location: number; - query: string; - } - - class ParseResult { - static fromObject(obj: ParseResult): ParseResult; - static encode(msg: ParseResult): { finish(): Uint8Array }; - } - } -} \ No newline at end of file diff --git a/full/17/test/errors.test.js b/full/17/test/errors.test.js deleted file mode 100644 index f6c0fd61..00000000 --- a/full/17/test/errors.test.js +++ /dev/null @@ -1,325 +0,0 @@ -const { describe, it, before } = require('node:test'); -const assert = require('node:assert/strict'); -const { parseSync, loadModule, formatSqlError, hasSqlDetails } = require('../wasm/index.cjs'); - -describe('Enhanced Error Handling', () => { - before(async () => { - await loadModule(); - }); - - describe('Error Details Structure', () => { - it('should include sqlDetails property on parse errors', () => { - assert.throws(() => { - parseSync('SELECT * FROM users WHERE id = @'); - }); - - try { - parseSync('SELECT * FROM users WHERE id = @'); - } catch (error) { - assert.ok('sqlDetails' in error); - assert.ok('message' in error.sqlDetails); - assert.ok('cursorPosition' in error.sqlDetails); - assert.ok('fileName' in error.sqlDetails); - assert.ok('functionName' in error.sqlDetails); - assert.ok('lineNumber' in error.sqlDetails); - } - }); - - it('should have correct cursor position (0-based)', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 32); - } - }); - - it('should identify error source file', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.fileName, 'scan.l'); - assert.equal(error.sqlDetails.functionName, 'scanner_yyerror'); - } - }); - }); - - describe('Error Position Accuracy', () => { - const positionTests = [ - { query: '@ SELECT * FROM users', expectedPos: 0, desc: 'error at start' }, - { query: 'SELECT @ FROM users', expectedPos: 9, desc: 'error after SELECT' }, - { query: 'SELECT * FROM users WHERE @ = 1', expectedPos: 28, desc: 'error after WHERE' }, - { query: 'SELECT * FROM users WHERE id = @', expectedPos: 32, desc: 'error at end' }, - { query: 'INSERT INTO users (id, name) VALUES (1, @)', expectedPos: 41, desc: 'error in VALUES' }, - { query: 'UPDATE users SET name = @ WHERE id = 1', expectedPos: 26, desc: 'error in SET' }, - { query: 'CREATE TABLE test (id INT, name @)', expectedPos: 32, desc: 'error in CREATE TABLE' }, - ]; - - positionTests.forEach(({ query, expectedPos, desc }) => { - it(`should correctly identify position for ${desc}`, () => { - try { - parseSync(query); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, expectedPos); - } - }); - }); - }); - - describe('Error Types', () => { - it('should handle unterminated string literals', () => { - try { - parseSync("SELECT * FROM users WHERE name = 'unclosed"); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('unterminated quoted string')); - assert.equal(error.sqlDetails.cursorPosition, 33); - } - }); - - it('should handle unterminated quoted identifiers', () => { - try { - parseSync('SELECT * FROM users WHERE name = "unclosed'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('unterminated quoted identifier')); - assert.equal(error.sqlDetails.cursorPosition, 33); - } - }); - - it('should handle invalid tokens', () => { - try { - parseSync('SELECT * FROM users WHERE id = $'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('syntax error at or near "$"')); - assert.equal(error.sqlDetails.cursorPosition, 31); - } - }); - - it('should handle reserved keywords', () => { - try { - parseSync('SELECT * FROM table'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('syntax error at or near "table"')); - assert.equal(error.sqlDetails.cursorPosition, 14); - } - }); - - it('should handle syntax error in WHERE clause', () => { - try { - parseSync('SELECT * FROM users WHERE'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('syntax error at end of input')); - assert.equal(error.sqlDetails.cursorPosition, 25); - } - }); - }); - - describe('formatSqlError Helper', () => { - it('should format error with position indicator', () => { - try { - parseSync("SELECT * FROM users WHERE id = 'unclosed"); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, "SELECT * FROM users WHERE id = 'unclosed"); - assert.ok(formatted.includes('Error: unterminated quoted string')); - assert.ok(formatted.includes('Position: 31')); - assert.ok(formatted.includes("SELECT * FROM users WHERE id = 'unclosed")); - assert.ok(formatted.includes(' ^')); - } - }); - - it('should respect showPosition option', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { - showPosition: false - }); - assert.ok(!formatted.includes('^')); - assert.ok(formatted.includes('Position: 32')); - } - }); - - it('should respect showQuery option', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { - showQuery: false - }); - assert.ok(!formatted.includes('SELECT * FROM users')); - assert.ok(formatted.includes('Error:')); - assert.ok(formatted.includes('Position:')); - } - }); - - it('should truncate long queries', () => { - const longQuery = 'SELECT ' + 'a, '.repeat(50) + 'z FROM users WHERE id = @'; - try { - parseSync(longQuery); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, longQuery, { maxQueryLength: 50 }); - assert.ok(formatted.includes('...')); - const lines = formatted.split('\n'); - const queryLine = lines.find(line => line.includes('...')); - assert.ok(queryLine.length <= 56); // 50 + 2*3 for ellipsis - } - }); - - it('should handle color option without breaking output', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { - color: true - }); - assert.ok(formatted.includes('Error:')); - assert.ok(formatted.includes('Position:')); - // Should contain ANSI codes but still be readable - const cleanFormatted = formatted.replace(/\x1b\[[0-9;]*m/g, ''); - assert.ok(cleanFormatted.includes('syntax error')); - } - }); - }); - - describe('hasSqlDetails Type Guard', () => { - it('should return true for SQL parse errors', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(hasSqlDetails(error), true); - } - }); - - it('should return false for regular errors', () => { - const regularError = new Error('Regular error'); - assert.equal(hasSqlDetails(regularError), false); - }); - - it('should return false for non-Error objects', () => { - assert.equal(hasSqlDetails('string'), false); - assert.equal(hasSqlDetails(123), false); - assert.equal(hasSqlDetails(null), false); - assert.equal(hasSqlDetails(undefined), false); - assert.equal(hasSqlDetails({}), false); - }); - - it('should return false for Error with incomplete sqlDetails', () => { - const error = new Error('Test'); - error.sqlDetails = { message: 'test' }; // Missing cursorPosition - assert.equal(hasSqlDetails(error), false); - }); - }); - - describe('Edge Cases', () => { - it('should handle empty query', () => { - assert.throws(() => parseSync(''), { - message: 'Query cannot be empty' - }); - }); - - it('should handle null query', () => { - assert.throws(() => parseSync(null), { - message: 'Query cannot be null or undefined' - }); - }); - - it('should handle undefined query', () => { - assert.throws(() => parseSync(undefined), { - message: 'Query cannot be null or undefined' - }); - }); - - it('should handle @ in comments', () => { - const query = 'SELECT * FROM users /* @ in comment */ WHERE id = 1'; - assert.doesNotThrow(() => parseSync(query)); - }); - - it('should handle @ in strings', () => { - const query = 'SELECT * FROM users WHERE email = \'user@example.com\''; - assert.doesNotThrow(() => parseSync(query)); - }); - }); - - describe('Complex Error Scenarios', () => { - it('should handle errors in CASE statements', () => { - try { - parseSync('SELECT CASE WHEN id = 1 THEN "one" WHEN id = 2 THEN @ ELSE "other" END FROM users'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 54); - } - }); - - it('should handle errors in subqueries', () => { - try { - parseSync('SELECT * FROM users WHERE id IN (SELECT @ FROM orders)'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 42); - } - }); - - it('should handle errors in function calls', () => { - try { - parseSync('SELECT COUNT(@) FROM users'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 14); - } - }); - - it('should handle errors in second statement', () => { - try { - parseSync('SELECT * FROM users; SELECT * FROM orders WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 54); - } - }); - - it('should handle errors in CTE', () => { - try { - parseSync('WITH cte AS (SELECT * FROM users WHERE id = @) SELECT * FROM cte'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 45); - } - }); - }); - - describe('Backward Compatibility', () => { - it('should maintain Error instance', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error instanceof Error); - assert.ok(error.message); - assert.ok(error.stack); - } - }); - - it('should work with standard error handling', () => { - let caught = false; - try { - parseSync('SELECT * FROM users WHERE id = @'); - } catch (e) { - caught = true; - assert.ok(e.message.includes('syntax error')); - } - assert.equal(caught, true); - }); - }); -}); \ No newline at end of file diff --git a/full/17/test/parsing.test.js b/full/17/test/parsing.test.js deleted file mode 100644 index 5477c6a0..00000000 --- a/full/17/test/parsing.test.js +++ /dev/null @@ -1,84 +0,0 @@ -const query = require("../"); -const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); -const assert = require('node:assert/strict'); - -function removeLocationProperties(obj) { - if (typeof obj !== 'object' || obj === null) { - return obj; - } - - if (Array.isArray(obj)) { - return obj.map(item => removeLocationProperties(item)); - } - - const result = {}; - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { - continue; // Skip location-related properties - } - result[key] = removeLocationProperties(obj[key]); - } - } - return result; -} - -describe("Query Parsing", () => { - before(async () => { - await query.parse("SELECT 1"); - }); - - describe("Sync Parsing", () => { - it("should return a single-item parse result for common queries", () => { - const queries = ["select 1", "select null", "select ''", "select a, b"]; - const results = queries.map(query.parseSync); - results.forEach((res) => { - assert.equal(res.stmts.length, 1); - }); - - const selectedDatas = results.map( - (it) => it.stmts[0].stmt.SelectStmt.targetList - ); - - assert.equal(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival, 1); - assert.equal(selectedDatas[1][0].ResTarget.val.A_Const.isnull, true); - assert.equal(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval, ""); - assert.equal(selectedDatas[3].length, 2); - }); - - it("should support parsing multiple queries", () => { - const res = query.parseSync("select 1; select null;"); - assert.deepEqual(res.stmts.map(removeLocationProperties), [ - ...query.parseSync("select 1;").stmts.map(removeLocationProperties), - ...query.parseSync("select null;").stmts.map(removeLocationProperties), - ]); - }); - - it("should not parse a bogus query", () => { - assert.throws(() => query.parseSync("NOT A QUERY"), Error); - }); - }); - - describe("Async parsing", () => { - it("should return a promise resolving to same result", async () => { - const testQuery = "select * from john;"; - const resPromise = query.parse(testQuery); - const res = await resPromise; - - assert.ok(resPromise instanceof Promise); - assert.deepEqual(res, query.parseSync(testQuery)); - }); - - it("should reject on bogus queries", async () => { - return query.parse("NOT A QUERY").then( - () => { - throw new Error("should have rejected"); - }, - (e) => { - assert.ok(e instanceof Error); - assert.match(e.message, /NOT/); - } - ); - }); - }); -}); diff --git a/full/17/tsconfig.esm.json b/full/17/tsconfig.esm.json deleted file mode 100644 index 63012c17..00000000 --- a/full/17/tsconfig.esm.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "module": "es2022", - "rootDir": "src/", - "declaration": false, - "outDir": "esm/" - } - } \ No newline at end of file diff --git a/full/17/tsconfig.json b/full/17/tsconfig.json deleted file mode 100644 index d7da207b..00000000 --- a/full/17/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "commonjs", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "strictNullChecks": false, - "skipLibCheck": true, - "sourceMap": false, - "declaration": true, - "resolveJsonModule": true, - "moduleResolution": "node", - "outDir": "cjs/", - "rootDir": "src" - }, - "exclude": ["cjs", "esm", "wasm", "node_modules", "**/*.test.ts"] -} \ No newline at end of file diff --git a/full/18/CHANGELOG.md b/full/18/CHANGELOG.md deleted file mode 100644 index d40399b2..00000000 --- a/full/18/CHANGELOG.md +++ /dev/null @@ -1,44 +0,0 @@ -# 18.0.0 -* PostgreSQL 18 full build, compiled from the `18-constructive` branch of constructive-io/libpg_query -* Full API: parse, parsePlPgSQL, scan, fingerprint, normalize + sync variants -* Uses @pgsql/types ^18.0.0 -* Published as @libpg-query/parser under the `pg18` dist-tag - -# 17.2.1 -* Add normalize() and normalizeSync() functions for SQL normalization -* Add fingerprint() and fingerprintSync() functions for query fingerprinting -* Add isReady() function to check WASM module initialization status -* Improve memory management and error handling in WASM wrapper -* Remove unused dependencies (lodash, deasync, @launchql/protobufjs) -* Split test suite into separate files for better organization -* Update documentation with comprehensive function examples - -# 1.2.1 -* Rename package - -# 1.2.0 -* Add TS typings - -# 1.1.2 (07/23/19) -* Fix build script - -# 1.1.0 (Republish, 07/23/19) -* Rewrite to use Node-Addons-API (C++ wrapper for the stable N-API) -* Support PL/PGSql Parsing -* Support async parsing - -## 0.0.5 (November 19, 2015) -* Update libpg_query to include latest A_CONST fixes - -## 0.0.4 (November 16, 2015) -* Update libpg_query to include fix for empty string constants -* Warning fixed in 0.0.3 is back for now until libpg_query has support for custom CFLAGS - -## 0.0.3 (November 14, 2015) -* Update OSX static library to include `-mmacosx-version-min=10.5` to fix warnings on installation - -## 0.0.2 (November 14, 2015) -* Rename repo to pg-query-native - -## 0.0.1 (November 14, 2015) -* First version diff --git a/full/18/Makefile b/full/18/Makefile deleted file mode 100644 index c9685261..00000000 --- a/full/18/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -WASM_OUT_DIR := wasm -WASM_OUT_NAME := libpg-query -WASM_MODULE_NAME := PgQueryModule -# LIBPG_QUERY_REPO := https://github.com/pganalyze/libpg_query.git -# LIBPG_QUERY_TAG := 18.0.0 -LIBPG_QUERY_REPO := https://github.com/constructive-io/libpg_query.git -LIBPG_QUERY_TAG := 18-constructive - -CACHE_DIR := .cache - -OS ?= $(shell uname -s) -ARCH ?= $(shell uname -m) - -ifdef EMSCRIPTEN -PLATFORM := emscripten -else ifeq ($(OS),Darwin) -PLATFORM := darwin -else ifeq ($(OS),Linux) -PLATFORM := linux -else -$(error Unsupported platform: $(OS)) -endif - -ifdef EMSCRIPTEN -ARCH := wasm -endif - -PLATFORM_ARCH := $(PLATFORM)-$(ARCH) -SRC_FILES := src/wasm_wrapper.c -LIBPG_QUERY_DIR := $(CACHE_DIR)/$(PLATFORM_ARCH)/libpg_query/$(LIBPG_QUERY_TAG) -LIBPG_QUERY_ARCHIVE := $(LIBPG_QUERY_DIR)/libpg_query.a -LIBPG_QUERY_HEADER := $(LIBPG_QUERY_DIR)/pg_query.h -CXXFLAGS := -O3 - -ifdef EMSCRIPTEN -OUT_FILES := $(foreach EXT,.js .wasm,$(WASM_OUT_DIR)/$(WASM_OUT_NAME)$(EXT)) -else -$(error Native builds are no longer supported. Use EMSCRIPTEN=1 for WASM builds only.) -endif - -# Clone libpg_query source (lives in CACHE_DIR) -$(LIBPG_QUERY_DIR): - mkdir -p $(CACHE_DIR) - git clone -b $(LIBPG_QUERY_TAG) --single-branch $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) - -$(LIBPG_QUERY_HEADER): $(LIBPG_QUERY_DIR) - -# Build libpg_query -$(LIBPG_QUERY_ARCHIVE): $(LIBPG_QUERY_DIR) - cd $(LIBPG_QUERY_DIR); $(MAKE) build - -# Build libpg-query-node WASM module -$(OUT_FILES): $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) $(SRC_FILES) -ifdef EMSCRIPTEN - mkdir -p $(WASM_OUT_DIR) - $(CC) \ - -v \ - $(CXXFLAGS) \ - -I$(LIBPG_QUERY_DIR) \ - -I$(LIBPG_QUERY_DIR)/vendor \ - -L$(LIBPG_QUERY_DIR) \ - -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string','_wasm_parse_query_raw','_wasm_free_parse_result']" \ - -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','UTF8ToString','getValue','HEAPU8','HEAPU32']" \ - -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ - -sENVIRONMENT="web,node,worker" \ - -sASSERTIONS=0 \ - -sSINGLE_FILE=0 \ - -sMODULARIZE=1 \ - -sEXPORT_ES6=0 \ - -sALLOW_MEMORY_GROWTH=1 \ - -sINITIAL_MEMORY=134217728 \ - -sMAXIMUM_MEMORY=1073741824 \ - -sSTACK_SIZE=33554432 \ - -lpg_query \ - -o $@ \ - $(SRC_FILES) -else -$(error Native builds are no longer supported. Use EMSCRIPTEN=1 for WASM builds only.) -endif - -# Commands -build: $(OUT_FILES) - -build-cache: $(LIBPG_QUERY_ARCHIVE) $(LIBPG_QUERY_HEADER) - -rebuild: clean build - -rebuild-cache: clean-cache build-cache - -clean: - -@ rm -r $(OUT_FILES) > /dev/null 2>&1 - -clean-cache: - -@ rm -rf $(LIBPG_QUERY_DIR) - -.PHONY: build build-cache rebuild rebuild-cache clean clean-cache diff --git a/full/18/README.md b/full/18/README.md deleted file mode 100644 index 42d174f5..00000000 --- a/full/18/README.md +++ /dev/null @@ -1,358 +0,0 @@ -# @libpg-query/parser - -

- constructive.io -

- -

- - - -
- - - - -

- -# The Real PostgreSQL Parser for JavaScript - -### Bring the power of PostgreSQL’s native parser to your JavaScript projects β€” no native builds, no platform headaches. - -This is the official PostgreSQL parser, compiled to WebAssembly (WASM) for seamless, cross-platform compatibility. Use it in Node.js or the browser, on Linux, Windows, or anywhere JavaScript runs. - -Built to power [pgsql-parser](https://github.com/constructive-io/pgsql-parser), this library delivers full fidelity with the Postgres C codebase β€” no rewrites, no shortcuts. - -### Features - -* πŸ”§ **Powered by PostgreSQL** – Uses the official Postgres C parser compiled to WebAssembly -* πŸ–₯️ **Cross-Platform** – Runs smoothly on macOS, Linux, and Windows -* 🌐 **Node.js & Browser Support** – Consistent behavior in any JS environment -* πŸ“¦ **No Native Builds Required** – No compilation, no system-specific dependencies -* 🧠 **Spec-Accurate Parsing** – Produces faithful, standards-compliant ASTs -* πŸš€ **Production-Grade** – Millions of downloads and trusted by countless projects and top teams - -## πŸš€ For Round-trip Codegen - -> 🎯 **Want to parse + deparse (full round trip)?** -> We highly recommend using [`pgsql-parser`](https://github.com/constructive-io/pgsql-parser) which leverages a pure TypeScript deparser that has been battle-tested against 23,000+ SQL statements and is built on top of libpg-query. - -## Installation - -```sh -npm install @libpg-query/parser -``` - -## Usage - -### `parse(query: string): Promise` - -Parses the SQL and returns a Promise for the parse tree. May reject with a parse error. - -```typescript -import { parse } from '@libpg-query/parser'; - -const result = await parse('SELECT * FROM users WHERE active = true'); -// Returns: ParseResult - parsed query object -``` - -### `parseSync(query: string): ParseResult` - -Synchronous version that returns the parse tree directly. May throw a parse error. - -```typescript -import { parseSync } from '@libpg-query/parser'; - -const result = parseSync('SELECT * FROM users WHERE active = true'); -// Returns: ParseResult - parsed query object -``` - -### `parsePlPgSQL(funcsSql: string): Promise` - -Parses the contents of a PL/pgSQL function from a `CREATE FUNCTION` declaration. Returns a Promise for the parse tree. - -```typescript -import { parsePlPgSQL } from '@libpg-query/parser'; - -const functionSql = ` -CREATE FUNCTION get_user_count() RETURNS integer AS $$ -BEGIN - RETURN (SELECT COUNT(*) FROM users); -END; -$$ LANGUAGE plpgsql; -`; - -const result = await parsePlPgSQL(functionSql); -``` - -### `parsePlPgSQLSync(funcsSql: string): ParseResult` - -Synchronous version of PL/pgSQL parsing. - -```typescript -import { parsePlPgSQLSync } from '@libpg-query/parser'; - -const result = parsePlPgSQLSync(functionSql); -``` - -### `fingerprint(sql: string): Promise` - -Generates a unique fingerprint for a SQL query that can be used for query identification and caching. Returns a Promise for a 16-character fingerprint string. - -```typescript -import { fingerprint } from '@libpg-query/parser'; - -const fp = await fingerprint('SELECT * FROM users WHERE active = $1'); -// Returns: string - unique 16-character fingerprint (e.g., "50fde20626009aba") -``` - -### `fingerprintSync(sql: string): string` - -Synchronous version that generates a unique fingerprint for a SQL query directly. - -```typescript -import { fingerprintSync } from '@libpg-query/parser'; - -const fp = fingerprintSync('SELECT * FROM users WHERE active = $1'); -// Returns: string - unique 16-character fingerprint -``` - -### `normalize(sql: string): Promise` - -Normalizes a SQL query by removing comments, standardizing whitespace, and converting to a canonical form. Returns a Promise for the normalized SQL string. - -```typescript -import { normalize } from '@libpg-query/parser'; - -const normalized = await normalize('SELECT * FROM users WHERE active = true'); -// Returns: string - normalized SQL query -``` - -### `normalizeSync(sql: string): string` - -Synchronous version that normalizes a SQL query directly. - -```typescript -import { normalizeSync } from '@libpg-query/parser'; - -const normalized = normalizeSync('SELECT * FROM users WHERE active = true'); -// Returns: string - normalized SQL query -``` - -### `scan(sql: string): Promise` - -Scans (tokenizes) a SQL query and returns detailed information about each token. Returns a Promise for a ScanResult containing all tokens with their positions, types, and classifications. - -```typescript -import { scan } from '@libpg-query/parser'; - -const result = await scan('SELECT * FROM users WHERE id = $1'); -// Returns: ScanResult - detailed tokenization information -console.log(result.tokens[0]); // { start: 0, end: 6, text: "SELECT", tokenType: 651, tokenName: "UNKNOWN", keywordKind: 4, keywordName: "RESERVED_KEYWORD" } -``` - -### `scanSync(sql: string): ScanResult` - -Synchronous version that scans (tokenizes) a SQL query directly. - -```typescript -import { scanSync } from '@libpg-query/parser'; - -const result = scanSync('SELECT * FROM users WHERE id = $1'); -// Returns: ScanResult - detailed tokenization information -``` - -### Initialization - -The library provides both async and sync methods. Async methods handle initialization automatically, while sync methods require explicit initialization. - -#### Async Methods (Recommended) - -Async methods handle initialization automatically and are always safe to use: - -```typescript -import { parse, scan } from '@libpg-query/parser'; - -// These handle initialization automatically -const result = await parse('SELECT * FROM users'); -const tokens = await scan('SELECT * FROM users'); -``` - -#### Sync Methods - -Sync methods require explicit initialization using `loadModule()`: - -```typescript -import { loadModule, parseSync, scanSync } from '@libpg-query/parser'; - -// Initialize first -await loadModule(); - -// Now safe to use sync methods -const result = parseSync('SELECT * FROM users'); -const tokens = scanSync('SELECT * FROM users'); -``` - -### `loadModule(): Promise` - -Explicitly initializes the WASM module. Required before using any sync methods. - -```typescript -import { loadModule, parseSync, scanSync } from '@libpg-query/parser'; - -// Initialize before using sync methods -await loadModule(); -const result = parseSync('SELECT * FROM users'); -const tokens = scanSync('SELECT * FROM users'); -``` - -Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. - -### Type Definitions - -```typescript -interface ParseResult { - version: number; - stmts: Statement[]; -} - -interface Statement { - stmt_type: string; - stmt_len: number; - stmt_location: number; - query: string; -} - -interface ScanResult { - version: number; - tokens: ScanToken[]; -} - -interface ScanToken { - start: number; // Starting position in the SQL string - end: number; // Ending position in the SQL string - text: string; // The actual token text - tokenType: number; // Numeric token type identifier - tokenName: string; // Human-readable token type name - keywordKind: number; // Numeric keyword classification - keywordName: string; // Human-readable keyword classification -} -``` - -**Note:** The return value is an array, as multiple queries may be provided in a single string (semicolon-delimited, as PostgreSQL expects). - -## Build Instructions - -This package uses a **WASM-only build system** for true cross-platform compatibility without native compilation dependencies. - -### Prerequisites - -- Node.js (version 16 or higher recommended) -- [pnpm](https://pnpm.io/) (v8+ recommended) - -### Building WASM Artifacts - -1. **Install dependencies:** - ```bash - pnpm install - ``` - -2. **Build WASM artifacts:** - ```bash - pnpm run build - ``` - -3. **Clean WASM build (if needed):** - ```bash - pnpm run clean - ``` - -4. **Rebuild WASM artifacts from scratch:** - ```bash - pnpm run clean && pnpm run build - ``` - -### Build Process Details - -The WASM build process: -- Uses Emscripten SDK for compilation -- Compiles C wrapper code to WebAssembly -- Generates `wasm/libpg-query.js` and `wasm/libpg-query.wasm` files -- No native compilation or node-gyp dependencies required - -## Testing - -### Running Tests - -```bash -pnpm run test -``` - -### Test Requirements - -- WASM artifacts must be built before running tests -- If tests fail with "fetch failed" errors, rebuild WASM artifacts: - ```bash - pnpm run clean && pnpm run build && pnpm run test - ``` - - - -## Versions - -Built from the `18-constructive` branch of [constructive-io/libpg_query](https://github.com/constructive-io/libpg_query) (PostgreSQL 18, with PL/pgSQL serializer fixes). - - -## Troubleshooting - -### Common Issues - -**"fetch failed" errors during tests:** -- This indicates stale or missing WASM artifacts -- Solution: `pnpm run clean && pnpm run build` - -**"WASM module not initialized" errors:** -- Ensure you call an async method first to initialize the WASM module -- Or use the async versions of methods which handle initialization automatically - -**Build environment issues:** -- Ensure Emscripten SDK is properly installed and configured -- Check that all required build dependencies are available - -### Build Artifacts - -The build process generates these files: -- `wasm/libpg-query.js` - Emscripten-generated JavaScript loader -- `wasm/libpg-query.wasm` - WebAssembly binary -- `wasm/index.js` - ES module exports -- `wasm/index.cjs` - CommonJS exports with sync wrappers - -## Credits - -Built on the excellent work of several contributors: - -* **[Dan Lynch](https://github.com/pyramation)** β€” official maintainer since 2018 and architect of the current implementation -* **[Lukas Fittl](https://github.com/lfittl)** for [libpg_query](https://github.com/pganalyze/libpg_query) β€” the core PostgreSQL parser that powers this project -* **[Greg Richardson](https://github.com/gregnr)** for AST guidance and pushing the transition to WASM and multiple PG runtimes for better interoperability -* **[Ethan Resnick](https://github.com/ethanresnick)** for the original Node.js N-API bindings -* **[Zac McCormick](https://github.com/zhm)** for the foundational [node-pg-query-native](https://github.com/zhm/node-pg-query-native) parser - -**πŸ›  Built by the [Constructive](https://constructive.io) team β€” creators of modular Postgres tooling for secure, composable backends. If you like our work, contribute on [GitHub](https://github.com/constructive-io).** - - -## Related - -* [pgsql-parser](https://www.npmjs.com/package/pgsql-parser): The real PostgreSQL parser for Node.js, providing symmetric parsing and deparsing of SQL statements with actual PostgreSQL parser integration. -* [pgsql-deparser](https://www.npmjs.com/package/pgsql-deparser): A streamlined tool designed for converting PostgreSQL ASTs back into SQL queries, focusing solely on deparser functionality to complement `pgsql-parser`. -* [@pgsql/parser](https://www.npmjs.com/package/@pgsql/parser): Multi-version PostgreSQL parser with dynamic version selection at runtime, supporting PostgreSQL 15, 16, and 17 in a single package. -* [@pgsql/types](https://www.npmjs.com/package/@pgsql/types): Offers TypeScript type definitions for PostgreSQL AST nodes, facilitating type-safe construction, analysis, and manipulation of ASTs. -* [@pgsql/enums](https://www.npmjs.com/package/@pgsql/enums): Provides TypeScript enum definitions for PostgreSQL constants, enabling type-safe usage of PostgreSQL enums and constants in your applications. -* [@pgsql/utils](https://www.npmjs.com/package/@pgsql/utils): A comprehensive utility library for PostgreSQL, offering type-safe AST node creation and enum value conversions, simplifying the construction and manipulation of PostgreSQL ASTs. -* [pg-proto-parser](https://www.npmjs.com/package/pg-proto-parser): A TypeScript tool that parses PostgreSQL Protocol Buffers definitions to generate TypeScript interfaces, utility functions, and JSON mappings for enums. -* [libpg-query](https://github.com/constructive-io/libpg-query-node): The real PostgreSQL parser exposed for Node.js, used primarily in `pgsql-parser` for parsing and deparsing SQL queries. - -## Disclaimer - -AS DESCRIBED IN THE LICENSES, THE SOFTWARE IS PROVIDED "AS IS", AT YOUR OWN RISK, AND WITHOUT WARRANTIES OF ANY KIND. - -No developer or entity involved in creating Software will be liable for any claims or damages whatsoever associated with your use, inability to use, or your interaction with other users of the Software code or Software CLI, including any direct, indirect, incidental, special, exemplary, punitive or consequential damages, or loss of profits, cryptocurrencies, tokens, or anything else of value. \ No newline at end of file diff --git a/full/18/SCAN.md b/full/18/SCAN.md deleted file mode 100644 index aafbc4fe..00000000 --- a/full/18/SCAN.md +++ /dev/null @@ -1,439 +0,0 @@ -# Scan API Documentation - -## Overview - -The scan API provides detailed tokenization of PostgreSQL SQL queries, returning information about each token including its position, type, and keyword classification. This document explains how to use the scan functionality and how it relates to the parsing API. - -## Basic Usage - -### Async Scanning -```typescript -import { scan } from 'libpg-query'; - -const result = await scan('SELECT id, name FROM users WHERE active = true'); -console.log(result.tokens); -``` - -### Sync Scanning -```typescript -import { scanSync, loadModule } from 'libpg-query'; - -// Initialize WASM module first -await loadModule(); - -const result = scanSync('SELECT id, name FROM users WHERE active = true'); -console.log(result.tokens); -``` - -## Token Information - -Each token in the scan result contains: - -```typescript -interface ScanToken { - start: number; // Starting position in the SQL string (0-based) - end: number; // Ending position in the SQL string (exclusive) - text: string; // The actual token text extracted from SQL - tokenType: number; // Numeric token type identifier - tokenName: string; // Human-readable token type name - keywordKind: number; // Numeric keyword classification - keywordName: string; // Human-readable keyword classification -} -``` - -## Token Types - -### Common Token Types - -| tokenName | Description | Example | -|-----------|-------------|---------| -| `IDENT` | Regular identifier | `users`, `id`, `column_name` | -| `SCONST` | String constant | `'hello world'`, `'value'` | -| `ICONST` | Integer constant | `42`, `123`, `0` | -| `FCONST` | Float constant | `3.14`, `2.718` | -| `PARAM` | Parameter marker | `$1`, `$2`, `$3` | -| `ASCII_40` | Left parenthesis | `(` | -| `ASCII_41` | Right parenthesis | `)` | -| `ASCII_42` | Asterisk | `*` | -| `ASCII_44` | Comma | `,` | -| `ASCII_59` | Semicolon | `;` | -| `ASCII_61` | Equals sign | `=` | -| `TYPECAST` | Type casting operator | `::` | -| `GREATER_EQUALS` | Greater than or equal | `>=` | -| `LESS_EQUALS` | Less than or equal | `<=` | -| `NOT_EQUALS` | Not equal operator | `<>`, `!=` | -| `SQL_COMMENT` | SQL comment | `-- comment` | -| `C_COMMENT` | C-style comment | `/* comment */` | -| `UNKNOWN` | Keywords and other tokens | `SELECT`, `FROM`, `WHERE` | - -### Keyword Classifications - -| keywordName | Description | Examples | -|-------------|-------------|----------| -| `NO_KEYWORD` | Not a keyword | identifiers, operators, literals | -| `UNRESERVED_KEYWORD` | Unreserved keyword | `INSERT`, `UPDATE`, `name` | -| `COL_NAME_KEYWORD` | Column name keyword | `VALUES`, `BETWEEN` | -| `TYPE_FUNC_NAME_KEYWORD` | Type/function name keyword | `INNER`, `JOIN`, `IS` | -| `RESERVED_KEYWORD` | Reserved keyword | `SELECT`, `FROM`, `WHERE` | - -## Relationship with Parse Tree - -The scan token positions directly correspond to `location` fields in the Abstract Syntax Tree (AST) produced by the parse API. This allows you to map between tokens and AST nodes. - -### Example Mapping - -Given this SQL: -```sql -SELECT id, name FROM users WHERE active = true -``` - -**Scan tokens:** -``` -[0] "SELECT" (0-6) -[1] "id" (7-9) -[2] "," (9-10) -[3] "name" (11-15) -[4] "FROM" (16-20) -[5] "users" (21-26) -[6] "WHERE" (27-32) -[7] "active" (33-39) -[8] "=" (40-41) -[9] "true" (42-46) -``` - -**Corresponding AST locations:** -```json -{ - "SelectStmt": { - "targetList": [ - { - "ResTarget": { - "val": { - "ColumnRef": { - "fields": [{"String": {"sval": "id"}}], - "location": 7 // matches "id" token at position 7-9 - } - }, - "location": 7 - } - }, - { - "ResTarget": { - "val": { - "ColumnRef": { - "fields": [{"String": {"sval": "name"}}], - "location": 11 // matches "name" token at position 11-15 - } - }, - "location": 11 - } - } - ], - "fromClause": [ - { - "RangeVar": { - "relname": "users", - "location": 21 // matches "users" token at position 21-26 - } - } - ], - "whereClause": { - "A_Expr": { - "lexpr": { - "ColumnRef": { - "fields": [{"String": {"sval": "active"}}], - "location": 33 // matches "active" token at position 33-39 - } - }, - "rexpr": { - "A_Const": { - "boolval": {"boolval": true}, - "location": 42 // matches "true" token at position 42-46 - } - }, - "location": 40 // matches "=" token at position 40-41 - } - } - } -} -``` - -### Token-to-AST Mapping Function - -Here's a utility function to map between scan tokens and AST nodes: - -```typescript -import { scan, parse } from 'libpg-query'; - -interface TokenASTMapping { - token: ScanToken; - astNodes: Array<{path: string; node: any}>; -} - -async function mapTokensToAST(sql: string): Promise { - const [scanResult, parseResult] = await Promise.all([ - scan(sql), - parse(sql) - ]); - - // Collect all AST nodes with locations - const astNodes: Array<{path: string; location: number; node: any}> = []; - - function traverse(obj: any, path: string = '') { - if (obj && typeof obj === 'object') { - if (typeof obj.location === 'number') { - astNodes.push({path, location: obj.location, node: obj}); - } - - for (const [key, value] of Object.entries(obj)) { - const newPath = path ? `${path}.${key}` : key; - traverse(value, newPath); - } - } - } - - traverse(parseResult); - - // Map tokens to AST nodes - return scanResult.tokens.map(token => { - const matchingNodes = astNodes.filter(astNode => { - // AST location points to start of token - return astNode.location >= token.start && astNode.location < token.end; - }); - - return { - token, - astNodes: matchingNodes.map(n => ({path: n.path, node: n.node})) - }; - }); -} - -// Usage -const mappings = await mapTokensToAST('SELECT id FROM users WHERE active = true'); -mappings.forEach(({token, astNodes}) => { - console.log(`Token "${token.text}" (${token.start}-${token.end}):`); - astNodes.forEach(({path, node}) => { - console.log(` AST: ${path} at location ${node.location}`); - }); -}); -``` - -## Use Cases - -### 1. Syntax Highlighting - -Use scan results to apply syntax highlighting in code editors: - -```typescript -function applySyntaxHighlighting(sql: string, tokens: ScanToken[]): string { - let highlighted = ''; - let lastEnd = 0; - - for (const token of tokens) { - // Add any whitespace between tokens - highlighted += sql.substring(lastEnd, token.start); - - // Apply highlighting based on token type - const cssClass = getHighlightClass(token); - highlighted += `${token.text}`; - - lastEnd = token.end; - } - - // Add remaining SQL - highlighted += sql.substring(lastEnd); - - return highlighted; -} - -function getHighlightClass(token: ScanToken): string { - if (token.keywordName === 'RESERVED_KEYWORD') return 'sql-keyword'; - if (token.tokenName === 'SCONST') return 'sql-string'; - if (token.tokenName === 'ICONST' || token.tokenName === 'FCONST') return 'sql-number'; - if (token.tokenName === 'PARAM') return 'sql-parameter'; - if (token.tokenName === 'SQL_COMMENT' || token.tokenName === 'C_COMMENT') return 'sql-comment'; - return 'sql-default'; -} -``` - -### 2. Parameter Extraction - -Extract all parameters from a query: - -```typescript -function extractParameters(sql: string): Array<{param: string; position: number}> { - const result = scanSync(sql); - return result.tokens - .filter(token => token.tokenName === 'PARAM') - .map(token => ({ - param: token.text, - position: token.start - })); -} - -const params = extractParameters('SELECT * FROM users WHERE id = $1 AND name = $2'); -// Returns: [{param: '$1', position: 38}, {param: '$2', position: 52}] -``` - -### 3. Query Complexity Analysis - -Analyze query complexity based on token types: - -```typescript -function analyzeQueryComplexity(sql: string): { - joins: number; - subqueries: number; - parameters: number; - aggregates: string[]; - totalTokens: number; -} { - const result = scanSync(sql); - const tokens = result.tokens; - - const joins = tokens.filter(t => - t.text.toUpperCase() === 'JOIN' || - t.text.toUpperCase() === 'INNER' || - t.text.toUpperCase() === 'LEFT' || - t.text.toUpperCase() === 'RIGHT' - ).length; - - const subqueries = tokens.filter(t => t.text.toUpperCase() === 'SELECT').length - 1; - - const parameters = tokens.filter(t => t.tokenName === 'PARAM').length; - - const aggregates = tokens - .filter(t => ['COUNT', 'SUM', 'AVG', 'MIN', 'MAX'].includes(t.text.toUpperCase())) - .map(t => t.text.toUpperCase()); - - return { - joins, - subqueries: Math.max(0, subqueries), - parameters, - aggregates, - totalTokens: tokens.length - }; -} -``` - -### 4. SQL Formatting - -Use token positions for intelligent SQL formatting: - -```typescript -function formatSQL(sql: string): string { - const result = scanSync(sql); - let formatted = ''; - let indentLevel = 0; - let lastEnd = 0; - - for (const token of result.tokens) { - // Add whitespace between tokens - const gap = sql.substring(lastEnd, token.start); - - // Format based on token type - if (token.text.toUpperCase() === 'SELECT') { - formatted += '\\n' + ' '.repeat(indentLevel) + token.text; - } else if (token.text.toUpperCase() === 'FROM') { - formatted += '\\n' + ' '.repeat(indentLevel) + token.text; - } else if (token.text.toUpperCase() === 'WHERE') { - formatted += '\\n' + ' '.repeat(indentLevel) + token.text; - } else if (token.text === '(') { - formatted += token.text; - indentLevel++; - } else if (token.text === ')') { - indentLevel--; - formatted += token.text; - } else { - formatted += (gap.trim() ? ' ' : '') + token.text; - } - - lastEnd = token.end; - } - - return formatted.trim(); -} -``` - -## Performance Considerations - -- Scanning is generally faster than full parsing -- For large SQL strings, consider streaming or chunked processing -- Token positions are 0-based and use exclusive end positions -- The scan operation is stateless and thread-safe - -## Error Handling - -The scan API is more permissive than the parse API and will attempt to tokenize even malformed SQL: - -```typescript -try { - const result = scanSync('SELECT * FROM invalid$$$'); - // May still return tokens for recognizable parts - console.log(result.tokens); -} catch (error) { - console.error('Scan error:', error.message); -} -``` - -## Integration with Other Tools - -### ESLint Rules -Create custom ESLint rules for SQL: - -```typescript -function createSQLLintRule() { - return { - meta: { - type: 'problem', - docs: { description: 'Detect SQL injection risks' } - }, - create(context) { - return { - TemplateLiteral(node) { - if (node.quasiSConst) { - const sql = node.quasiSConst[0].value.raw; - const tokens = scanSync(sql); - - // Check for unparameterized string concatenation - const hasStringLiterals = tokens.some(t => t.tokenName === 'SCONST'); - const hasParameters = tokens.some(t => t.tokenName === 'PARAM'); - - if (hasStringLiterals && !hasParameters) { - context.report({ - node, - message: 'Potential SQL injection: use parameterized queries' - }); - } - } - } - }; - } - }; -} -``` - -### Database Migration Tools -Analyze schema changes: - -```typescript -function detectSchemaChanges(oldSQL: string, newSQL: string): string[] { - const oldTokens = scanSync(oldSQL); - const newTokens = scanSync(newSQL); - - const changes: string[] = []; - - // Detect table name changes - const oldTables = extractTableNames(oldTokens); - const newTables = extractTableNames(newTokens); - - const addedTables = newTables.filter(t => !oldTables.includes(t)); - const removedTables = oldTables.filter(t => !newTables.includes(t)); - - changes.push(...addedTables.map(t => `Added table: ${t}`)); - changes.push(...removedTables.map(t => `Removed table: ${t}`)); - - return changes; -} -``` - -This comprehensive relationship between scan tokens and AST locations enables powerful SQL analysis, transformation, and tooling capabilities. \ No newline at end of file diff --git a/full/18/libpg_query.md b/full/18/libpg_query.md deleted file mode 100644 index 2b4e4e8b..00000000 --- a/full/18/libpg_query.md +++ /dev/null @@ -1,558 +0,0 @@ -# libpg_query API Documentation - -This document provides comprehensive documentation for the libpg_query API, focusing on the core parsing, scanning, and deparsing functionality. - -## Overview - -libpg_query is a C library that provides PostgreSQL SQL parsing functionality. It exposes the PostgreSQL parser as a standalone library, allowing you to parse SQL statements into parse trees, scan for tokens, and deparse parse trees back into SQL. - -## Core Data Structures - -### PgQueryError -```c -typedef struct { - char* message; // exception message - char* funcname; // source function of exception (e.g. SearchSysCache) - char* filename; // source of exception (e.g. parse.l) - int lineno; // source of exception (e.g. 104) - int cursorpos; // char in query at which exception occurred - char* context; // additional context (optional, can be NULL) -} PgQueryError; -``` - -### PgQueryProtobuf -```c -typedef struct { - size_t len; - char* data; -} PgQueryProtobuf; -``` - -### Parser Options -```c -typedef enum { - PG_QUERY_PARSE_DEFAULT = 0, - PG_QUERY_PARSE_TYPE_NAME, - PG_QUERY_PARSE_PLPGSQL_EXPR, - PG_QUERY_PARSE_PLPGSQL_ASSIGN1, - PG_QUERY_PARSE_PLPGSQL_ASSIGN2, - PG_QUERY_PARSE_PLPGSQL_ASSIGN3 -} PgQueryParseMode; -``` - -### Parser Option Flags -- `PG_QUERY_DISABLE_BACKSLASH_QUOTE` (16) - backslash_quote = off -- `PG_QUERY_DISABLE_STANDARD_CONFORMING_STRINGS` (32) - standard_conforming_strings = off -- `PG_QUERY_DISABLE_ESCAPE_STRING_WARNING` (64) - escape_string_warning = off - -## Core API Functions - -### Scanning Functions - -#### pg_query_scan -```c -PgQueryScanResult pg_query_scan(const char* input); -``` -**Description**: Scans SQL input and returns tokens in protobuf format. - -**Parameters**: -- `input`: SQL string to scan - -**Returns**: `PgQueryScanResult` containing: -- `pbuf`: Protobuf data with scan results -- `stderr_buffer`: Any stderr output during scanning -- `error`: Error information if scanning failed - -**Usage**: Use this when you need to tokenize SQL without full parsing. - -## Scanning and Token Processing - -### Working with Scan Results - -The `pg_query_scan` function returns tokens in protobuf format that need to be unpacked to access individual tokens. Here's the complete workflow: - -### Step 1: Scan SQL -```c -const char* sql = "SELECT * FROM users WHERE id = $1"; -PgQueryScanResult result = pg_query_scan(sql); - -if (result.error) { - printf("Scan error: %s at position %d\n", - result.error->message, result.error->cursorpos); - pg_query_free_scan_result(result); - return; -} -``` - -### Step 2: Unpack Protobuf Data -```c -#include "protobuf/pg_query.pb-c.h" - -PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, // Use default allocator - result.pbuf.len, // Length of protobuf data - (void *) result.pbuf.data // Protobuf data -); - -printf("Found %zu tokens\n", scan_result->n_tokens); -``` - -### Step 3: Process Individual Tokens -```c -for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; - - // Extract token text from original SQL - int token_length = token->end - token->start; - char token_text[token_length + 1]; - strncpy(token_text, &sql[token->start], token_length); - token_text[token_length] = '\0'; - - // Get token type name - const ProtobufCEnumValue *token_kind = - protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, token->token); - - // Get keyword classification - const ProtobufCEnumValue *keyword_kind = - protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, token->keyword_kind); - - printf("Token %zu: \"%s\" [%d-%d] Type: %s, Keyword: %s\n", - i, token_text, token->start, token->end, - token_kind->name, keyword_kind->name); -} -``` - -### Step 4: Clean Up Memory -```c -// Free the unpacked protobuf data -pg_query__scan_result__free_unpacked(scan_result, NULL); - -// Free the original scan result -pg_query_free_scan_result(result); -``` - -## Token Structure Details - -### PgQuery__ScanResult Structure -```c -struct PgQuery__ScanResult { - ProtobufCMessage base; - int32_t version; // Protocol version - size_t n_tokens; // Number of tokens - PgQuery__ScanToken **tokens; // Array of token pointers -}; -``` - -### PgQuery__ScanToken Structure -```c -struct PgQuery__ScanToken { - ProtobufCMessage base; - int32_t start; // Starting position in SQL string - int32_t end; // Ending position in SQL string - PgQuery__Token token; // Token type enum - PgQuery__KeywordKind keyword_kind; // Keyword classification -}; -``` - -## Token Types and Classifications - -### Keyword Classifications (PgQuery__KeywordKind) -- `PG_QUERY__KEYWORD_KIND__NO_KEYWORD` (0) - Not a keyword -- `PG_QUERY__KEYWORD_KIND__UNRESERVED_KEYWORD` (1) - Unreserved keyword -- `PG_QUERY__KEYWORD_KIND__COL_NAME_KEYWORD` (2) - Column name keyword -- `PG_QUERY__KEYWORD_KIND__TYPE_FUNC_NAME_KEYWORD` (3) - Type/function name keyword -- `PG_QUERY__KEYWORD_KIND__RESERVED_KEYWORD` (4) - Reserved keyword - -### Common Token Types (PgQuery__Token) -**Special/Control Tokens:** -- `PG_QUERY__TOKEN__NUL` (0) - Null token - -**Single-Character Operators:** -- `PG_QUERY__TOKEN__ASCII_40` (40) - "(" -- `PG_QUERY__TOKEN__ASCII_41` (41) - ")" -- `PG_QUERY__TOKEN__ASCII_42` (42) - "*" -- `PG_QUERY__TOKEN__ASCII_44` (44) - "," -- `PG_QUERY__TOKEN__ASCII_59` (59) - ";" -- `PG_QUERY__TOKEN__ASCII_61` (61) - "=" - -**Named Lexical Tokens:** -- `PG_QUERY__TOKEN__IDENT` (258) - Regular identifier -- `PG_QUERY__TOKEN__SCONST` (261) - String constant -- `PG_QUERY__TOKEN__ICONST` (266) - Integer constant -- `PG_QUERY__TOKEN__FCONST` (260) - Float constant -- `PG_QUERY__TOKEN__PARAM` (267) - Parameter marker ($1, $2, etc.) - -**Multi-Character Operators:** -- `PG_QUERY__TOKEN__TYPECAST` (268) - "::" -- `PG_QUERY__TOKEN__DOT_DOT` (269) - ".." -- `PG_QUERY__TOKEN__LESS_EQUALS` (272) - "<=" -- `PG_QUERY__TOKEN__GREATER_EQUALS` (273) - ">=" -- `PG_QUERY__TOKEN__NOT_EQUALS` (274) - "!=" or "<>" - -**Common SQL Keywords:** -- `PG_QUERY__TOKEN__SELECT` - SELECT keyword -- `PG_QUERY__TOKEN__FROM` - FROM keyword -- `PG_QUERY__TOKEN__WHERE` - WHERE keyword -- `PG_QUERY__TOKEN__INSERT` - INSERT keyword -- `PG_QUERY__TOKEN__UPDATE` - UPDATE keyword -- `PG_QUERY__TOKEN__DELETE` - DELETE keyword - -**Comments:** -- `PG_QUERY__TOKEN__SQL_COMMENT` (275) - SQL-style comment (-- comment) -- `PG_QUERY__TOKEN__C_COMMENT` (276) - C-style comment (/* comment */) - -## Protobuf Helper Functions - -### Unpacking Functions -```c -// Unpack scan result -PgQuery__ScanResult *pg_query__scan_result__unpack( - ProtobufCAllocator *allocator, // NULL for default - size_t len, // Length of data - const uint8_t *data // Protobuf data -); - -// Free unpacked scan result -void pg_query__scan_result__free_unpacked( - PgQuery__ScanResult *message, // Message to free - ProtobufCAllocator *allocator // NULL for default -); -``` - -### Enum Value Lookup Functions -```c -// Get token type name -const ProtobufCEnumValue *protobuf_c_enum_descriptor_get_value( - const ProtobufCEnumDescriptor *desc, // &pg_query__token__descriptor - int value // token->token -); - -// Get keyword kind name -const ProtobufCEnumValue *protobuf_c_enum_descriptor_get_value( - const ProtobufCEnumDescriptor *desc, // &pg_query__keyword_kind__descriptor - int value // token->keyword_kind -); -``` - -## Complete Example: SQL Tokenizer - -```c -#include -#include -#include -#include -#include "protobuf/pg_query.pb-c.h" - -void print_tokens(const char* sql) { - PgQueryScanResult result = pg_query_scan(sql); - - if (result.error) { - printf("Error: %s at position %d\n", - result.error->message, result.error->cursorpos); - pg_query_free_scan_result(result); - return; - } - - PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, result.pbuf.len, (void *) result.pbuf.data); - - printf("SQL: %s\n", sql); - printf("Tokens (%zu):\n", scan_result->n_tokens); - - for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; - - // Extract token text - int len = token->end - token->start; - printf(" [%zu] \"%.*s\" (%d-%d) ", - i, len, &sql[token->start], token->start, token->end); - - // Get token type - const ProtobufCEnumValue *token_kind = - protobuf_c_enum_descriptor_get_value(&pg_query__token__descriptor, token->token); - printf("Type: %s", token_kind->name); - - // Get keyword classification if applicable - if (token->keyword_kind != PG_QUERY__KEYWORD_KIND__NO_KEYWORD) { - const ProtobufCEnumValue *keyword_kind = - protobuf_c_enum_descriptor_get_value(&pg_query__keyword_kind__descriptor, token->keyword_kind); - printf(", Keyword: %s", keyword_kind->name); - } - printf("\n"); - } - - pg_query__scan_result__free_unpacked(scan_result, NULL); - pg_query_free_scan_result(result); -} - -int main() { - print_tokens("SELECT * FROM users WHERE id = $1"); - print_tokens("INSERT INTO table VALUES (1, 'text', 3.14)"); - print_tokens("-- Comment\nUPDATE table SET col = col + 1"); - - pg_query_exit(); - return 0; -} -``` - -## Build Requirements - -To use scanning functionality, compile with: -```bash -gcc -I. -I./protobuf your_program.c -lpg_query -lprotobuf-c -``` - -Make sure to include: -- `pg_query.h` - Main API header -- `protobuf/pg_query.pb-c.h` - Protobuf definitions - -### Parsing Functions - -#### pg_query_parse -```c -PgQueryParseResult pg_query_parse(const char* input); -``` -**Description**: Parses SQL input into a JSON parse tree. - -**Parameters**: -- `input`: SQL string to parse - -**Returns**: `PgQueryParseResult` containing: -- `parse_tree`: JSON representation of the parse tree -- `stderr_buffer`: Any stderr output during parsing -- `error`: Error information if parsing failed - -#### pg_query_parse_opts -```c -PgQueryParseResult pg_query_parse_opts(const char* input, int parser_options); -``` -**Description**: Parses SQL input with custom parser options. - -**Parameters**: -- `input`: SQL string to parse -- `parser_options`: Bitwise OR of parser options and flags - -**Returns**: Same as `pg_query_parse` - -#### pg_query_parse_protobuf -```c -PgQueryProtobufParseResult pg_query_parse_protobuf(const char* input); -``` -**Description**: Parses SQL input into protobuf format parse tree. - -**Parameters**: -- `input`: SQL string to parse - -**Returns**: `PgQueryProtobufParseResult` containing: -- `parse_tree`: Protobuf representation of the parse tree -- `stderr_buffer`: Any stderr output during parsing -- `error`: Error information if parsing failed - -#### pg_query_parse_protobuf_opts -```c -PgQueryProtobufParseResult pg_query_parse_protobuf_opts(const char* input, int parser_options); -``` -**Description**: Parses SQL input into protobuf format with custom options. - -**Parameters**: -- `input`: SQL string to parse -- `parser_options`: Bitwise OR of parser options and flags - -**Returns**: Same as `pg_query_parse_protobuf` - -#### pg_query_parse_plpgsql -```c -PgQueryPlpgsqlParseResult pg_query_parse_plpgsql(const char* input); -``` -**Description**: Parses PL/pgSQL code. - -**Parameters**: -- `input`: PL/pgSQL code to parse - -**Returns**: `PgQueryPlpgsqlParseResult` containing: -- `plpgsql_funcs`: JSON representation of PL/pgSQL functions -- `error`: Error information if parsing failed - -### Deparsing Functions - -#### pg_query_deparse_protobuf -```c -PgQueryDeparseResult pg_query_deparse_protobuf(PgQueryProtobuf parse_tree); -``` -**Description**: Converts a protobuf parse tree back into SQL. - -**Parameters**: -- `parse_tree`: Protobuf parse tree to deparse - -**Returns**: `PgQueryDeparseResult` containing: -- `query`: Deparsed SQL string -- `error`: Error information if deparsing failed - -**Usage**: Use this to convert parse trees back to SQL, useful for query transformation. - -### Utility Functions - -#### pg_query_normalize -```c -PgQueryNormalizeResult pg_query_normalize(const char* input); -``` -**Description**: Normalizes a SQL query by removing comments and standardizing formatting. - -**Parameters**: -- `input`: SQL string to normalize - -**Returns**: `PgQueryNormalizeResult` containing: -- `normalized_query`: Normalized SQL string -- `error`: Error information if normalization failed - -#### pg_query_normalize_utility -```c -PgQueryNormalizeResult pg_query_normalize_utility(const char* input); -``` -**Description**: Normalizes utility statements (DDL, etc.). - -**Parameters**: -- `input`: SQL string to normalize - -**Returns**: Same as `pg_query_normalize` - -#### pg_query_fingerprint -```c -PgQueryFingerprintResult pg_query_fingerprint(const char* input); -``` -**Description**: Generates a fingerprint for a SQL query. - -**Parameters**: -- `input`: SQL string to fingerprint - -**Returns**: `PgQueryFingerprintResult` containing: -- `fingerprint`: 64-bit fingerprint hash -- `fingerprint_str`: String representation of fingerprint -- `stderr_buffer`: Any stderr output -- `error`: Error information if fingerprinting failed - -#### pg_query_fingerprint_opts -```c -PgQueryFingerprintResult pg_query_fingerprint_opts(const char* input, int parser_options); -``` -**Description**: Generates a fingerprint with custom parser options. - -**Parameters**: -- `input`: SQL string to fingerprint -- `parser_options`: Bitwise OR of parser options and flags - -**Returns**: Same as `pg_query_fingerprint` - -### Statement Splitting Functions - -#### pg_query_split_with_scanner -```c -PgQuerySplitResult pg_query_split_with_scanner(const char *input); -``` -**Description**: Splits multi-statement SQL using the scanner. Use when statements may contain parse errors. - -**Parameters**: -- `input`: SQL string containing multiple statements - -**Returns**: `PgQuerySplitResult` containing: -- `stmts`: Array of statement locations and lengths -- `n_stmts`: Number of statements found -- `stderr_buffer`: Any stderr output -- `error`: Error information if splitting failed - -#### pg_query_split_with_parser -```c -PgQuerySplitResult pg_query_split_with_parser(const char *input); -``` -**Description**: Splits multi-statement SQL using the parser (recommended for better accuracy). - -**Parameters**: -- `input`: SQL string containing multiple statements - -**Returns**: Same as `pg_query_split_with_scanner` - -## Memory Management - -### Cleanup Functions -All result structures must be freed using their corresponding cleanup functions: - -```c -void pg_query_free_normalize_result(PgQueryNormalizeResult result); -void pg_query_free_scan_result(PgQueryScanResult result); -void pg_query_free_parse_result(PgQueryParseResult result); -void pg_query_free_split_result(PgQuerySplitResult result); -void pg_query_free_deparse_result(PgQueryDeparseResult result); -void pg_query_free_protobuf_parse_result(PgQueryProtobufParseResult result); -void pg_query_free_plpgsql_parse_result(PgQueryPlpgsqlParseResult result); -void pg_query_free_fingerprint_result(PgQueryFingerprintResult result); -``` - -### Global Cleanup -```c -void pg_query_exit(void); -``` -**Description**: Optional cleanup of top-level memory context. Automatically done for threads that exit. - -## Error Handling - -All functions return result structures that include an `error` field. Always check this field before using other result data: - -```c -PgQueryParseResult result = pg_query_parse(sql); -if (result.error) { - printf("Parse error: %s\n", result.error->message); - printf("Location: %s:%d\n", result.error->filename, result.error->lineno); - if (result.error->cursorpos > 0) { - printf("Position: %d\n", result.error->cursorpos); - } -} else { - // Use result.parse_tree -} -pg_query_free_parse_result(result); -``` - -## Example Usage - -### Basic Parsing -```c -#include "pg_query.h" - -const char* sql = "SELECT * FROM users WHERE id = $1"; -PgQueryParseResult result = pg_query_parse(sql); - -if (result.error) { - printf("Error: %s\n", result.error->message); -} else { - printf("Parse tree: %s\n", result.parse_tree); -} - -pg_query_free_parse_result(result); -``` - -### Parse and Deparse Cycle -```c -// Parse to protobuf -PgQueryProtobufParseResult parse_result = pg_query_parse_protobuf(sql); -if (!parse_result.error) { - // Deparse back to SQL - PgQueryDeparseResult deparse_result = pg_query_deparse_protobuf(parse_result.parse_tree); - if (!deparse_result.error) { - printf("Deparsed: %s\n", deparse_result.query); - } - pg_query_free_deparse_result(deparse_result); -} -pg_query_free_protobuf_parse_result(parse_result); -``` - -## Version Information - -- PostgreSQL Version: 17.4 (PG_VERSION_NUM: 170004) -- Major Version: 17 - -## Notes - -- The library is thread-safe -- Always free result structures to avoid memory leaks -- Use protobuf format for better performance when doing parse/deparse cycles -- Scanner-based splitting is more robust for malformed SQL -- Parser-based splitting is more accurate for well-formed SQL \ No newline at end of file diff --git a/full/18/libpg_query/protobuf/.gitkeep b/full/18/libpg_query/protobuf/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/full/18/package.json b/full/18/package.json deleted file mode 100644 index a06bb5a2..00000000 --- a/full/18/package.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "name": "@libpg-query/full18", - "version": "18.0.0", - "description": "The real PostgreSQL query parser", - "homepage": "https://github.com/constructive-io/libpg-query-node", - "main": "./wasm/index.cjs", - "module": "./wasm/index.js", - "typings": "./wasm/index.d.ts", - "publishConfig": { - "access": "public" - }, - "x-publish": { - "publishName": "@libpg-query/parser", - "pgVersion": "18", - "distTag": "pg18", - "libpgQueryTag": "18-constructive" - }, - "files": [ - "wasm/*" - ], - "scripts": { - "clean": "pnpm wasm:clean && rimraf cjs esm", - "build:js": "node scripts/build.js", - "build": "pnpm clean; pnpm wasm:build; pnpm build:js", - "publish:pkg": "node ../../scripts/publish-single-version.js", - "wasm:make": "docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk emmake make", - "wasm:build": "pnpm wasm:make build", - "wasm:rebuild": "pnpm wasm:make rebuild", - "wasm:clean": "pnpm wasm:make clean", - "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js", - "yamlize": "node ./scripts/yamlize.js" - }, - "author": "Constructive ", - "license": "MIT", - "repository": { - "type": "git", - "url": "git://github.com/constructive-io/libpg-query-node.git" - }, - "devDependencies": { - "@yamlize/cli": "^0.8.0" - }, - "dependencies": { - "@pgsql/types": "^18.0.0" - }, - "keywords": [ - "sql", - "postgres", - "postgresql", - "pg", - "query", - "plpgsql", - "database" - ] -} diff --git a/full/18/scripts/build.js b/full/18/scripts/build.js deleted file mode 100644 index 55d83240..00000000 --- a/full/18/scripts/build.js +++ /dev/null @@ -1,38 +0,0 @@ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Run TypeScript compilation -console.log('Compiling TypeScript...'); -execSync('tsc', { stdio: 'inherit' }); -execSync('tsc -p tsconfig.esm.json', { stdio: 'inherit' }); - -// Rename files to have correct extensions -const wasmDir = path.join(__dirname, '../wasm'); -const cjsDir = path.join(__dirname, '../cjs'); -const esmDir = path.join(__dirname, '../esm'); - -// Ensure wasm directory exists -if (!fs.existsSync(wasmDir)) { - fs.mkdirSync(wasmDir, { recursive: true }); -} - -// Rename CommonJS files -fs.renameSync( - path.join(cjsDir, 'index.js'), - path.join(wasmDir, 'index.cjs') -); - -// Rename ESM files -fs.renameSync( - path.join(esmDir, 'index.js'), - path.join(wasmDir, 'index.js') -); - -// Rename declaration files -fs.renameSync( - path.join(cjsDir, 'index.d.ts'), - path.join(wasmDir, 'index.d.ts') -); - -console.log('Build completed successfully!'); \ No newline at end of file diff --git a/full/18/src/index.ts b/full/18/src/index.ts deleted file mode 100644 index b616d923..00000000 --- a/full/18/src/index.ts +++ /dev/null @@ -1,484 +0,0 @@ -import { ParseResult } from "@pgsql/types"; -export * from "@pgsql/types"; - -export interface ScanToken { - start: number; - end: number; - text: string; - tokenType: number; - tokenName: string; - keywordKind: number; - keywordName: string; -} - -export interface ScanResult { - version: number; - tokens: ScanToken[]; -} - -export interface SqlErrorDetails { - message: string; - cursorPosition: number; - fileName?: string; - functionName?: string; - lineNumber?: number; - context?: string; -} - -export class SqlError extends Error { - sqlDetails?: SqlErrorDetails; - - constructor(message: string, details?: SqlErrorDetails) { - super(message); - this.name = 'SqlError'; - this.sqlDetails = details; - } -} - -export function hasSqlDetails(error: unknown): error is SqlError { - return error instanceof SqlError && error.sqlDetails !== undefined; -} - -export function formatSqlError( - error: SqlError, - query: string, - options: { - showPosition?: boolean; - showQuery?: boolean; - color?: boolean; - maxQueryLength?: number; - } = {} -): string { - const { - showPosition = true, - showQuery = true, - color = false, - maxQueryLength - } = options; - - const lines: string[] = []; - - // ANSI color codes - const red = color ? '\x1b[31m' : ''; - const yellow = color ? '\x1b[33m' : ''; - const reset = color ? '\x1b[0m' : ''; - - // Add error message - lines.push(`${red}Error: ${error.message}${reset}`); - - // Add SQL details if available - if (error.sqlDetails) { - const { cursorPosition, fileName, functionName, lineNumber } = error.sqlDetails; - - if (cursorPosition !== undefined && cursorPosition >= 0) { - lines.push(`Position: ${cursorPosition}`); - } - - if (fileName || functionName || lineNumber) { - const details = []; - if (fileName) details.push(`file: ${fileName}`); - if (functionName) details.push(`function: ${functionName}`); - if (lineNumber) details.push(`line: ${lineNumber}`); - lines.push(`Source: ${details.join(', ')}`); - } - - // Show query with position marker - if (showQuery && showPosition && cursorPosition !== undefined && cursorPosition >= 0) { - let displayQuery = query; - let adjustedPosition = cursorPosition; - - // Truncate if needed - if (maxQueryLength && query.length > maxQueryLength) { - const start = Math.max(0, cursorPosition - Math.floor(maxQueryLength / 2)); - const end = Math.min(query.length, start + maxQueryLength); - displayQuery = (start > 0 ? '...' : '') + - query.substring(start, end) + - (end < query.length ? '...' : ''); - // Adjust cursor position for truncation - adjustedPosition = cursorPosition - start + (start > 0 ? 3 : 0); - } - - lines.push(displayQuery); - lines.push(' '.repeat(adjustedPosition) + `${yellow}^${reset}`); - } - } else if (showQuery) { - // No SQL details, just show the query if requested - let displayQuery = query; - if (maxQueryLength && query.length > maxQueryLength) { - displayQuery = query.substring(0, maxQueryLength) + '...'; - } - lines.push(`Query: ${displayQuery}`); - } - - return lines.join('\n'); -} - -// @ts-ignore -import PgQueryModule from './libpg-query.js'; - -interface WasmModule { - _malloc: (size: number) => number; - _free: (ptr: number) => void; - _wasm_free_string: (ptr: number) => void; - _wasm_parse_query: (queryPtr: number) => number; - _wasm_parse_query_raw: (queryPtr: number) => number; - _wasm_free_parse_result: (ptr: number) => void; - _wasm_parse_plpgsql: (queryPtr: number) => number; - _wasm_fingerprint: (queryPtr: number) => number; - _wasm_normalize_query: (queryPtr: number) => number; - _wasm_scan: (queryPtr: number) => number; - lengthBytesUTF8: (str: string) => number; - stringToUTF8: (str: string, ptr: number, len: number) => void; - UTF8ToString: (ptr: number) => string; - getValue: (ptr: number, type: string) => number; -} - -let wasmModule: WasmModule; - -const initPromise = PgQueryModule().then((module: WasmModule) => { - wasmModule = module; -}); - -function ensureLoaded() { - if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); -} - -export async function loadModule(): Promise { - if (!wasmModule) { - await initPromise; - } -} - -function awaitInit Promise>(fn: T): T { - return (async (...args: Parameters) => { - await initPromise; - return fn(...args); - }) as T; -} - -function stringToPtr(str: string): number { - ensureLoaded(); - if (typeof str !== 'string') { - throw new TypeError(`Expected a string, got ${typeof str}`); - } - const len = wasmModule.lengthBytesUTF8(str) + 1; - const ptr = wasmModule._malloc(len); - try { - wasmModule.stringToUTF8(str, ptr, len); - return ptr; - } catch (error) { - wasmModule._free(ptr); - throw error; - } -} - -function ptrToString(ptr: number): string { - ensureLoaded(); - if (typeof ptr !== 'number') { - throw new TypeError(`Expected a number, got ${typeof ptr}`); - } - return wasmModule.UTF8ToString(ptr); -} - -export const parse = awaitInit(async (query: string): Promise => { - // Input validation - if (query === null || query === undefined) { - throw new Error('Query cannot be null or undefined'); - } - - if (query === '') { - throw new Error('Query cannot be empty'); - } - - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_parse_query_raw(queryPtr); - if (!resultPtr) { - throw new Error('Failed to parse query: memory allocation failed'); - } - - // Read the PgQueryParseResult struct - const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); - const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); - const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); - - if (errorPtr) { - // Read PgQueryError struct - const messagePtr = wasmModule.getValue(errorPtr, 'i32'); - const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); - const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); - const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); - const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); - - const message = messagePtr ? wasmModule.UTF8ToString(messagePtr) : 'Unknown error'; - const funcname = funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined; - const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : undefined; - - throw new SqlError(message, { - message, - cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, // Convert to 0-based - fileName: filename, - functionName: funcname, - lineNumber: lineno > 0 ? lineno : undefined - }); - } - - if (!parseTreePtr) { - throw new Error('No parse tree generated'); - } - - const parseTreeStr = wasmModule.UTF8ToString(parseTreePtr); - return JSON.parse(parseTreeStr); - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_parse_result(resultPtr); - } - } -}); - -export const parsePlPgSQL = awaitInit(async (query: string): Promise => { - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return JSON.parse(resultStr); - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -}); - -export const fingerprint = awaitInit(async (query: string): Promise => { - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_fingerprint(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return resultStr; - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -}); - -export const normalize = awaitInit(async (query: string): Promise => { - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_normalize_query(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return resultStr; - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -}); - -// Sync versions -export function parseSync(query: string): ParseResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } - - // Input validation - if (query === null || query === undefined) { - throw new Error('Query cannot be null or undefined'); - } - - if (query === '') { - throw new Error('Query cannot be empty'); - } - - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_parse_query_raw(queryPtr); - if (!resultPtr) { - throw new Error('Failed to parse query: memory allocation failed'); - } - - // Read the PgQueryParseResult struct - const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); - const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); - const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); - - if (errorPtr) { - // Read PgQueryError struct - const messagePtr = wasmModule.getValue(errorPtr, 'i32'); - const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); - const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); - const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); - const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); - - const message = messagePtr ? wasmModule.UTF8ToString(messagePtr) : 'Unknown error'; - const funcname = funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined; - const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : undefined; - - throw new SqlError(message, { - message, - cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, // Convert to 0-based - fileName: filename, - functionName: funcname, - lineNumber: lineno > 0 ? lineno : undefined - }); - } - - if (!parseTreePtr) { - throw new Error('No parse tree generated'); - } - - const parseTreeStr = wasmModule.UTF8ToString(parseTreePtr); - return JSON.parse(parseTreeStr); - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_parse_result(resultPtr); - } - } -} - -export function parsePlPgSQLSync(query: string): ParseResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return JSON.parse(resultStr); - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -} - -export function fingerprintSync(query: string): string { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_fingerprint(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return resultStr; - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -} - -export function normalizeSync(query: string): string { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_normalize_query(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return resultStr; - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -} - -export const scan = awaitInit(async (query: string): Promise => { - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_scan(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return JSON.parse(resultStr); - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -}); - -export function scanSync(query: string): ScanResult { - if (!wasmModule) { - throw new Error('WASM module not initialized. Call loadModule() first.'); - } - const queryPtr = stringToPtr(query); - let resultPtr = 0; - - try { - resultPtr = wasmModule._wasm_scan(queryPtr); - const resultStr = ptrToString(resultPtr); - - if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { - throw new Error(resultStr); - } - - return JSON.parse(resultStr); - } finally { - wasmModule._free(queryPtr); - if (resultPtr) { - wasmModule._wasm_free_string(resultPtr); - } - } -} \ No newline at end of file diff --git a/full/18/src/libpg-query.d.ts b/full/18/src/libpg-query.d.ts deleted file mode 100644 index a7c80962..00000000 --- a/full/18/src/libpg-query.d.ts +++ /dev/null @@ -1,17 +0,0 @@ -declare module './libpg-query.js' { - interface WasmModule { - _malloc: (size: number) => number; - _free: (ptr: number) => void; - _wasm_free_string: (ptr: number) => void; - _wasm_parse_query: (queryPtr: number) => number; - _wasm_parse_plpgsql: (queryPtr: number) => number; - _wasm_fingerprint: (queryPtr: number) => number; - _wasm_normalize_query: (queryPtr: number) => number; - lengthBytesUTF8: (str: string) => number; - stringToUTF8: (str: string, ptr: number, len: number) => void; - UTF8ToString: (ptr: number) => string; - } - - const PgQueryModule: () => Promise; - export default PgQueryModule; -} \ No newline at end of file diff --git a/full/18/src/proto.d.ts b/full/18/src/proto.d.ts deleted file mode 100644 index 55a504f3..00000000 --- a/full/18/src/proto.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -declare module '../proto.js' { - export namespace pg_query { - interface ParseResult { - version: number; - stmts: Statement[]; - } - - interface Statement { - stmt_type: string; - stmt_len: number; - stmt_location: number; - query: string; - } - - class ParseResult { - static fromObject(obj: ParseResult): ParseResult; - static encode(msg: ParseResult): { finish(): Uint8Array }; - } - } -} \ No newline at end of file diff --git a/full/18/src/wasm_wrapper.c b/full/18/src/wasm_wrapper.c deleted file mode 100644 index 3844d960..00000000 --- a/full/18/src/wasm_wrapper.c +++ /dev/null @@ -1,395 +0,0 @@ -#include "pg_query.h" -#include "protobuf/pg_query.pb-c.h" -#include -#include -#include -#include - -static int validate_input(const char* input) { - return input != NULL && strlen(input) > 0; -} - -static char* safe_strdup(const char* str) { - if (!str) return NULL; - char* result = strdup(str); - if (!result) { - return NULL; - } - return result; -} - -static void* safe_malloc(size_t size) { - void* ptr = malloc(size); - if (!ptr && size > 0) { - return NULL; - } - return ptr; -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_parse_query(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryParseResult result = pg_query_parse(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_parse_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* parse_tree = safe_strdup(result.parse_tree); - pg_query_free_parse_result(result); - return parse_tree; -} - -EMSCRIPTEN_KEEPALIVE -PgQueryParseResult* wasm_parse_query_raw(const char* input) { - PgQueryParseResult* result = (PgQueryParseResult*)malloc(sizeof(PgQueryParseResult)); - if (!result) { - return NULL; - } - - *result = pg_query_parse(input); - return result; -} - -EMSCRIPTEN_KEEPALIVE -void wasm_free_parse_result(PgQueryParseResult* result) { - if (result) { - pg_query_free_parse_result(*result); - free(result); - } -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_parse_plpgsql(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_plpgsql_parse_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - if (!result.plpgsql_funcs) { - pg_query_free_plpgsql_parse_result(result); - return safe_strdup("{\"plpgsql_funcs\":[]}"); - } - - size_t funcs_len = strlen(result.plpgsql_funcs); - size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; - char* wrapped_result = safe_malloc(json_len); - - if (!wrapped_result) { - pg_query_free_plpgsql_parse_result(result); - return safe_strdup("Memory allocation failed"); - } - - int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); - - if (written >= json_len) { - free(wrapped_result); - pg_query_free_plpgsql_parse_result(result); - return safe_strdup("Buffer overflow prevented"); - } - - pg_query_free_plpgsql_parse_result(result); - return wrapped_result; -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_fingerprint(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryFingerprintResult result = pg_query_fingerprint(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_fingerprint_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* fingerprint_str = safe_strdup(result.fingerprint_str); - pg_query_free_fingerprint_result(result); - return fingerprint_str; -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_normalize_query(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryNormalizeResult result = pg_query_normalize(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_normalize_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - char* normalized = safe_strdup(result.normalized_query); - pg_query_free_normalize_result(result); - - if (!normalized) { - return safe_strdup("Memory allocation failed"); - } - - return normalized; -} - - - - - -typedef struct { - int has_error; - char* message; - char* funcname; - char* filename; - int lineno; - int cursorpos; - char* context; - char* data; - size_t data_len; -} WasmDetailedResult; - -EMSCRIPTEN_KEEPALIVE -WasmDetailedResult* wasm_parse_query_detailed(const char* input) { - WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); - if (!result) { - return NULL; - } - memset(result, 0, sizeof(WasmDetailedResult)); - - if (!validate_input(input)) { - result->has_error = 1; - result->message = safe_strdup("Invalid input: query cannot be null or empty"); - return result; - } - - PgQueryParseResult parse_result = pg_query_parse(input); - - if (parse_result.error) { - result->has_error = 1; - size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; - char* prefixed_message = safe_malloc(message_len); - if (!prefixed_message) { - result->has_error = 1; - result->message = safe_strdup("Memory allocation failed"); - pg_query_free_parse_result(parse_result); - return result; - } - snprintf(prefixed_message, message_len, - "Parse error: %s at line %d, position %d", - parse_result.error->message, - parse_result.error->lineno, - parse_result.error->cursorpos); - result->message = prefixed_message; - char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; - char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; - char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; - - result->funcname = funcname_copy; - result->filename = filename_copy; - result->lineno = parse_result.error->lineno; - result->cursorpos = parse_result.error->cursorpos; - result->context = context_copy; - } else { - result->data = safe_strdup(parse_result.parse_tree); - if (result->data) { - result->data_len = strlen(result->data); - } else { - result->has_error = 1; - result->message = safe_strdup("Memory allocation failed"); - } - } - - pg_query_free_parse_result(parse_result); - return result; -} - -EMSCRIPTEN_KEEPALIVE -void wasm_free_detailed_result(WasmDetailedResult* result) { - if (result) { - free(result->message); - free(result->funcname); - free(result->filename); - free(result->context); - free(result->data); - free(result); - } -} - -static const char* get_token_name(PgQuery__Token token_type) { - // Map some common token types to readable names - // Note: This is a simplified mapping - full enum lookup would require more complexity - switch(token_type) { - case 258: return "IDENT"; - case 261: return "SCONST"; - case 266: return "ICONST"; - case 260: return "FCONST"; - case 267: return "PARAM"; - case 40: return "ASCII_40"; // ( - case 41: return "ASCII_41"; // ) - case 42: return "ASCII_42"; // * - case 44: return "ASCII_44"; // , - case 59: return "ASCII_59"; // ; - case 61: return "ASCII_61"; // = - case 268: return "TYPECAST"; - case 272: return "LESS_EQUALS"; - case 273: return "GREATER_EQUALS"; - case 274: return "NOT_EQUALS"; - case 275: return "SQL_COMMENT"; - case 276: return "C_COMMENT"; - default: return "UNKNOWN"; - } -} - -static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { - switch(keyword_kind) { - case 0: return "NO_KEYWORD"; - case 1: return "UNRESERVED_KEYWORD"; - case 2: return "COL_NAME_KEYWORD"; - case 3: return "TYPE_FUNC_NAME_KEYWORD"; - case 4: return "RESERVED_KEYWORD"; - default: return "UNKNOWN_KEYWORD"; - } -} - -static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { - if (!scan_result || !original_sql) { - return safe_strdup("{\"version\":0,\"tokens\":[]}"); - } - - // Calculate rough JSON size estimate - size_t estimated_size = 1024 + (scan_result->n_tokens * 200); - char* json = safe_malloc(estimated_size); - if (!json) { - return safe_strdup("{\"version\":0,\"tokens\":[]}"); - } - - // Start building JSON - int pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version); - - for (size_t i = 0; i < scan_result->n_tokens; i++) { - PgQuery__ScanToken *token = scan_result->tokens[i]; - - // Extract token text from original SQL - int token_length = token->end - token->start; - if (token_length < 0) token_length = 0; // Safety check - - char* token_text = safe_malloc(token_length + 1); - if (!token_text) continue; - - if (token_length > 0) { - strncpy(token_text, &original_sql[token->start], token_length); - } - token_text[token_length] = '\0'; - - // Escape token text for JSON - char* escaped_text = safe_malloc(token_length * 2 + 1); - if (!escaped_text) { - free(token_text); - continue; - } - - int escaped_pos = 0; - for (int j = 0; j < token_length; j++) { - char c = token_text[j]; - if (c == '"' || c == '\\') { - escaped_text[escaped_pos++] = '\\'; - escaped_text[escaped_pos++] = c; - } else if (c == '\n') { - escaped_text[escaped_pos++] = '\\'; - escaped_text[escaped_pos++] = 'n'; - } else if (c == '\r') { - escaped_text[escaped_pos++] = '\\'; - escaped_text[escaped_pos++] = 'r'; - } else if (c == '\t') { - escaped_text[escaped_pos++] = '\\'; - escaped_text[escaped_pos++] = 't'; - } else { - escaped_text[escaped_pos++] = c; - } - } - escaped_text[escaped_pos] = '\0'; - - // Get token type name and keyword kind name - const char* token_name = get_token_name(token->token); - const char* keyword_name = get_keyword_name(token->keyword_kind); - - // Add comma if not first token - if (i > 0) { - pos += snprintf(json + pos, estimated_size - pos, ","); - } - - // Add token object to JSON - pos += snprintf(json + pos, estimated_size - pos, - "{\"start\":%d,\"end\":%d,\"text\":\"%s\",\"tokenType\":%d,\"tokenName\":\"%s\",\"keywordKind\":%d,\"keywordName\":\"%s\"}", - token->start, token->end, escaped_text, token->token, token_name, token->keyword_kind, keyword_name); - - free(token_text); - free(escaped_text); - - // Check if we're running out of space - if (pos >= estimated_size - 200) { - char* new_json = realloc(json, estimated_size * 2); - if (!new_json) break; - json = new_json; - estimated_size *= 2; - } - } - - // Close JSON - snprintf(json + pos, estimated_size - pos, "]}"); - - return json; -} - -EMSCRIPTEN_KEEPALIVE -char* wasm_scan(const char* input) { - if (!validate_input(input)) { - return safe_strdup("Invalid input: query cannot be null or empty"); - } - - PgQueryScanResult result = pg_query_scan(input); - - if (result.error) { - char* error_msg = safe_strdup(result.error->message); - pg_query_free_scan_result(result); - return error_msg ? error_msg : safe_strdup("Memory allocation failed"); - } - - // Unpack protobuf data - PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( - NULL, result.pbuf.len, (void *) result.pbuf.data); - - if (!scan_result) { - pg_query_free_scan_result(result); - return safe_strdup("Failed to unpack scan result"); - } - - // Convert to JSON - char* json_result = build_scan_json(scan_result, input); - - // Clean up - pg_query__scan_result__free_unpacked(scan_result, NULL); - pg_query_free_scan_result(result); - - return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); -} - -EMSCRIPTEN_KEEPALIVE -void wasm_free_string(char* str) { - free(str); -} diff --git a/full/18/test/errors.test.js b/full/18/test/errors.test.js deleted file mode 100644 index f6c0fd61..00000000 --- a/full/18/test/errors.test.js +++ /dev/null @@ -1,325 +0,0 @@ -const { describe, it, before } = require('node:test'); -const assert = require('node:assert/strict'); -const { parseSync, loadModule, formatSqlError, hasSqlDetails } = require('../wasm/index.cjs'); - -describe('Enhanced Error Handling', () => { - before(async () => { - await loadModule(); - }); - - describe('Error Details Structure', () => { - it('should include sqlDetails property on parse errors', () => { - assert.throws(() => { - parseSync('SELECT * FROM users WHERE id = @'); - }); - - try { - parseSync('SELECT * FROM users WHERE id = @'); - } catch (error) { - assert.ok('sqlDetails' in error); - assert.ok('message' in error.sqlDetails); - assert.ok('cursorPosition' in error.sqlDetails); - assert.ok('fileName' in error.sqlDetails); - assert.ok('functionName' in error.sqlDetails); - assert.ok('lineNumber' in error.sqlDetails); - } - }); - - it('should have correct cursor position (0-based)', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 32); - } - }); - - it('should identify error source file', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.fileName, 'scan.l'); - assert.equal(error.sqlDetails.functionName, 'scanner_yyerror'); - } - }); - }); - - describe('Error Position Accuracy', () => { - const positionTests = [ - { query: '@ SELECT * FROM users', expectedPos: 0, desc: 'error at start' }, - { query: 'SELECT @ FROM users', expectedPos: 9, desc: 'error after SELECT' }, - { query: 'SELECT * FROM users WHERE @ = 1', expectedPos: 28, desc: 'error after WHERE' }, - { query: 'SELECT * FROM users WHERE id = @', expectedPos: 32, desc: 'error at end' }, - { query: 'INSERT INTO users (id, name) VALUES (1, @)', expectedPos: 41, desc: 'error in VALUES' }, - { query: 'UPDATE users SET name = @ WHERE id = 1', expectedPos: 26, desc: 'error in SET' }, - { query: 'CREATE TABLE test (id INT, name @)', expectedPos: 32, desc: 'error in CREATE TABLE' }, - ]; - - positionTests.forEach(({ query, expectedPos, desc }) => { - it(`should correctly identify position for ${desc}`, () => { - try { - parseSync(query); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, expectedPos); - } - }); - }); - }); - - describe('Error Types', () => { - it('should handle unterminated string literals', () => { - try { - parseSync("SELECT * FROM users WHERE name = 'unclosed"); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('unterminated quoted string')); - assert.equal(error.sqlDetails.cursorPosition, 33); - } - }); - - it('should handle unterminated quoted identifiers', () => { - try { - parseSync('SELECT * FROM users WHERE name = "unclosed'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('unterminated quoted identifier')); - assert.equal(error.sqlDetails.cursorPosition, 33); - } - }); - - it('should handle invalid tokens', () => { - try { - parseSync('SELECT * FROM users WHERE id = $'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('syntax error at or near "$"')); - assert.equal(error.sqlDetails.cursorPosition, 31); - } - }); - - it('should handle reserved keywords', () => { - try { - parseSync('SELECT * FROM table'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('syntax error at or near "table"')); - assert.equal(error.sqlDetails.cursorPosition, 14); - } - }); - - it('should handle syntax error in WHERE clause', () => { - try { - parseSync('SELECT * FROM users WHERE'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error.message.includes('syntax error at end of input')); - assert.equal(error.sqlDetails.cursorPosition, 25); - } - }); - }); - - describe('formatSqlError Helper', () => { - it('should format error with position indicator', () => { - try { - parseSync("SELECT * FROM users WHERE id = 'unclosed"); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, "SELECT * FROM users WHERE id = 'unclosed"); - assert.ok(formatted.includes('Error: unterminated quoted string')); - assert.ok(formatted.includes('Position: 31')); - assert.ok(formatted.includes("SELECT * FROM users WHERE id = 'unclosed")); - assert.ok(formatted.includes(' ^')); - } - }); - - it('should respect showPosition option', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { - showPosition: false - }); - assert.ok(!formatted.includes('^')); - assert.ok(formatted.includes('Position: 32')); - } - }); - - it('should respect showQuery option', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { - showQuery: false - }); - assert.ok(!formatted.includes('SELECT * FROM users')); - assert.ok(formatted.includes('Error:')); - assert.ok(formatted.includes('Position:')); - } - }); - - it('should truncate long queries', () => { - const longQuery = 'SELECT ' + 'a, '.repeat(50) + 'z FROM users WHERE id = @'; - try { - parseSync(longQuery); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, longQuery, { maxQueryLength: 50 }); - assert.ok(formatted.includes('...')); - const lines = formatted.split('\n'); - const queryLine = lines.find(line => line.includes('...')); - assert.ok(queryLine.length <= 56); // 50 + 2*3 for ellipsis - } - }); - - it('should handle color option without breaking output', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - const formatted = formatSqlError(error, 'SELECT * FROM users WHERE id = @', { - color: true - }); - assert.ok(formatted.includes('Error:')); - assert.ok(formatted.includes('Position:')); - // Should contain ANSI codes but still be readable - const cleanFormatted = formatted.replace(/\x1b\[[0-9;]*m/g, ''); - assert.ok(cleanFormatted.includes('syntax error')); - } - }); - }); - - describe('hasSqlDetails Type Guard', () => { - it('should return true for SQL parse errors', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(hasSqlDetails(error), true); - } - }); - - it('should return false for regular errors', () => { - const regularError = new Error('Regular error'); - assert.equal(hasSqlDetails(regularError), false); - }); - - it('should return false for non-Error objects', () => { - assert.equal(hasSqlDetails('string'), false); - assert.equal(hasSqlDetails(123), false); - assert.equal(hasSqlDetails(null), false); - assert.equal(hasSqlDetails(undefined), false); - assert.equal(hasSqlDetails({}), false); - }); - - it('should return false for Error with incomplete sqlDetails', () => { - const error = new Error('Test'); - error.sqlDetails = { message: 'test' }; // Missing cursorPosition - assert.equal(hasSqlDetails(error), false); - }); - }); - - describe('Edge Cases', () => { - it('should handle empty query', () => { - assert.throws(() => parseSync(''), { - message: 'Query cannot be empty' - }); - }); - - it('should handle null query', () => { - assert.throws(() => parseSync(null), { - message: 'Query cannot be null or undefined' - }); - }); - - it('should handle undefined query', () => { - assert.throws(() => parseSync(undefined), { - message: 'Query cannot be null or undefined' - }); - }); - - it('should handle @ in comments', () => { - const query = 'SELECT * FROM users /* @ in comment */ WHERE id = 1'; - assert.doesNotThrow(() => parseSync(query)); - }); - - it('should handle @ in strings', () => { - const query = 'SELECT * FROM users WHERE email = \'user@example.com\''; - assert.doesNotThrow(() => parseSync(query)); - }); - }); - - describe('Complex Error Scenarios', () => { - it('should handle errors in CASE statements', () => { - try { - parseSync('SELECT CASE WHEN id = 1 THEN "one" WHEN id = 2 THEN @ ELSE "other" END FROM users'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 54); - } - }); - - it('should handle errors in subqueries', () => { - try { - parseSync('SELECT * FROM users WHERE id IN (SELECT @ FROM orders)'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 42); - } - }); - - it('should handle errors in function calls', () => { - try { - parseSync('SELECT COUNT(@) FROM users'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 14); - } - }); - - it('should handle errors in second statement', () => { - try { - parseSync('SELECT * FROM users; SELECT * FROM orders WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 54); - } - }); - - it('should handle errors in CTE', () => { - try { - parseSync('WITH cte AS (SELECT * FROM users WHERE id = @) SELECT * FROM cte'); - assert.fail('Expected error'); - } catch (error) { - assert.equal(error.sqlDetails.cursorPosition, 45); - } - }); - }); - - describe('Backward Compatibility', () => { - it('should maintain Error instance', () => { - try { - parseSync('SELECT * FROM users WHERE id = @'); - assert.fail('Expected error'); - } catch (error) { - assert.ok(error instanceof Error); - assert.ok(error.message); - assert.ok(error.stack); - } - }); - - it('should work with standard error handling', () => { - let caught = false; - try { - parseSync('SELECT * FROM users WHERE id = @'); - } catch (e) { - caught = true; - assert.ok(e.message.includes('syntax error')); - } - assert.equal(caught, true); - }); - }); -}); \ No newline at end of file diff --git a/full/18/test/fingerprint.test.js b/full/18/test/fingerprint.test.js deleted file mode 100644 index 07171c82..00000000 --- a/full/18/test/fingerprint.test.js +++ /dev/null @@ -1,67 +0,0 @@ -const query = require("../"); -const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); -const assert = require('node:assert/strict'); - -describe("Query Fingerprinting", () => { - before(async () => { - await query.parse("SELECT 1"); - }); - - describe("Sync Fingerprinting", () => { - it("should return a fingerprint for a simple query", () => { - const fingerprint = query.fingerprintSync("select 1"); - assert.equal(typeof fingerprint, "string"); - assert.equal(fingerprint.length, 16); - }); - - it("should return same fingerprint for equivalent queries", () => { - const fp1 = query.fingerprintSync("select 1"); - const fp2 = query.fingerprintSync("SELECT 1"); - const fp3 = query.fingerprintSync("select 1"); - - assert.equal(fp1, fp2); - assert.equal(fp1, fp3); - }); - - it("should return different fingerprints for different queries", () => { - const fp1 = query.fingerprintSync("select name from users"); - const fp2 = query.fingerprintSync("select id from customers"); - - assert.notEqual(fp1, fp2); - }); - - it("should normalize parameter values", () => { - const fp1 = query.fingerprintSync("select * from users where id = 123"); - const fp2 = query.fingerprintSync("select * from users where id = 456"); - - assert.equal(fp1, fp2); - }); - - it("should fail on invalid queries", () => { - assert.throws(() => query.fingerprintSync("NOT A QUERY"), Error); - }); - }); - - describe("Async Fingerprinting", () => { - it("should return a promise resolving to same result", async () => { - const testQuery = "select * from users"; - const fpPromise = query.fingerprint(testQuery); - const fp = await fpPromise; - - assert.ok(fpPromise instanceof Promise); - assert.equal(fp, query.fingerprintSync(testQuery)); - }); - - it("should reject on bogus queries", async () => { - return query.fingerprint("NOT A QUERY").then( - () => { - throw new Error("should have rejected"); - }, - (e) => { - assert.ok(e instanceof Error); - assert.match(e.message, /NOT/); - } - ); - }); - }); -}); diff --git a/full/18/test/normalize.test.js b/full/18/test/normalize.test.js deleted file mode 100644 index 14f0a431..00000000 --- a/full/18/test/normalize.test.js +++ /dev/null @@ -1,69 +0,0 @@ -const query = require("../"); -const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); -const assert = require('node:assert/strict'); - -describe("Query Normalization", () => { - before(async () => { - await query.parse("SELECT 1"); - }); - - describe("Sync Normalization", () => { - it("should normalize a simple query", () => { - const normalized = query.normalizeSync("select 1"); - assert.equal(typeof normalized, "string"); - assert.ok(normalized.includes("$1")); - }); - - it("should normalize parameter values", () => { - const normalized1 = query.normalizeSync("select * from users where id = 123"); - const normalized2 = query.normalizeSync("select * from users where id = 456"); - - assert.equal(normalized1, normalized2); - assert.ok(normalized1.includes("$1")); - }); - - it("should normalize string literals", () => { - const normalized1 = query.normalizeSync("select * from users where name = 'john'"); - const normalized2 = query.normalizeSync("select * from users where name = 'jane'"); - - assert.equal(normalized1, normalized2); - assert.ok(normalized1.includes("$1")); - }); - - it("should preserve query structure", () => { - const normalized = query.normalizeSync("SELECT id, name FROM users WHERE active = true ORDER BY name"); - - assert.ok(normalized.includes("SELECT")); - assert.ok(normalized.includes("FROM")); - assert.ok(normalized.includes("WHERE")); - assert.ok(normalized.includes("ORDER BY")); - }); - - it("should fail on invalid queries", () => { - assert.throws(() => query.normalizeSync("NOT A QUERY"), Error); - }); - }); - - describe("Async Normalization", () => { - it("should return a promise resolving to same result", async () => { - const testQuery = "select * from users where id = 123"; - const normalizedPromise = query.normalize(testQuery); - const normalized = await normalizedPromise; - - assert.ok(normalizedPromise instanceof Promise); - assert.equal(normalized, query.normalizeSync(testQuery)); - }); - - it("should reject on bogus queries", async () => { - return query.normalize("NOT A QUERY").then( - () => { - throw new Error("should have rejected"); - }, - (e) => { - assert.ok(e instanceof Error); - assert.match(e.message, /NOT/); - } - ); - }); - }); -}); diff --git a/full/18/test/parsing.test.js b/full/18/test/parsing.test.js deleted file mode 100644 index 5477c6a0..00000000 --- a/full/18/test/parsing.test.js +++ /dev/null @@ -1,84 +0,0 @@ -const query = require("../"); -const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); -const assert = require('node:assert/strict'); - -function removeLocationProperties(obj) { - if (typeof obj !== 'object' || obj === null) { - return obj; - } - - if (Array.isArray(obj)) { - return obj.map(item => removeLocationProperties(item)); - } - - const result = {}; - for (const key in obj) { - if (obj.hasOwnProperty(key)) { - if (key === 'location' || key === 'stmt_len' || key === 'stmt_location') { - continue; // Skip location-related properties - } - result[key] = removeLocationProperties(obj[key]); - } - } - return result; -} - -describe("Query Parsing", () => { - before(async () => { - await query.parse("SELECT 1"); - }); - - describe("Sync Parsing", () => { - it("should return a single-item parse result for common queries", () => { - const queries = ["select 1", "select null", "select ''", "select a, b"]; - const results = queries.map(query.parseSync); - results.forEach((res) => { - assert.equal(res.stmts.length, 1); - }); - - const selectedDatas = results.map( - (it) => it.stmts[0].stmt.SelectStmt.targetList - ); - - assert.equal(selectedDatas[0][0].ResTarget.val.A_Const.ival.ival, 1); - assert.equal(selectedDatas[1][0].ResTarget.val.A_Const.isnull, true); - assert.equal(selectedDatas[2][0].ResTarget.val.A_Const.sval.sval, ""); - assert.equal(selectedDatas[3].length, 2); - }); - - it("should support parsing multiple queries", () => { - const res = query.parseSync("select 1; select null;"); - assert.deepEqual(res.stmts.map(removeLocationProperties), [ - ...query.parseSync("select 1;").stmts.map(removeLocationProperties), - ...query.parseSync("select null;").stmts.map(removeLocationProperties), - ]); - }); - - it("should not parse a bogus query", () => { - assert.throws(() => query.parseSync("NOT A QUERY"), Error); - }); - }); - - describe("Async parsing", () => { - it("should return a promise resolving to same result", async () => { - const testQuery = "select * from john;"; - const resPromise = query.parse(testQuery); - const res = await resPromise; - - assert.ok(resPromise instanceof Promise); - assert.deepEqual(res, query.parseSync(testQuery)); - }); - - it("should reject on bogus queries", async () => { - return query.parse("NOT A QUERY").then( - () => { - throw new Error("should have rejected"); - }, - (e) => { - assert.ok(e instanceof Error); - assert.match(e.message, /NOT/); - } - ); - }); - }); -}); diff --git a/full/18/test/plpgsql.test.js b/full/18/test/plpgsql.test.js deleted file mode 100644 index e3235c83..00000000 --- a/full/18/test/plpgsql.test.js +++ /dev/null @@ -1,75 +0,0 @@ -const query = require("../"); -const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); -const assert = require('node:assert/strict'); - -describe("PL/pgSQL Parsing", () => { - before(async () => { - await query.parse("SELECT 1"); - }); - - describe("Sync PL/pgSQL Parsing", () => { - it("should parse a simple PL/pgSQL function", () => { - const funcSql = ` - CREATE OR REPLACE FUNCTION test_func() - RETURNS INTEGER AS $$ - BEGIN - RETURN 42; - END; - $$ LANGUAGE plpgsql; - `; - - const result = query.parsePlPgSQLSync(funcSql); - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.plpgsql_funcs)); - }); - - it("should parse function with parameters", () => { - const funcSql = ` - CREATE OR REPLACE FUNCTION add_numbers(a INTEGER, b INTEGER) - RETURNS INTEGER AS $$ - BEGIN - RETURN a + b; - END; - $$ LANGUAGE plpgsql; - `; - - const result = query.parsePlPgSQLSync(funcSql); - assert.ok(result.plpgsql_funcs.length > 0); - }); - - it("should fail on invalid PL/pgSQL", () => { - assert.throws(() => query.parsePlPgSQLSync("NOT A FUNCTION"), Error); - }); - }); - - describe("Async PL/pgSQL Parsing", () => { - it("should return a promise resolving to same result", async () => { - const funcSql = ` - CREATE OR REPLACE FUNCTION test_func() - RETURNS INTEGER AS $$ - BEGIN - RETURN 42; - END; - $$ LANGUAGE plpgsql; - `; - - const resultPromise = query.parsePlPgSQL(funcSql); - const result = await resultPromise; - - assert.ok(resultPromise instanceof Promise); - assert.deepEqual(result, query.parsePlPgSQLSync(funcSql)); - }); - - it("should reject on invalid PL/pgSQL", async () => { - return query.parsePlPgSQL("NOT A FUNCTION").then( - () => { - throw new Error("should have rejected"); - }, - (e) => { - assert.ok(e instanceof Error); - assert.match(e.message, /NOT/); - } - ); - }); - }); -}); diff --git a/full/18/test/scan.test.js b/full/18/test/scan.test.js deleted file mode 100644 index 24d7b4cf..00000000 --- a/full/18/test/scan.test.js +++ /dev/null @@ -1,279 +0,0 @@ -const query = require("../"); -const { describe, it, before, after, beforeEach, afterEach } = require('node:test'); -const assert = require('node:assert/strict'); - -describe("Query Scanning", () => { - before(async () => { - await query.parse("SELECT 1"); - }); - - describe("Sync Scanning", () => { - it("should return a scan result with version and tokens", () => { - const result = query.scanSync("SELECT 1"); - - assert.equal(typeof result, "object"); - assert.ok("version" in result); - assert.ok("tokens" in result); - assert.equal(typeof result.version, "number"); - assert.ok(Array.isArray(result.tokens)); - }); - - it("should scan a simple SELECT query correctly", () => { - const result = query.scanSync("SELECT 1"); - - assert.equal(result.tokens.length, 2); - - // First token should be SELECT - const selectToken = result.tokens[0]; - assert.equal(selectToken.text, "SELECT"); - assert.equal(selectToken.start, 0); - assert.equal(selectToken.end, 6); - assert.equal(selectToken.tokenName, "UNKNOWN"); // SELECT is mapped as UNKNOWN in our simplified mapping - assert.equal(selectToken.keywordName, "RESERVED_KEYWORD"); - - // Second token should be 1 - const numberToken = result.tokens[1]; - assert.equal(numberToken.text, "1"); - assert.equal(numberToken.start, 7); - assert.equal(numberToken.end, 8); - assert.equal(numberToken.tokenName, "ICONST"); - assert.equal(numberToken.keywordName, "NO_KEYWORD"); - }); - - it("should scan tokens with correct positions", () => { - const sql = "SELECT * FROM users"; - const result = query.scanSync(sql); - - assert.equal(result.tokens.length, 4); - - // Verify each token position matches the original SQL - result.tokens.forEach(token => { - const actualText = sql.substring(token.start, token.end); - assert.equal(token.text, actualText); - }); - }); - - it("should identify different token types", () => { - const result = query.scanSync("SELECT 'string', 123, 3.14, $1 FROM users"); - - const tokenTypes = result.tokens.map(t => t.tokenName); - assert.ok(tokenTypes.includes("SCONST")); // String constant - assert.ok(tokenTypes.includes("ICONST")); // Integer constant - assert.ok(tokenTypes.includes("FCONST")); // Float constant - assert.ok(tokenTypes.includes("PARAM")); // Parameter marker - // Note: keywords like FROM may be tokenized as UNKNOWN in our simplified mapping - assert.ok(tokenTypes.includes("UNKNOWN")); // Keywords and identifiers - }); - - it("should identify operators and punctuation", () => { - const result = query.scanSync("SELECT * FROM users WHERE id = 1"); - - const operators = result.tokens.filter(t => - t.tokenName.startsWith("ASCII_") || t.text === "=" - ); - - assert.ok(operators.length > 0); - assert.equal(operators.some(t => t.text === "*"), true); - assert.equal(operators.some(t => t.text === "="), true); - }); - - it("should classify keyword types correctly", () => { - const result = query.scanSync("SELECT COUNT(*) FROM users WHERE active = true"); - - const reservedKeywords = result.tokens.filter(t => - t.keywordName === "RESERVED_KEYWORD" - ); - const unreservedKeywords = result.tokens.filter(t => - t.keywordName === "UNRESERVED_KEYWORD" - ); - - assert.ok(reservedKeywords.length > 0); - // SELECT, FROM, WHERE should be reserved keywords - assert.equal(reservedKeywords.some(t => t.text === "SELECT"), true); - assert.equal(reservedKeywords.some(t => t.text === "FROM"), true); - assert.equal(reservedKeywords.some(t => t.text === "WHERE"), true); - }); - - it("should handle complex queries with parameters", () => { - const result = query.scanSync("SELECT * FROM users WHERE id = $1 AND name = $2"); - - const params = result.tokens.filter(t => t.tokenName === "PARAM"); - assert.equal(params.length, 2); - assert.equal(params[0].text, "$1"); - assert.equal(params[1].text, "$2"); - }); - - it("should handle string escaping in JSON output", () => { - const result = query.scanSync("SELECT 'text with \"quotes\" and \\backslash'"); - - const stringToken = result.tokens.find(t => t.tokenName === "SCONST"); - assert.ok(stringToken); - assert.ok(stringToken.text.includes('"')); - assert.ok(stringToken.text.includes('\\')); - }); - - it("should scan INSERT statements", () => { - const result = query.scanSync("INSERT INTO table VALUES (1, 'text', 3.14)"); - - assert.equal(result.tokens.some(t => t.text === "INSERT"), true); - assert.equal(result.tokens.some(t => t.text === "INTO"), true); - assert.equal(result.tokens.some(t => t.text === "VALUES"), true); - assert.equal(result.tokens.some(t => t.tokenName === "ICONST"), true); - assert.equal(result.tokens.some(t => t.tokenName === "SCONST"), true); - assert.equal(result.tokens.some(t => t.tokenName === "FCONST"), true); - }); - - it("should scan UPDATE statements", () => { - const result = query.scanSync("UPDATE users SET name = 'John' WHERE id = 1"); - - assert.equal(result.tokens.some(t => t.text === "UPDATE"), true); - assert.equal(result.tokens.some(t => t.text === "SET"), true); - assert.equal(result.tokens.some(t => t.text === "="), true); - }); - - it("should scan DELETE statements", () => { - const result = query.scanSync("DELETE FROM users WHERE active = false"); - - assert.equal(result.tokens.some(t => t.text === "DELETE"), true); - assert.equal(result.tokens.some(t => t.text === "FROM"), true); - assert.equal(result.tokens.some(t => t.text === "WHERE"), true); - }); - - it("should handle empty or whitespace-only input", () => { - const result = query.scanSync(" "); - assert.equal(result.tokens.length, 0); - }); - - it("should handle unusual input gracefully", () => { - // The scanner is more permissive than the parser and may tokenize unusual input - const result = query.scanSync("$$$INVALID$$$"); - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.tokens)); - // Scanner may still produce tokens even for unusual input - }); - - it("should preserve original token order", () => { - const sql = "SELECT id, name FROM users ORDER BY name"; - const result = query.scanSync(sql); - - // Tokens should be in order of appearance - for (let i = 1; i < result.tokens.length; i++) { - assert.ok(result.tokens[i].start >= result.tokens[i-1].end); - } - }); - }); - - describe("Async Scanning", () => { - it("should return a promise resolving to same result as sync", async () => { - const testQuery = "SELECT * FROM users WHERE id = $1"; - const resultPromise = query.scan(testQuery); - const result = await resultPromise; - - assert.ok(resultPromise instanceof Promise); - assert.deepEqual(result, query.scanSync(testQuery)); - }); - - it("should handle complex queries asynchronously", async () => { - const testQuery = "SELECT COUNT(*) as total FROM orders WHERE status = 'completed' AND created_at > '2023-01-01'"; - const result = await query.scan(testQuery); - - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.tokens)); - assert.ok(result.tokens.length > 10); - }); - - it("should handle unusual input asynchronously", async () => { - // Scanner is more permissive than parser - const result = await query.scan("$$$INVALID$$$"); - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.tokens)); - }); - }); - - describe("Edge Cases", () => { - it("should handle queries with comments", () => { - const result = query.scanSync("SELECT 1 -- this is a comment"); - - // Should have at least SELECT and 1 tokens - assert.ok(result.tokens.length >= 2); - assert.equal(result.tokens.some(t => t.text === "SELECT"), true); - assert.equal(result.tokens.some(t => t.text === "1"), true); - }); - - it("should handle very long identifiers", () => { - const longIdentifier = "a".repeat(100); - const result = query.scanSync(`SELECT ${longIdentifier} FROM table`); - - const identToken = result.tokens.find(t => t.text === longIdentifier); - assert.ok(identToken); - assert.equal(identToken.tokenName, "IDENT"); - }); - - it("should handle special PostgreSQL operators", () => { - const result = query.scanSync("SELECT id::text FROM users"); - - assert.equal(result.tokens.some(t => t.text === "::"), true); - const typecastToken = result.tokens.find(t => t.text === "::"); - assert.equal(typecastToken?.tokenName, "TYPECAST"); - }); - - it("should provide consistent version information", () => { - const result1 = query.scanSync("SELECT 1"); - const result2 = query.scanSync("INSERT INTO table VALUES (1)"); - - assert.equal(result1.version, result2.version); - assert.equal(typeof result1.version, "number"); - assert.ok(result1.version > 0); - }); - - it("should handle multi-line dollar-quoted strings without JSON errors", () => { - // Without the fix, scanSync throws: - // "Bad control character in string literal" - // because build_scan_json() doesn't escape \n in the token text. - const sql = `CREATE FUNCTION test() RETURNS void AS $$ -BEGIN - RAISE NOTICE 'hello'; -END; -$$ LANGUAGE plpgsql`; - - const result = query.scanSync(sql); - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.tokens)); - assert.ok(result.tokens.length > 0); - - // The dollar-quoted body spans multiple lines - const dollarToken = result.tokens.find(t => t.text.includes('BEGIN')); - assert.ok(dollarToken, "should have a token containing the function body"); - assert.ok(dollarToken.text.includes('\n'), "token text should preserve newlines"); - }); - - it("should handle dollar-quoted tokens with tabs", () => { - // Tab characters also break JSON.parse when unescaped. - const sql = `SELECT $$line1 - indented -line3$$`; - - const result = query.scanSync(sql); - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.tokens)); - - const dollarToken = result.tokens.find(t => t.text.includes('indented')); - assert.ok(dollarToken, "should have a token containing the tabbed content"); - }); - - it("should handle multi-line block comments", () => { - // C-style block comments spanning multiple lines hit the same bug. - const sql = `SELECT 1; /* multi -line -comment */ SELECT 2`; - - const result = query.scanSync(sql); - assert.equal(typeof result, "object"); - assert.ok(Array.isArray(result.tokens)); - - const commentToken = result.tokens.find(t => t.tokenName === "C_COMMENT"); - assert.ok(commentToken, "should have a C_COMMENT token"); - assert.ok(commentToken.text.includes('\n'), "comment text should preserve newlines"); - }); - }); -}); diff --git a/full/18/tsconfig.esm.json b/full/18/tsconfig.esm.json deleted file mode 100644 index 63012c17..00000000 --- a/full/18/tsconfig.esm.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "module": "es2022", - "rootDir": "src/", - "declaration": false, - "outDir": "esm/" - } - } \ No newline at end of file diff --git a/full/18/tsconfig.json b/full/18/tsconfig.json deleted file mode 100644 index d7da207b..00000000 --- a/full/18/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "compilerOptions": { - "target": "es2022", - "module": "commonjs", - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "strictNullChecks": false, - "skipLibCheck": true, - "sourceMap": false, - "declaration": true, - "resolveJsonModule": true, - "moduleResolution": "node", - "outDir": "cjs/", - "rootDir": "src" - }, - "exclude": ["cjs", "esm", "wasm", "node_modules", "**/*.test.ts"] -} \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f2564356..eef64ba6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,26 +72,6 @@ importers: version: 1.28.2 publishDirectory: dist - full/17: - dependencies: - '@pgsql/types': - specifier: ^17.6.2 - version: 17.6.2 - devDependencies: - '@yamlize/cli': - specifier: ^0.8.0 - version: 0.8.0 - - full/18: - dependencies: - '@pgsql/types': - specifier: ^18.0.0 - version: 18.0.0 - devDependencies: - '@yamlize/cli': - specifier: ^0.8.0 - version: 0.8.0 - parser: devDependencies: cross-env: @@ -475,17 +455,6 @@ packages: undici-types: 6.21.0 dev: true - /@yamlize/cli@0.8.0: - resolution: {integrity: sha512-OuhQ/gYLCuMjENdLMF8UXgM32p7blBB0FxwS4R2Mw4jk/9uvv87uCz2ptq9VB7GjNTNbnRTQKw+bAbwCXyngCA==} - hasBin: true - dependencies: - chalk: 4.1.0 - inquirerer: 1.9.1 - js-yaml: 4.1.0 - minimist: 1.2.8 - yamlize: 0.8.0 - dev: true - /acorn-walk@8.3.4: resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==} engines: {node: '>=0.4.0'} @@ -525,10 +494,6 @@ packages: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} dev: true - /argparse@2.0.1: - resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} - dev: true - /balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} dev: true @@ -551,22 +516,6 @@ packages: engines: {node: '>= 0.8.0'} dev: true - /chalk@4.1.0: - resolution: {integrity: sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==} - engines: {node: '>=10'} - dependencies: - ansi-styles: 4.3.0 - supports-color: 7.2.0 - dev: true - - /chalk@4.1.2: - resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} - engines: {node: '>=10'} - dependencies: - ansi-styles: 4.3.0 - supports-color: 7.2.0 - dev: true - /cliui@7.0.4: resolution: {integrity: sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==} dependencies: @@ -726,11 +675,6 @@ packages: engines: {node: '>=4'} dev: true - /has-flag@4.0.0: - resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} - engines: {node: '>=8'} - dev: true - /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -743,15 +687,6 @@ packages: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} dev: true - /inquirerer@1.9.1: - resolution: {integrity: sha512-c7N3Yd9warVEpWdyX04dJUtYSad1qZFnNQYsKdqk0Av4qRg83lmxSnhWLn8Ok+UNzj87xXxo/ww0ReIL3ZO92g==} - dependencies: - chalk: 4.1.2 - deepmerge: 4.3.1 - js-yaml: 4.1.0 - minimist: 1.2.8 - dev: true - /is-fullwidth-code-point@3.0.0: resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} engines: {node: '>=8'} @@ -788,13 +723,6 @@ packages: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} dev: true - /js-yaml@4.1.0: - resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} - hasBin: true - dependencies: - argparse: 2.0.1 - dev: true - /jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} @@ -838,10 +766,6 @@ packages: brace-expansion: 2.0.2 dev: true - /minimist@1.2.8: - resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - dev: true - /minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} @@ -853,12 +777,6 @@ packages: hasBin: true dev: true - /mkdirp@3.0.1: - resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==} - engines: {node: '>=10'} - hasBin: true - dev: true - /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} dev: true @@ -1036,13 +954,6 @@ packages: ansi-regex: 6.1.0 dev: true - /supports-color@7.2.0: - resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} - engines: {node: '>=8'} - dependencies: - has-flag: 4.0.0 - dev: true - /through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} dependencies: @@ -1149,14 +1060,6 @@ packages: engines: {node: '>=10'} dev: true - /yamlize@0.8.0: - resolution: {integrity: sha512-2sXxXTr4gZuIP1TmVmm9yJc/WirEKsqctk/gk4MzPGuochfSAY4+OxKXXqFj02HejQmEgAFRun7b0Ec6YjlE7A==} - dependencies: - js-yaml: 4.1.0 - mkdirp: 3.0.1 - nested-obj: 0.0.1 - dev: true - /yargs-parser@20.2.9: resolution: {integrity: sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==} engines: {node: '>=10'} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0a36f3f1..953012e7 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,5 @@ packages: - 'parser' - - 'full/18' - - 'full/17' - 'versions/18' - 'versions/17' - 'versions/16' diff --git a/scripts/analyze-sizes.js b/scripts/analyze-sizes.js index 959cb7b5..c93e45df 100644 --- a/scripts/analyze-sizes.js +++ b/scripts/analyze-sizes.js @@ -74,10 +74,7 @@ function analyzePackage(packagePath, packageName) { // Get all version packages dynamically function getVersionPackages() { const versionsDir = './versions'; - const packages = [ - { path: './full/17', name: 'full/17 (Full)', version: 'original' }, - { path: './full/18', name: 'full/18 (Full)', version: 'original' } - ]; + const packages = []; if (fs.existsSync(versionsDir)) { const versions = fs.readdirSync(versionsDir) diff --git a/scripts/copy-templates.js b/scripts/copy-templates.js index a2a6fb79..6d8d312d 100755 --- a/scripts/copy-templates.js +++ b/scripts/copy-templates.js @@ -32,13 +32,15 @@ function loadVersionConfigs() { const packageData = JSON.parse(fs.readFileSync(packageFile, 'utf8')); const version = packageData['x-publish']?.pgVersion; const libpgQueryTag = packageData['x-publish']?.libpgQueryTag; + const fullApi = packageData['x-publish']?.fullApi === true; if (version && libpgQueryTag) { configs[version] = { tag: libpgQueryTag, - hasEmscriptenPatch: version === '13' // Only version 13 needs the patch + hasEmscriptenPatch: version === '13', // Only version 13 needs the patch + fullApi }; - console.log(` Version ${version}: tag ${libpgQueryTag}`); + console.log(` Version ${version}: tag ${libpgQueryTag}${fullApi ? ' (full API)' : ''}`); } else { console.warn(` Warning: Missing x-publish data in ${packageFile}`); } @@ -54,14 +56,18 @@ function loadVersionConfigs() { // Load configurations const VERSION_CONFIGS = loadVersionConfigs(); -// Files to copy from templates +// Files to copy from templates. Full-API versions use the variants in +// templates/full/ for the source files. const TEMPLATE_FILES = [ - { src: 'LICENSE', dest: 'LICENSE', header: false }, - { src: 'wasm_wrapper.c', dest: 'src/wasm_wrapper.c', header: HEADER }, - { src: 'libpg-query.d.ts', dest: 'src/libpg-query.d.ts', header: HEADER }, - { src: 'index.ts', dest: 'src/index.ts', header: HEADER } + { src: 'LICENSE', dest: 'LICENSE', header: false, hasFullVariant: false }, + { src: 'wasm_wrapper.c', dest: 'src/wasm_wrapper.c', header: HEADER, hasFullVariant: true }, + { src: 'libpg-query.d.ts', dest: 'src/libpg-query.d.ts', header: HEADER, hasFullVariant: true }, + { src: 'index.ts', dest: 'src/index.ts', header: HEADER, hasFullVariant: true } ]; +const SLIM_EXPORTED_FUNCTIONS = "['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']"; +const FULL_EXPORTED_FUNCTIONS = "['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string','_wasm_parse_query_raw','_wasm_free_parse_result']"; + function copyTemplates() { const templatesDir = path.join(__dirname, '..', 'templates'); const versionsDir = path.join(__dirname, '..', 'versions'); @@ -73,7 +79,9 @@ function copyTemplates() { // Copy template files for (const file of TEMPLATE_FILES) { - const srcPath = path.join(templatesDir, file.src); + const srcPath = (config.fullApi && file.hasFullVariant) + ? path.join(templatesDir, 'full', file.src) + : path.join(templatesDir, file.src); const destPath = path.join(versionDir, file.dest); // Ensure destination directory exists @@ -98,6 +106,10 @@ function copyTemplates() { // Process Makefile template const makefileTemplate = fs.readFileSync(path.join(templatesDir, 'Makefile.template'), 'utf8'); let makefileContent = makefileTemplate.replace(/{{VERSION_TAG}}/g, config.tag); + makefileContent = makefileContent.replace( + /{{EXPORTED_FUNCTIONS}}/g, + config.fullApi ? FULL_EXPORTED_FUNCTIONS : SLIM_EXPORTED_FUNCTIONS + ); // Handle the USE_EMSCRIPTEN_PATCH placeholder if (config.hasEmscriptenPatch) { diff --git a/scripts/publish-versions.js b/scripts/publish-versions.js index 1db54967..f9e08a86 100755 --- a/scripts/publish-versions.js +++ b/scripts/publish-versions.js @@ -29,27 +29,16 @@ async function main() { .filter(dir => /^\d+$/.test(dir)) .sort((a, b) => parseInt(b) - parseInt(a)); // Sort descending - // Also check for full package versions (full/17, full/18, ...) - const fullDir = path.join(__dirname, '..', 'full'); - const fullVersionDirs = fs.existsSync(fullDir) - ? fs.readdirSync(fullDir) - .filter(dir => /^\d+$/.test(dir) && fs.existsSync(path.join(fullDir, dir, 'package.json'))) - .sort((a, b) => parseInt(b) - parseInt(a)) // Sort descending - : []; - console.log('πŸ“¦ Available packages:'); versionDirs.forEach(v => console.log(` - PostgreSQL ${v} (versions/${v})`)); - fullVersionDirs.forEach(v => console.log(` - Full package PostgreSQL ${v} (full/${v})`)); console.log(); // Ask which versions to publish const publishAll = await question('Publish all packages? (y/N): '); let selectedVersions = []; - let selectedFullVersions = []; if (publishAll.toLowerCase() === 'y') { selectedVersions = versionDirs; - selectedFullVersions = fullVersionDirs; } else { // Let user select versions for (const version of versionDirs) { @@ -58,16 +47,9 @@ async function main() { selectedVersions.push(version); } } - - for (const version of fullVersionDirs) { - const publishFull = await question(`Publish full package PostgreSQL ${version} (full/${version})? (y/N): `); - if (publishFull.toLowerCase() === 'y') { - selectedFullVersions.push(version); - } - } } - if (selectedVersions.length === 0 && selectedFullVersions.length === 0) { + if (selectedVersions.length === 0) { console.log('\n❌ No packages selected for publishing.'); rl.close(); return; @@ -82,7 +64,6 @@ async function main() { console.log(`\nπŸ“‹ Will publish:`); selectedVersions.forEach(v => console.log(` - PostgreSQL ${v} (${bump} bump)`)); - selectedFullVersions.forEach(v => console.log(` - Full package PostgreSQL ${v} (${bump} bump)`)); // Ask about building const skipBuild = await question('\nSkip build step? (y/N): '); @@ -143,75 +124,20 @@ async function main() { } } - // Process full package versions if selected - for (const version of selectedFullVersions) { - console.log(`\nπŸ“¦ Publishing full package PostgreSQL ${version}...`); - const fullPath = path.join(fullDir, version); - - try { - // Version bump - console.log(` πŸ“ Bumping version (${bump})...`); - execSync(`pnpm version ${bump}`, { cwd: fullPath, stdio: 'inherit' }); - - // Commit - console.log(` πŸ’Ύ Committing version bump...`); - execSync(`git add package.json`, { cwd: fullPath }); - execSync(`git commit -m "release: bump @libpg-query/parser v${version} version"`, { stdio: 'inherit' }); - - // Build (if not skipped) - if (shouldBuild) { - console.log(` πŸ”¨ Building...`); - execSync('pnpm build', { cwd: fullPath, stdio: 'inherit' }); - } else { - console.log(` ⏭️ Skipping build step`); - } - - // Test (always run) - console.log(` πŸ§ͺ Running tests...`); - execSync('pnpm test', { cwd: fullPath, stdio: 'inherit' }); - - // Publish with the pg tag (uses x-publish metadata) - console.log(` πŸ“€ Publishing to npm with pg${version} tag...`); - execSync('pnpm run publish:pkg', { cwd: fullPath, stdio: 'inherit' }); - - console.log(` βœ… Full package published successfully with pg${version} tag!`); - } catch (error) { - console.error(` ❌ Failed to publish full package PostgreSQL ${version}:`, error.message); - const continuePublish = await question('Continue with other versions? (y/N): '); - if (continuePublish.toLowerCase() !== 'y') { - rl.close(); - process.exit(1); - } - } - } - // Ask about promoting to latest - if (selectedVersions.includes('17') || selectedFullVersions.includes('17')) { + if (selectedVersions.includes('17')) { console.log('\n🏷️ Tag Management'); - if (selectedVersions.includes('17')) { - const promoteVersions = await question('Promote libpg-query@pg17 to latest? (y/N): '); - if (promoteVersions.toLowerCase() === 'y') { - try { - execSync('npm dist-tag add libpg-query@pg17 latest', { stdio: 'inherit' }); - console.log('βœ… libpg-query@pg17 promoted to latest'); - } catch (error) { - console.error('❌ Failed to promote tag:', error.message); - } - } - } - - if (selectedFullVersions.includes('17')) { - const promoteFullPackage = await question('Promote @libpg-query/parser@pg17 to latest? (y/N): '); - if (promoteFullPackage.toLowerCase() === 'y') { - try { - execSync('npm dist-tag add @libpg-query/parser@pg17 latest', { stdio: 'inherit' }); - console.log('βœ… @libpg-query/parser@pg17 promoted to latest'); - } catch (error) { - console.error('❌ Failed to promote tag:', error.message); - } + const promoteVersions = await question('Promote libpg-query@pg17 to latest? (y/N): '); + if (promoteVersions.toLowerCase() === 'y') { + try { + execSync('npm dist-tag add libpg-query@pg17 latest', { stdio: 'inherit' }); + console.log('βœ… libpg-query@pg17 promoted to latest'); + } catch (error) { + console.error('❌ Failed to promote tag:', error.message); } } + } console.log('\n✨ Publishing complete!'); diff --git a/templates/Makefile.template b/templates/Makefile.template index 422f286b..340d51ff 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -62,7 +62,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR) \ -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ - -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ + -sEXPORTED_FUNCTIONS="{{EXPORTED_FUNCTIONS}}" \ -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ diff --git a/full/17/src/index.ts b/templates/full/index.ts similarity index 100% rename from full/17/src/index.ts rename to templates/full/index.ts diff --git a/full/17/src/libpg-query.d.ts b/templates/full/libpg-query.d.ts similarity index 100% rename from full/17/src/libpg-query.d.ts rename to templates/full/libpg-query.d.ts diff --git a/full/17/src/wasm_wrapper.c b/templates/full/wasm_wrapper.c similarity index 100% rename from full/17/src/wasm_wrapper.c rename to templates/full/wasm_wrapper.c diff --git a/versions/18/Makefile b/versions/18/Makefile index 27bec91b..2074e8ec 100644 --- a/versions/18/Makefile +++ b/versions/18/Makefile @@ -64,7 +64,7 @@ ifdef EMSCRIPTEN -I$(LIBPG_QUERY_DIR) \ -I$(LIBPG_QUERY_DIR)/vendor \ -L$(LIBPG_QUERY_DIR) \ - -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query_raw','_wasm_free_parse_result']" \ + -sEXPORTED_FUNCTIONS="['_malloc','_free','_wasm_parse_query','_wasm_parse_plpgsql','_wasm_fingerprint','_wasm_normalize_query','_wasm_scan','_wasm_parse_query_detailed','_wasm_free_detailed_result','_wasm_free_string','_wasm_parse_query_raw','_wasm_free_parse_result']" \ -sEXPORTED_RUNTIME_METHODS="['lengthBytesUTF8','stringToUTF8','getValue','UTF8ToString','HEAPU8','HEAPU32']" \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ -sENVIRONMENT="web,node,worker" \ diff --git a/versions/18/README.md b/versions/18/README.md index 45a1156e..be93bf2f 100644 --- a/versions/18/README.md +++ b/versions/18/README.md @@ -54,7 +54,9 @@ const result = await parse('SELECT * FROM users WHERE active = true'); ## Versions -Our latest is built with the `18-latest` branch from libpg_query +Our latest is built with the `18-constructive` branch of [constructive-io/libpg_query](https://github.com/constructive-io/libpg_query) + +Starting with PostgreSQL 18, this package ships the **full API**: `parse`, `parsePlPgSQL`, `scan`, `fingerprint`, `normalize` and their sync variants. Versions 13–17 are parse-only. | PG Major Version | libpg_query | npm dist-tag |--------------------------|-------------|---------| @@ -89,7 +91,56 @@ const result = parseSync('SELECT * FROM users WHERE active = true'); // Returns: ParseResult - parsed query object ``` -⚠ **Note:** If you need additional functionality like `fingerprint`, `scan`, `deparse`, or `normalize`, check out the full package (`@libpg-query/parser`) in the [./full](https://github.com/constructive-io/libpg-query-node/tree/main/full) folder of the repo. +### `parsePlPgSQL(funcsSql: string): Promise` / `parsePlPgSQLSync` + +Parses the contents of a PL/pgSQL function from a `CREATE FUNCTION` declaration. + +```typescript +import { parsePlPgSQL } from 'libpg-query'; + +const result = await parsePlPgSQL(` +CREATE FUNCTION get_count() RETURNS integer AS $$ +BEGIN + RETURN (SELECT COUNT(*) FROM users); +END; +$$ LANGUAGE plpgsql; +`); +// { plpgsql_funcs: [...] } +``` + +### `scan(sql: string): Promise` / `scanSync` + +Tokenizes the SQL, returning token positions, text, types, and keyword kinds. See [SCAN.md](./SCAN.md). + +```typescript +import { scan } from 'libpg-query'; + +const result = await scan('SELECT id FROM users'); +// { version: 180004, tokens: [{ start, end, text, tokenType, tokenName, keywordKind, keywordName }, ...] } +``` + +### `fingerprint(sql: string): Promise` / `fingerprintSync` + +Generates a unique fingerprint for query identification/caching (equivalent queries share a fingerprint). + +```typescript +import { fingerprint } from 'libpg-query'; + +await fingerprint('SELECT * FROM users WHERE id = 1'); // e.g. 'a0ead580058af585' +``` + +### `normalize(sql: string): Promise` / `normalizeSync` + +Normalizes the SQL, replacing constants with placeholders. + +```typescript +import { normalize } from 'libpg-query'; + +await normalize("SELECT * FROM users WHERE name = 'alice'"); +// SELECT * FROM users WHERE name = $1 +``` + +⚠ **Note:** the full API (`parsePlPgSQL`, `scan`, `fingerprint`, `normalize`) is available on `pg18`+ only; `pg13`–`pg17` builds expose `parse`/`parseSync` only. ### Initialization diff --git a/full/17/SCAN.md b/versions/18/SCAN.md similarity index 100% rename from full/17/SCAN.md rename to versions/18/SCAN.md diff --git a/full/17/libpg_query.md b/versions/18/libpg_query.md similarity index 100% rename from full/17/libpg_query.md rename to versions/18/libpg_query.md diff --git a/versions/18/package.json b/versions/18/package.json index 132f0570..ea50f34b 100644 --- a/versions/18/package.json +++ b/versions/18/package.json @@ -13,7 +13,8 @@ "publishName": "libpg-query", "pgVersion": "18", "distTag": "pg18", - "libpgQueryTag": "18.0.0" + "libpgQueryTag": "18-constructive", + "fullApi": true }, "files": [ "wasm/*" @@ -28,7 +29,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js" + "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index 27eeb49f..0dd8e982 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -5,34 +5,36 @@ * npm run copy:templates */ +import { ParseResult } from "@pgsql/types"; export * from "@pgsql/types"; -// @ts-ignore -import PgQueryModule from './libpg-query.js'; +export interface ScanToken { + start: number; + end: number; + text: string; + tokenType: number; + tokenName: string; + keywordKind: number; + keywordName: string; +} -let wasmModule: any; +export interface ScanResult { + version: number; + tokens: ScanToken[]; +} -// SQL error details interface export interface SqlErrorDetails { message: string; - cursorPosition: number; // 0-based position in the query - fileName?: string; // Source file where error occurred (e.g., 'scan.l', 'gram.y') - functionName?: string; // Internal function name - lineNumber?: number; // Line number in source file - context?: string; // Additional context -} - -// Options for formatting SQL errors -export interface SqlErrorFormatOptions { - showPosition?: boolean; // Show the error position marker (default: true) - showQuery?: boolean; // Show the query text (default: true) - color?: boolean; // Use ANSI colors (default: false) - maxQueryLength?: number; // Max query length to display (default: no limit) + cursorPosition: number; + fileName?: string; + functionName?: string; + lineNumber?: number; + context?: string; } export class SqlError extends Error { sqlDetails?: SqlErrorDetails; - + constructor(message: string, details?: SqlErrorDetails) { super(message); this.name = 'SqlError'; @@ -40,21 +42,19 @@ export class SqlError extends Error { } } - - -// Helper function to classify error source -function getErrorSource(filename: string | null): string { - if (!filename) return 'unknown'; - if (filename === 'scan.l') return 'lexer'; // Lexical analysis errors - if (filename === 'gram.y') return 'parser'; // Grammar/parsing errors - return filename; +export function hasSqlDetails(error: unknown): error is SqlError { + return error instanceof SqlError && error.sqlDetails !== undefined; } -// Format SQL error with visual position indicator export function formatSqlError( - error: Error & { sqlDetails?: SqlErrorDetails }, + error: SqlError, query: string, - options: SqlErrorFormatOptions = {} + options: { + showPosition?: boolean; + showQuery?: boolean; + color?: boolean; + maxQueryLength?: number; + } = {} ): string { const { showPosition = true, @@ -64,23 +64,23 @@ export function formatSqlError( } = options; const lines: string[] = []; - + // ANSI color codes const red = color ? '\x1b[31m' : ''; const yellow = color ? '\x1b[33m' : ''; const reset = color ? '\x1b[0m' : ''; - + // Add error message lines.push(`${red}Error: ${error.message}${reset}`); - + // Add SQL details if available if (error.sqlDetails) { const { cursorPosition, fileName, functionName, lineNumber } = error.sqlDetails; - + if (cursorPosition !== undefined && cursorPosition >= 0) { lines.push(`Position: ${cursorPosition}`); } - + if (fileName || functionName || lineNumber) { const details = []; if (fileName) details.push(`file: ${fileName}`); @@ -88,26 +88,25 @@ export function formatSqlError( if (lineNumber) details.push(`line: ${lineNumber}`); lines.push(`Source: ${details.join(', ')}`); } - + // Show query with position marker if (showQuery && showPosition && cursorPosition !== undefined && cursorPosition >= 0) { let displayQuery = query; - + let adjustedPosition = cursorPosition; + // Truncate if needed if (maxQueryLength && query.length > maxQueryLength) { const start = Math.max(0, cursorPosition - Math.floor(maxQueryLength / 2)); const end = Math.min(query.length, start + maxQueryLength); - displayQuery = (start > 0 ? '...' : '') + - query.substring(start, end) + + displayQuery = (start > 0 ? '...' : '') + + query.substring(start, end) + (end < query.length ? '...' : ''); // Adjust cursor position for truncation - const adjustedPosition = cursorPosition - start + (start > 0 ? 3 : 0); - lines.push(displayQuery); - lines.push(' '.repeat(adjustedPosition) + `${yellow}^${reset}`); - } else { - lines.push(displayQuery); - lines.push(' '.repeat(cursorPosition) + `${yellow}^${reset}`); + adjustedPosition = cursorPosition - start + (start > 0 ? 3 : 0); } + + lines.push(displayQuery); + lines.push(' '.repeat(adjustedPosition) + `${yellow}^${reset}`); } } else if (showQuery) { // No SQL details, just show the query if requested @@ -117,21 +116,33 @@ export function formatSqlError( } lines.push(`Query: ${displayQuery}`); } - + return lines.join('\n'); } -// Check if an error has SQL details -export function hasSqlDetails(error: any): error is Error & { sqlDetails: SqlErrorDetails } { - return error instanceof Error && - 'sqlDetails' in error && - typeof (error as any).sqlDetails === 'object' && - (error as any).sqlDetails !== null && - 'message' in (error as any).sqlDetails && - 'cursorPosition' in (error as any).sqlDetails; +// @ts-ignore +import PgQueryModule from './libpg-query.js'; + +interface WasmModule { + _malloc: (size: number) => number; + _free: (ptr: number) => void; + _wasm_free_string: (ptr: number) => void; + _wasm_parse_query: (queryPtr: number) => number; + _wasm_parse_query_raw: (queryPtr: number) => number; + _wasm_free_parse_result: (ptr: number) => void; + _wasm_parse_plpgsql: (queryPtr: number) => number; + _wasm_fingerprint: (queryPtr: number) => number; + _wasm_normalize_query: (queryPtr: number) => number; + _wasm_scan: (queryPtr: number) => number; + lengthBytesUTF8: (str: string) => number; + stringToUTF8: (str: string, ptr: number, len: number) => void; + UTF8ToString: (ptr: number) => string; + getValue: (ptr: number, type: string) => number; } -const initPromise = PgQueryModule().then((module: any) => { +let wasmModule: WasmModule; + +const initPromise = PgQueryModule().then((module: WasmModule) => { wasmModule = module; }); @@ -139,13 +150,13 @@ function ensureLoaded() { if (!wasmModule) throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +export async function loadModule(): Promise { if (!wasmModule) { await initPromise; } } -function awaitInit any>(fn: T): T { +function awaitInit Promise>(fn: T): T { return (async (...args: Parameters) => { await initPromise; return fn(...args); @@ -176,68 +187,58 @@ function ptrToString(ptr: number): string { return wasmModule.UTF8ToString(ptr); } -export const parse = awaitInit(async (query: string) => { - // Pre-validation +export const parse = awaitInit(async (query: string): Promise => { + // Input validation if (query === null || query === undefined) { throw new Error('Query cannot be null or undefined'); } - if (typeof query !== 'string') { - throw new Error(`Query must be a string, got ${typeof query}`); - } - if (query.trim() === '') { + + if (query === '') { throw new Error('Query cannot be empty'); } - + const queryPtr = stringToPtr(query); let resultPtr = 0; try { - // Call the raw function that returns a struct pointer resultPtr = wasmModule._wasm_parse_query_raw(queryPtr); if (!resultPtr) { - throw new Error('Failed to allocate memory for parse result'); + throw new Error('Failed to parse query: memory allocation failed'); } - // Read the PgQueryParseResult struct fields - // struct { char* parse_tree; char* stderr_buffer; PgQueryError* error; } - const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); // offset 0 - const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); // offset 4 - const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); // offset 8 + // Read the PgQueryParseResult struct + const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); + const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); + const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); - // Check for error if (errorPtr) { - // Read PgQueryError struct fields - // struct { char* message; char* funcname; char* filename; int lineno; int cursorpos; char* context; } - const messagePtr = wasmModule.getValue(errorPtr, 'i32'); // offset 0 - const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); // offset 4 - const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); // offset 8 - const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); // offset 12 - const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); // offset 16 - const contextPtr = wasmModule.getValue(errorPtr + 20, 'i32'); // offset 20 + // Read PgQueryError struct + const messagePtr = wasmModule.getValue(errorPtr, 'i32'); + const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); + const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); + const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); + const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); const message = messagePtr ? wasmModule.UTF8ToString(messagePtr) : 'Unknown error'; - const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : null; + const funcname = funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined; + const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : undefined; - const errorDetails: SqlErrorDetails = { - message: message, + throw new SqlError(message, { + message, cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, // Convert to 0-based - fileName: filename || undefined, - functionName: funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined, - lineNumber: lineno > 0 ? lineno : undefined, - context: contextPtr ? wasmModule.UTF8ToString(contextPtr) : undefined - }; - - throw new SqlError(message, errorDetails); + fileName: filename, + functionName: funcname, + lineNumber: lineno > 0 ? lineno : undefined + }); } if (!parseTreePtr) { - throw new Error('Parse result is null'); + throw new Error('No parse tree generated'); } - const parseTree = wasmModule.UTF8ToString(parseTreePtr); - return JSON.parse(parseTree); - } - finally { + const parseTreeStr = wasmModule.UTF8ToString(parseTreePtr); + return JSON.parse(parseTreeStr); + } finally { wasmModule._free(queryPtr); if (resultPtr) { wasmModule._wasm_free_parse_result(resultPtr); @@ -245,71 +246,246 @@ export const parse = awaitInit(async (query: string) => { } }); -export function parseSync(query: string) { - // Pre-validation +export const parsePlPgSQL = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const fingerprint = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export const normalize = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +// Sync versions +export function parseSync(query: string): ParseResult { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + + // Input validation if (query === null || query === undefined) { throw new Error('Query cannot be null or undefined'); } - if (typeof query !== 'string') { - throw new Error(`Query must be a string, got ${typeof query}`); - } - if (query.trim() === '') { + + if (query === '') { throw new Error('Query cannot be empty'); } - + const queryPtr = stringToPtr(query); let resultPtr = 0; try { - // Call the raw function that returns a struct pointer resultPtr = wasmModule._wasm_parse_query_raw(queryPtr); if (!resultPtr) { - throw new Error('Failed to allocate memory for parse result'); + throw new Error('Failed to parse query: memory allocation failed'); } - // Read the PgQueryParseResult struct fields - // struct { char* parse_tree; char* stderr_buffer; PgQueryError* error; } - const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); // offset 0 - const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); // offset 4 - const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); // offset 8 + // Read the PgQueryParseResult struct + const parseTreePtr = wasmModule.getValue(resultPtr, 'i32'); + const stderrBufferPtr = wasmModule.getValue(resultPtr + 4, 'i32'); + const errorPtr = wasmModule.getValue(resultPtr + 8, 'i32'); - // Check for error if (errorPtr) { - // Read PgQueryError struct fields - // struct { char* message; char* funcname; char* filename; int lineno; int cursorpos; char* context; } - const messagePtr = wasmModule.getValue(errorPtr, 'i32'); // offset 0 - const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); // offset 4 - const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); // offset 8 - const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); // offset 12 - const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); // offset 16 - const contextPtr = wasmModule.getValue(errorPtr + 20, 'i32'); // offset 20 + // Read PgQueryError struct + const messagePtr = wasmModule.getValue(errorPtr, 'i32'); + const funcnamePtr = wasmModule.getValue(errorPtr + 4, 'i32'); + const filenamePtr = wasmModule.getValue(errorPtr + 8, 'i32'); + const lineno = wasmModule.getValue(errorPtr + 12, 'i32'); + const cursorpos = wasmModule.getValue(errorPtr + 16, 'i32'); const message = messagePtr ? wasmModule.UTF8ToString(messagePtr) : 'Unknown error'; - const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : null; + const funcname = funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined; + const filename = filenamePtr ? wasmModule.UTF8ToString(filenamePtr) : undefined; - const errorDetails: SqlErrorDetails = { - message: message, + throw new SqlError(message, { + message, cursorPosition: cursorpos > 0 ? cursorpos - 1 : 0, // Convert to 0-based - fileName: filename || undefined, - functionName: funcnamePtr ? wasmModule.UTF8ToString(funcnamePtr) : undefined, - lineNumber: lineno > 0 ? lineno : undefined, - context: contextPtr ? wasmModule.UTF8ToString(contextPtr) : undefined - }; - - throw new SqlError(message, errorDetails); + fileName: filename, + functionName: funcname, + lineNumber: lineno > 0 ? lineno : undefined + }); } if (!parseTreePtr) { - throw new Error('Parse result is null'); + throw new Error('No parse tree generated'); } - const parseTree = wasmModule.UTF8ToString(parseTreePtr); - return JSON.parse(parseTree); - } - finally { + const parseTreeStr = wasmModule.UTF8ToString(parseTreePtr); + return JSON.parse(parseTreeStr); + } finally { wasmModule._free(queryPtr); if (resultPtr) { wasmModule._wasm_free_parse_result(resultPtr); } } -} \ No newline at end of file +} + +export function parsePlPgSQLSync(query: string): ParseResult { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_parse_plpgsql(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function fingerprintSync(query: string): string { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_fingerprint(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export function normalizeSync(query: string): string { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_normalize_query(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return resultStr; + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} + +export const scan = awaitInit(async (query: string): Promise => { + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_scan(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +}); + +export function scanSync(query: string): ScanResult { + if (!wasmModule) { + throw new Error('WASM module not initialized. Call loadModule() first.'); + } + const queryPtr = stringToPtr(query); + let resultPtr = 0; + + try { + resultPtr = wasmModule._wasm_scan(queryPtr); + const resultStr = ptrToString(resultPtr); + + if (resultStr.startsWith('syntax error') || resultStr.startsWith('deparse error') || resultStr.startsWith('ERROR')) { + throw new Error(resultStr); + } + + return JSON.parse(resultStr); + } finally { + wasmModule._free(queryPtr); + if (resultPtr) { + wasmModule._wasm_free_string(resultPtr); + } + } +} \ No newline at end of file diff --git a/versions/18/src/libpg-query.d.ts b/versions/18/src/libpg-query.d.ts index 2098ee1c..b00f1e7f 100644 --- a/versions/18/src/libpg-query.d.ts +++ b/versions/18/src/libpg-query.d.ts @@ -11,10 +11,12 @@ declare module './libpg-query.js' { _free: (ptr: number) => void; _wasm_free_string: (ptr: number) => void; _wasm_parse_query: (queryPtr: number) => number; + _wasm_parse_plpgsql: (queryPtr: number) => number; + _wasm_fingerprint: (queryPtr: number) => number; + _wasm_normalize_query: (queryPtr: number) => number; lengthBytesUTF8: (str: string) => number; stringToUTF8: (str: string, ptr: number, len: number) => void; UTF8ToString: (ptr: number) => string; - HEAPU8: Uint8Array; } const PgQueryModule: () => Promise; diff --git a/versions/18/src/wasm_wrapper.c b/versions/18/src/wasm_wrapper.c index 38151683..ca6ea740 100644 --- a/versions/18/src/wasm_wrapper.c +++ b/versions/18/src/wasm_wrapper.c @@ -5,15 +5,26 @@ * npm run copy:templates */ +#include "pg_query.h" +#include "protobuf/pg_query.pb-c.h" #include -#include #include #include +#include static int validate_input(const char* input) { return input != NULL && strlen(input) > 0; } +static char* safe_strdup(const char* str) { + if (!str) return NULL; + char* result = strdup(str); + if (!result) { + return NULL; + } + return result; +} + static void* safe_malloc(size_t size) { void* ptr = malloc(size); if (!ptr && size > 0) { @@ -22,14 +33,28 @@ static void* safe_malloc(size_t size) { return ptr; } -// Raw struct access functions for parse EMSCRIPTEN_KEEPALIVE -PgQueryParseResult* wasm_parse_query_raw(const char* input) { +char* wasm_parse_query(const char* input) { if (!validate_input(input)) { - return NULL; + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryParseResult result = pg_query_parse(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); } - PgQueryParseResult* result = (PgQueryParseResult*)safe_malloc(sizeof(PgQueryParseResult)); + char* parse_tree = safe_strdup(result.parse_tree); + pg_query_free_parse_result(result); + return parse_tree; +} + +EMSCRIPTEN_KEEPALIVE +PgQueryParseResult* wasm_parse_query_raw(const char* input) { + PgQueryParseResult* result = (PgQueryParseResult*)malloc(sizeof(PgQueryParseResult)); if (!result) { return NULL; } @@ -44,4 +69,334 @@ void wasm_free_parse_result(PgQueryParseResult* result) { pg_query_free_parse_result(*result); free(result); } -} \ No newline at end of file +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_parse_plpgsql(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryPlpgsqlParseResult result = pg_query_parse_plpgsql(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_plpgsql_parse_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + if (!result.plpgsql_funcs) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("{\"plpgsql_funcs\":[]}"); + } + + size_t funcs_len = strlen(result.plpgsql_funcs); + size_t json_len = strlen("{\"plpgsql_funcs\":}") + funcs_len + 1; + char* wrapped_result = safe_malloc(json_len); + + if (!wrapped_result) { + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Memory allocation failed"); + } + + int written = snprintf(wrapped_result, json_len, "{\"plpgsql_funcs\":%s}", result.plpgsql_funcs); + + if (written >= json_len) { + free(wrapped_result); + pg_query_free_plpgsql_parse_result(result); + return safe_strdup("Buffer overflow prevented"); + } + + pg_query_free_plpgsql_parse_result(result); + return wrapped_result; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_fingerprint(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryFingerprintResult result = pg_query_fingerprint(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_fingerprint_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* fingerprint_str = safe_strdup(result.fingerprint_str); + pg_query_free_fingerprint_result(result); + return fingerprint_str; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_normalize_query(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryNormalizeResult result = pg_query_normalize(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_normalize_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + char* normalized = safe_strdup(result.normalized_query); + pg_query_free_normalize_result(result); + + if (!normalized) { + return safe_strdup("Memory allocation failed"); + } + + return normalized; +} + + + + + +typedef struct { + int has_error; + char* message; + char* funcname; + char* filename; + int lineno; + int cursorpos; + char* context; + char* data; + size_t data_len; +} WasmDetailedResult; + +EMSCRIPTEN_KEEPALIVE +WasmDetailedResult* wasm_parse_query_detailed(const char* input) { + WasmDetailedResult* result = safe_malloc(sizeof(WasmDetailedResult)); + if (!result) { + return NULL; + } + memset(result, 0, sizeof(WasmDetailedResult)); + + if (!validate_input(input)) { + result->has_error = 1; + result->message = safe_strdup("Invalid input: query cannot be null or empty"); + return result; + } + + PgQueryParseResult parse_result = pg_query_parse(input); + + if (parse_result.error) { + result->has_error = 1; + size_t message_len = strlen("Parse error: at line , position ") + strlen(parse_result.error->message) + 20; + char* prefixed_message = safe_malloc(message_len); + if (!prefixed_message) { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + pg_query_free_parse_result(parse_result); + return result; + } + snprintf(prefixed_message, message_len, + "Parse error: %s at line %d, position %d", + parse_result.error->message, + parse_result.error->lineno, + parse_result.error->cursorpos); + result->message = prefixed_message; + char* funcname_copy = parse_result.error->funcname ? safe_strdup(parse_result.error->funcname) : NULL; + char* filename_copy = parse_result.error->filename ? safe_strdup(parse_result.error->filename) : NULL; + char* context_copy = parse_result.error->context ? safe_strdup(parse_result.error->context) : NULL; + + result->funcname = funcname_copy; + result->filename = filename_copy; + result->lineno = parse_result.error->lineno; + result->cursorpos = parse_result.error->cursorpos; + result->context = context_copy; + } else { + result->data = safe_strdup(parse_result.parse_tree); + if (result->data) { + result->data_len = strlen(result->data); + } else { + result->has_error = 1; + result->message = safe_strdup("Memory allocation failed"); + } + } + + pg_query_free_parse_result(parse_result); + return result; +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_detailed_result(WasmDetailedResult* result) { + if (result) { + free(result->message); + free(result->funcname); + free(result->filename); + free(result->context); + free(result->data); + free(result); + } +} + +static const char* get_token_name(PgQuery__Token token_type) { + // Map some common token types to readable names + // Note: This is a simplified mapping - full enum lookup would require more complexity + switch(token_type) { + case 258: return "IDENT"; + case 261: return "SCONST"; + case 266: return "ICONST"; + case 260: return "FCONST"; + case 267: return "PARAM"; + case 40: return "ASCII_40"; // ( + case 41: return "ASCII_41"; // ) + case 42: return "ASCII_42"; // * + case 44: return "ASCII_44"; // , + case 59: return "ASCII_59"; // ; + case 61: return "ASCII_61"; // = + case 268: return "TYPECAST"; + case 272: return "LESS_EQUALS"; + case 273: return "GREATER_EQUALS"; + case 274: return "NOT_EQUALS"; + case 275: return "SQL_COMMENT"; + case 276: return "C_COMMENT"; + default: return "UNKNOWN"; + } +} + +static const char* get_keyword_name(PgQuery__KeywordKind keyword_kind) { + switch(keyword_kind) { + case 0: return "NO_KEYWORD"; + case 1: return "UNRESERVED_KEYWORD"; + case 2: return "COL_NAME_KEYWORD"; + case 3: return "TYPE_FUNC_NAME_KEYWORD"; + case 4: return "RESERVED_KEYWORD"; + default: return "UNKNOWN_KEYWORD"; + } +} + +static char* build_scan_json(PgQuery__ScanResult *scan_result, const char* original_sql) { + if (!scan_result || !original_sql) { + return safe_strdup("{\"version\":0,\"tokens\":[]}"); + } + + // Calculate rough JSON size estimate + size_t estimated_size = 1024 + (scan_result->n_tokens * 200); + char* json = safe_malloc(estimated_size); + if (!json) { + return safe_strdup("{\"version\":0,\"tokens\":[]}"); + } + + // Start building JSON + int pos = snprintf(json, estimated_size, "{\"version\":%d,\"tokens\":[", scan_result->version); + + for (size_t i = 0; i < scan_result->n_tokens; i++) { + PgQuery__ScanToken *token = scan_result->tokens[i]; + + // Extract token text from original SQL + int token_length = token->end - token->start; + if (token_length < 0) token_length = 0; // Safety check + + char* token_text = safe_malloc(token_length + 1); + if (!token_text) continue; + + if (token_length > 0) { + strncpy(token_text, &original_sql[token->start], token_length); + } + token_text[token_length] = '\0'; + + // Escape token text for JSON + char* escaped_text = safe_malloc(token_length * 2 + 1); + if (!escaped_text) { + free(token_text); + continue; + } + + int escaped_pos = 0; + for (int j = 0; j < token_length; j++) { + char c = token_text[j]; + if (c == '"' || c == '\\') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = c; + } else if (c == '\n') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = 'n'; + } else if (c == '\r') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = 'r'; + } else if (c == '\t') { + escaped_text[escaped_pos++] = '\\'; + escaped_text[escaped_pos++] = 't'; + } else { + escaped_text[escaped_pos++] = c; + } + } + escaped_text[escaped_pos] = '\0'; + + // Get token type name and keyword kind name + const char* token_name = get_token_name(token->token); + const char* keyword_name = get_keyword_name(token->keyword_kind); + + // Add comma if not first token + if (i > 0) { + pos += snprintf(json + pos, estimated_size - pos, ","); + } + + // Add token object to JSON + pos += snprintf(json + pos, estimated_size - pos, + "{\"start\":%d,\"end\":%d,\"text\":\"%s\",\"tokenType\":%d,\"tokenName\":\"%s\",\"keywordKind\":%d,\"keywordName\":\"%s\"}", + token->start, token->end, escaped_text, token->token, token_name, token->keyword_kind, keyword_name); + + free(token_text); + free(escaped_text); + + // Check if we're running out of space + if (pos >= estimated_size - 200) { + char* new_json = realloc(json, estimated_size * 2); + if (!new_json) break; + json = new_json; + estimated_size *= 2; + } + } + + // Close JSON + snprintf(json + pos, estimated_size - pos, "]}"); + + return json; +} + +EMSCRIPTEN_KEEPALIVE +char* wasm_scan(const char* input) { + if (!validate_input(input)) { + return safe_strdup("Invalid input: query cannot be null or empty"); + } + + PgQueryScanResult result = pg_query_scan(input); + + if (result.error) { + char* error_msg = safe_strdup(result.error->message); + pg_query_free_scan_result(result); + return error_msg ? error_msg : safe_strdup("Memory allocation failed"); + } + + // Unpack protobuf data + PgQuery__ScanResult *scan_result = pg_query__scan_result__unpack( + NULL, result.pbuf.len, (void *) result.pbuf.data); + + if (!scan_result) { + pg_query_free_scan_result(result); + return safe_strdup("Failed to unpack scan result"); + } + + // Convert to JSON + char* json_result = build_scan_json(scan_result, input); + + // Clean up + pg_query__scan_result__free_unpacked(scan_result, NULL); + pg_query_free_scan_result(result); + + return json_result ? json_result : safe_strdup("{\"version\":0,\"tokens\":[]}"); +} + +EMSCRIPTEN_KEEPALIVE +void wasm_free_string(char* str) { + free(str); +} diff --git a/full/17/test/fingerprint.test.js b/versions/18/test/fingerprint.test.js similarity index 100% rename from full/17/test/fingerprint.test.js rename to versions/18/test/fingerprint.test.js diff --git a/full/17/test/normalize.test.js b/versions/18/test/normalize.test.js similarity index 100% rename from full/17/test/normalize.test.js rename to versions/18/test/normalize.test.js diff --git a/full/18/test/pg18.test.js b/versions/18/test/pg18.test.js similarity index 100% rename from full/18/test/pg18.test.js rename to versions/18/test/pg18.test.js diff --git a/full/17/test/plpgsql.test.js b/versions/18/test/plpgsql.test.js similarity index 100% rename from full/17/test/plpgsql.test.js rename to versions/18/test/plpgsql.test.js diff --git a/full/17/test/scan.test.js b/versions/18/test/scan.test.js similarity index 100% rename from full/17/test/scan.test.js rename to versions/18/test/scan.test.js