Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jefbarn/pgx_json_schema
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 0.1.0
Choose a base ref
...
head repository: jefbarn/pgx_json_schema
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref

Commits on Jul 27, 2021

  1. README first pass

    jefbarn committed Jul 27, 2021
    Copy the full SHA
    94cb3d6 View commit details
  2. README edits

    jefbarn committed Jul 27, 2021
    Copy the full SHA
    694bddd View commit details
  3. README edits

    jefbarn committed Jul 27, 2021
    Copy the full SHA
    481f185 View commit details
  4. README edits

    jefbarn committed Jul 27, 2021
    Copy the full SHA
    c5ede30 View commit details
  5. CHANGELOG added

    jefbarn committed Jul 27, 2021
    Copy the full SHA
    5e747aa View commit details

Commits on Jul 28, 2021

  1. Added Dockerfile

    jefbarn committed Jul 28, 2021
    Copy the full SHA
    aefe947 View commit details
  2. Added jtd validation

    jefbarn committed Jul 28, 2021
    Copy the full SHA
    cb8dec4 View commit details
  3. Added avro validation

    jefbarn committed Jul 28, 2021
    Copy the full SHA
    230a467 View commit details
  4. Copy the full SHA
    f4baa52 View commit details
  5. Updated README

    jefbarn committed Jul 28, 2021
    Copy the full SHA
    5a57c45 View commit details

Commits on Dec 15, 2021

  1. Copy the full SHA
    3775d3d View commit details

Commits on Mar 22, 2022

  1. Copy the full SHA
    235af6b View commit details

Commits on Apr 22, 2022

  1. Copy the full SHA
    b7d0db2 View commit details
  2. Merge pull request #1 from Stranger6667/dd/update-jsonschema

    chore: Update `jsonschema` to `0.16.0`
    jefbarn authored Apr 22, 2022
    Copy the full SHA
    d56eb61 View commit details

Commits on Aug 31, 2022

  1. a.k.a supabase/pg_jsonschema#12

    grmt committed Aug 31, 2022
    Copy the full SHA
    7b62d66 View commit details

Commits on Dec 16, 2022

  1. Merge pull request #2 from grmt/feature/disable-http-and-file-features

    Disable file and http default features
    jefbarn authored Dec 16, 2022
    Copy the full SHA
    3d0e2d7 View commit details
  2. 1
    Copy the full SHA
    db2f1dd View commit details
  3. Update README.md

    jefbarn authored Dec 16, 2022
    Copy the full SHA
    86946a2 View commit details
  4. Update README.md

    jefbarn authored Dec 16, 2022
    Copy the full SHA
    4f7668b View commit details
  5. Update README.md

    jefbarn authored Dec 16, 2022
    Copy the full SHA
    fde0196 View commit details
  6. Update CHANGELOG.md

    jefbarn authored Dec 16, 2022
    Copy the full SHA
    00a573b View commit details
Showing with 581 additions and 80 deletions.
  1. +1 −1 .cargo/config
  2. +18 −0 CHANGELOG.md
  3. +13 −10 Cargo.toml
  4. +56 −0 Dockerfile
  5. +157 −1 README.md
  6. +0 −1 sql/load-order.txt
  7. +64 −0 sql/pgx_json_schema-1.0.sql
  8. +101 −0 src/avro.rs
  9. +72 −0 src/json_schema.rs
  10. +95 −0 src/json_type_def.rs
  11. +4 −67 src/lib.rs
2 changes: 1 addition & 1 deletion .cargo/config
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[build]
# Postgres symbols won't ve available until runtime
# Postgres symbols won't be available until runtime
rustflags = ["-C", "link-args=-Wl,-undefined,dynamic_lookup"]
18 changes: 18 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Changelog
All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [0.3.0] - 2022-12-16
### Added
- Updated to latest PGX dependencies.
- Removed file and http access features from jsonschema.

## [0.2.0] - 2021-07-28
### Added
- JSON Type Definition and Apache Avro support

## [0.1.0] - 2021-07-27
### Added
- Initial testing release
23 changes: 13 additions & 10 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,31 +1,34 @@
[package]
name = "pgx_json_schema"
version = "0.1.0"
edition = "2018"
version = "0.3.0"
edition = "2021"

[lib]
crate-type = ["cdylib"]

[features]
default = ["pg11"]
pg10 = ["pgx/pg10"]
default = ["pg14"]
pg11 = ["pgx/pg11"]
pg12 = ["pgx/pg12"]
pg13 = ["pgx/pg13"]
pg14 = ["pgx/pg14"]
pg15 = ["pgx/pg15"]
pg_test = []

[dependencies]
pgx = "0.1.20"
pgx-macros = "0.1.20"
serde_json = "1.0.59"
jsonschema = "0.12.0"
#jtd = "0.3"
pgx = "0.6.1"
pgx-macros = "0.6.1"
serde_json = "1.0.79"
jsonschema = {version = "0.16.1", default-features = false, features = ["draft201909", "draft202012"]}
jtd = "0.3.1"
avro-rs = "0.13.0"

[dev-dependencies]
pgx-tests = "0.1.20"
pgx-tests = "0.6.1"

[profile.dev]
panic = "unwind"
lto = "thin"

[profile.release]
panic = "unwind"
56 changes: 56 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
ARG POSTGRES_VERSION=13
ARG PGX_JSON_SCHEMA_VERSION=main

# Use build stage to keep all the souce code out of the final image
FROM postgres:$POSTGRES_VERSION as build
ARG POSTGRES_VERSION
ARG PGX_JSON_SCHEMA_VERSION

# Update postgres and install libraries
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
postgresql-$POSTGRES_VERSION \
postgresql-server-dev-$POSTGRES_VERSION \
build-essential \
curl \
pkg-config \
libssl-dev \
libreadline-dev \
zlib1g-dev \
flex \
bison \
libxml2-dev \
libxslt-dev \
libxml2-utils \
xsltproc \
&& rm -rf /var/lib/apt/lists/*

USER postgres
SHELL ["/bin/bash", "-c"]

# Install Rust
# https://www.rust-lang.org/tools/install
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="~/.cargo/bin:${PATH}"

# Install PGX
WORKDIR $HOME
RUN cargo install cargo-pgx
RUN cargo pgx init --pg$POSTGRES_VERSION /usr/bin/pg_config

# Download this repo
RUN mkdir $HOME/pgx_json_schema
WORKDIR $HOME/pgx_json_schema/
RUN curl -L "https://github.com/jefbarn/pgx_json_schema/archive/${PGX_JSON_SCHEMA_VERSION}.tar.gz" \
| tar -xz --strip-components=1

# Build and install the extension package
RUN cargo pgx package


# Now create the final image
FROM postgres:$POSTGRES_VERSION
ARG POSTGRES_VERSION

COPY --from=build $HOME/pgx_json_schema/target/release/pgx_json_schema-pg$POSTGRES_VERSION /
158 changes: 157 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1 +1,157 @@
# pgx_json_schema
## pgx_json_schema

A [JSON Schema](https://json-schema.org/) validator for Postgres implemented in Rust

This repo is a lightweight connection between the following excellent packages:
* PGX framework for developing PostgreSQL extensions in Rust [(pgx crate)](https://docs.rs/pgx/)

https://github.com/zombodb/pgx
* jsonschema-rs Rust schema validation library [(jsonschema crate)](https://docs.rs/jsonschema/)

https://github.com/Stranger6667/jsonschema-rs

Supported drafts:

* Draft 7 (except optional idn-hostname)
* Draft 6
* Draft 4 (except optional bignum)

Partially supported drafts (some keywords are not implemented):

* Draft 2019-09
* Draft 2020-12

Bonus support added for:
* [JSON Type Definition (JTD)](https://jsontypedef.com/) via the [jtd crate](https://docs.rs/jtd/)
* [Apache Avro](https://avro.apache.org/) via the [avro_rs crate](https://docs.rs/avro-rs/)

### Installation:

```shell
# Install Rust
# https://www.rust-lang.org/tools/install
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh

# Install PGX
cargo install cargo-pgx
cargo pgx init

# Download this repo
curl -L 'https://github.com/jefbarn/pgx_json_schema/archive/refs/tags/0.1.0.tar.gz' \
| tar -xz --strip-components=1

# Build and install the extension package
cargo pgx package

# Enable the extension in your database
create extension pgx_json_schema;
```

### How to use:

#### JSON Schema
```
select * from json_schema_is_valid('{"maxLength": 5}'::jsonb, '"foobar"'::jsonb);
json_schema_is_valid
----------------------
f
select * from json_schema_get_errors('{"maxLength": 5}'::jsonb, '"foobar"'::jsonb);
error_value | description | details | instance_path | schema_path
------------+--------------------------------------+------------------------+---------------+-------------
"foobar" | "foobar" is longer than 5 characters | MaxLength { limit: 5 } | | /maxLength
```

> **Warning**
> A warning about performance.
>
> Because the jsonschema crate must complile the schema before use, and Postgres uses
> separate heap per thread, this extension must compile the schema every time the function is invoked. This leads to
> pretty terrible performance for validating any large amount of data.
>
> To fix this we'd need to get the jsonschema crate to implement Copy/Clone on the JSONSchema struct and then move the
> compiled schema into shared memory where it could be reused. Will explore this in the future.
#### JSON Type Definition

> **_NOTE:_** The jtd library only reports the position of the validation errors, not a description.
```
select jtd_is_valid('{
"properties": {
"name": { "type": "string" },
"age": { "type": "uint32" },
"phones": {
"elements": {
"type": "string"
}
}
}
}'::jsonb, '{
"age": "43",
"phones": ["+44 1234567", 442345678]
}'::jsonb);
jtd_is_valid
--------------
f
select instance_path, schema_path from jtd_get_errors('{
"properties": {
"name": { "type": "string" },
"age": { "type": "uint32" },
"phones": {
"elements": {
"type": "string"
}
}
}
}', '{
"age": "43",
"phones": ["+44 1234567", 442345678]
}'::jsonb);
instance_path | schema_path
---------------+----------------------------------
/age | /properties/age/type
| /properties/name
/phones/1 | /properties/phones/elements/type
```

#### Apache Avro

> **_NOTE:_** The avro library only does complete validation, there is no way to list the errors.
```
select avro_is_valid('{
"type": "record",
"name": "test",
"fields": [
{"name": "a", "type": "long", "default": 42},
{"name": "b", "type": "string"}
]
}'::jsonb, '{
"a": 27,
"b": "foo"
}'::jsonb);
avro_is_valid
---------------
t
```

### Things left to do:

- [ ] Use shared memory to store compiled validator (potential performance gain)
- [ ] More testing
- [ ] Benchmarking
- [x] Add more schema types like [JTD](https://jsontypedef.com/) and [Avro](https://avro.apache.org/)
- [X] Support newer JSON Schema drafts
- [x] Add Dockerfile with installation example

### Prior Art
- https://github.com/gavinwahl/postgres-json-schema
- https://github.com/furstenheim/is_jsonb_valid
1 change: 0 additions & 1 deletion sql/load-order.txt

This file was deleted.

64 changes: 64 additions & 0 deletions sql/pgx_json_schema-1.0.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
This file is auto generated by pgx.
The ordering of items is not stable, it is driven by a dependency graph.
*/

-- src/json_type_def.rs:4
-- pgx_json_schema::json_type_def::jtd_is_valid
CREATE OR REPLACE FUNCTION "jtd_is_valid"(
"schema" jsonb, /* pgx::datum::json::JsonB */
"instance" jsonb /* pgx::datum::json::JsonB */
) RETURNS bool /* bool */
STRICT
LANGUAGE c /* Rust */
AS 'MODULE_PATHNAME', 'jtd_is_valid_wrapper';

-- src/json_type_def.rs:13
-- pgx_json_schema::json_type_def::jtd_get_errors
CREATE OR REPLACE FUNCTION "jtd_get_errors"(
"schema" jsonb, /* pgx::datum::json::JsonB */
"instance" jsonb /* pgx::datum::json::JsonB */
) RETURNS TABLE (
"instance_path" text, /* alloc::string::String */
"schema_path" text /* alloc::string::String */
)
STRICT
LANGUAGE c /* Rust */
AS 'MODULE_PATHNAME', 'jtd_get_errors_wrapper';

-- src/json_schema.rs:4
-- pgx_json_schema::json_schema::json_schema_is_valid
CREATE OR REPLACE FUNCTION "json_schema_is_valid"(
"schema" jsonb, /* pgx::datum::json::JsonB */
"instance" jsonb /* pgx::datum::json::JsonB */
) RETURNS bool /* bool */
STRICT
LANGUAGE c /* Rust */
AS 'MODULE_PATHNAME', 'json_schema_is_valid_wrapper';

-- src/json_schema.rs:9
-- pgx_json_schema::json_schema::json_schema_get_errors
CREATE OR REPLACE FUNCTION "json_schema_get_errors"(
"schema" jsonb, /* pgx::datum::json::JsonB */
"instance" jsonb /* pgx::datum::json::JsonB */
) RETURNS TABLE (
"error_value" jsonb, /* pgx::datum::json::JsonB */
"description" text, /* alloc::string::String */
"details" text, /* alloc::string::String */
"instance_path" text, /* alloc::string::String */
"schema_path" text /* alloc::string::String */
)
STRICT
LANGUAGE c /* Rust */
AS 'MODULE_PATHNAME', 'json_schema_get_errors_wrapper';

-- src/avro.rs:5
-- pgx_json_schema::avro::avro_is_valid
CREATE OR REPLACE FUNCTION "avro_is_valid"(
"schema" jsonb, /* pgx::datum::json::JsonB */
"instance" jsonb /* pgx::datum::json::JsonB */
) RETURNS bool /* bool */
STRICT
LANGUAGE c /* Rust */
AS 'MODULE_PATHNAME', 'avro_is_valid_wrapper';
Loading