diff --git a/.dockerignore b/.dockerignore index f110a87..6521ce0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,15 @@ +.git +.github/ *.o *.a *.so *.dylib +*.bc *.pc results/ -regression.diffs -regression.out -.git +log/ +tmp_check/ +regression.* +packages/ +benchmark/ +docker-example/ diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e9dae38..fd2c77c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -2,12 +2,19 @@ name: Build and test on: push: # Run on any push pull_request: # Run on PRs + merge_group: +permissions: + contents: read +# Source build + `make installcheck` (pgxn-tools resolves the newest PGDG +# build of each major, so PG 19 tracks the current beta). The pinned, +# authoritative builds and tests are in packages.yml. jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - pg: [17, 18] + pg: [17, 18, 19] container: image: pgxn/pgxn-tools steps: diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml new file mode 100644 index 0000000..9c62ad7 --- /dev/null +++ b/.github/workflows/packages.yml @@ -0,0 +1,248 @@ +name: Packages + +# Builds pg_roaringbitmap .debs against pinned PGDG packages for every +# consumed PG major x Debian x arch, tests each exact package in a clean +# image, and publishes them (with .sha256 sidecars) on v tags. +# "Release inventory" is the single aggregate check that gates the release. + +on: + push: + branches: + - master + tags: + - 'v*' + pull_request: + merge_group: + workflow_dispatch: + +permissions: + contents: read + +jobs: + version: + name: Package version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Set package version + id: version + shell: bash + run: | + if [[ "$GITHUB_REF_TYPE" == tag ]]; then + # Same scheme as v1.2.0-1.pscale1; postgres-private downloads + # postgresql--roaringbitmap_-1_.deb. + if [[ ! "$GITHUB_REF_NAME" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.pscale[0-9]+$ ]]; then + echo "invalid release tag: $GITHUB_REF_NAME" >&2 + exit 1 + fi + version="${GITHUB_REF_NAME#v}-1" + else + version="1.2.0-1.pscale0~ci${GITHUB_RUN_NUMBER}-1" + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + + package: + name: PG ${{ matrix.pg }} / Debian ${{ matrix.debian }} / ${{ matrix.asset_arch }} + needs: version + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + # pg_package_version pins exact PGDG builds, and PGDG drops superseded + # ones (19~beta4 first). The renovate comments use postgres-private's + # deb-datasource form so Renovate can propose bumps as PRs; they are + # inert until Renovate is enabled here with a manager for this file. + include: + - pg: 17 + # renovate: datasource=deb depName=postgresql-17 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=bookworm-pgdg&binaryArch=amd64&components=main + pg_package_version: 17.11-1.pgdg12+2 + debian: 12 + codename: bookworm + base_image: debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe + asset_arch: x86_64 + runner: ubuntu-latest + - pg: 17 + # renovate: datasource=deb depName=postgresql-17 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=bookworm-pgdg&binaryArch=arm64&components=main + pg_package_version: 17.11-1.pgdg12+2 + debian: 12 + codename: bookworm + base_image: debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe + asset_arch: arm64 + runner: ubuntu-24.04-arm + - pg: 18 + # renovate: datasource=deb depName=postgresql-18 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=bookworm-pgdg&binaryArch=amd64&components=main + pg_package_version: 18.6-1.pgdg12+2 + debian: 12 + codename: bookworm + base_image: debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe + asset_arch: x86_64 + runner: ubuntu-latest + - pg: 18 + # renovate: datasource=deb depName=postgresql-18 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=bookworm-pgdg&binaryArch=arm64&components=main + pg_package_version: 18.6-1.pgdg12+2 + debian: 12 + codename: bookworm + base_image: debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe + asset_arch: arm64 + runner: ubuntu-24.04-arm + - pg: 18 + # renovate: datasource=deb depName=postgresql-18 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=trixie-pgdg&binaryArch=amd64&components=main + pg_package_version: 18.6-1.pgdg13+2 + debian: 13 + codename: trixie + base_image: debian:trixie-slim@sha256:28de0877c2189802884ccd20f15ee41c203573bd87bb6b883f5f46362d24c5c2 + asset_arch: x86_64 + runner: ubuntu-latest + - pg: 18 + # renovate: datasource=deb depName=postgresql-18 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=trixie-pgdg&binaryArch=arm64&components=main + pg_package_version: 18.6-1.pgdg13+2 + debian: 13 + codename: trixie + base_image: debian:trixie-slim@sha256:28de0877c2189802884ccd20f15ee41c203573bd87bb6b883f5f46362d24c5c2 + asset_arch: arm64 + runner: ubuntu-24.04-arm + - pg: 19 + # renovate: datasource=deb depName=postgresql-19 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=bookworm-pgdg&binaryArch=amd64&components=19 + pg_package_version: 19~beta4-1.pgdg12+1 + debian: 12 + codename: bookworm + base_image: debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe + asset_arch: x86_64 + runner: ubuntu-latest + - pg: 19 + # renovate: datasource=deb depName=postgresql-19 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=bookworm-pgdg&binaryArch=arm64&components=19 + pg_package_version: 19~beta4-1.pgdg12+1 + debian: 12 + codename: bookworm + base_image: debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe + asset_arch: arm64 + runner: ubuntu-24.04-arm + - pg: 19 + # renovate: datasource=deb depName=postgresql-19 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=trixie-pgdg&binaryArch=amd64&components=19 + pg_package_version: 19~beta4-1.pgdg13+1 + debian: 13 + codename: trixie + base_image: debian:trixie-slim@sha256:28de0877c2189802884ccd20f15ee41c203573bd87bb6b883f5f46362d24c5c2 + asset_arch: x86_64 + runner: ubuntu-latest + - pg: 19 + # renovate: datasource=deb depName=postgresql-19 registryUrl=https://apt.postgresql.org/pub/repos/apt?suite=trixie-pgdg&binaryArch=arm64&components=19 + pg_package_version: 19~beta4-1.pgdg13+1 + debian: 13 + codename: trixie + base_image: debian:trixie-slim@sha256:28de0877c2189802884ccd20f15ee41c203573bd87bb6b883f5f46362d24c5c2 + asset_arch: arm64 + runner: ubuntu-24.04-arm + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Build packages + shell: bash + env: + PACKAGE_VERSION: ${{ needs.version.outputs.version }} + run: | + docker build --target test \ + --build-arg BASE_IMAGE=${{ matrix.base_image }} \ + --build-arg DEBIAN_CODENAME=${{ matrix.codename }} \ + --build-arg DEBIAN_VERSION=${{ matrix.debian }} \ + --build-arg PACKAGE_VERSION="$PACKAGE_VERSION" \ + --build-arg PG_MAJOR=${{ matrix.pg }} \ + --build-arg PG_PACKAGE_VERSION=${{ matrix.pg_package_version }} \ + -t pg-roaringbitmap-test -f Dockerfile.package . + + - name: Test exact packages + shell: bash + env: + PACKAGE_VERSION: ${{ needs.version.outputs.version }} + run: | + docker run --rm pg-roaringbitmap-test \ + --pg ${{ matrix.pg }} \ + --debian ${{ matrix.debian }} \ + --asset-arch ${{ matrix.asset_arch }} \ + --version "$PACKAGE_VERSION" + + - name: Extract packages + shell: bash + run: | + container=$(docker create pg-roaringbitmap-test) + mkdir packages + docker cp "$container:/packages/." packages/ + docker rm "$container" + ls -l packages + + - name: Upload packages + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: pg_roaringbitmap-pg${{ matrix.pg }}-debian${{ matrix.debian }}-${{ matrix.asset_arch }} + path: packages/* + if-no-files-found: error + retention-days: 30 + + inventory: + name: Release inventory + # Always report, so a failed or skipped package row fails this aggregate + # check instead of skipping it (a skipped required check counts as passed). + if: always() + needs: [version, package] + runs-on: ubuntu-latest + steps: + - name: Require every package row to succeed + shell: bash + env: + NEEDS: ${{ toJSON(needs) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.result != "success") | "\(.key)=\(.value.result)"' <<<"$NEEDS") + if [[ -n "$failed" ]]; then + echo "required jobs did not succeed: $(xargs <<<"$failed")" >&2 + exit 1 + fi + + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: pg_roaringbitmap-pg* + path: packages + merge-multiple: true + + - name: Self-test the verifier + run: scripts/test-verify-release-assets + + - name: Verify release assets + run: scripts/verify-release-assets --version "${{ needs.version.outputs.version }}" packages + + release: + if: github.ref_type == 'tag' + needs: [version, inventory] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: pg_roaringbitmap-pg* + path: packages + merge-multiple: true + + - name: Verify release assets + run: scripts/verify-release-assets --version "${{ needs.version.outputs.version }}" packages + + - name: Upload .deb packages and checksums to release + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2 + with: + tag_name: ${{ github.ref_name }} + name: ${{ github.ref_name }} + draft: false + prerelease: false + generate_release_notes: true + files: packages/* + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 3faf001..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,143 +0,0 @@ -name: Release - -on: - push: - tags: - - 'v*' - -jobs: - build: - strategy: - matrix: - include: - - arch: amd64 - runner: ubuntu-latest - - arch: arm64 - runner: ubuntu-latest - fail-fast: false - runs-on: ${{ matrix.runner }} - outputs: - version: ${{ steps.version.outputs.VERSION }} - - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - - name: Set up QEMU - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - if: matrix.arch == 'arm64' - - - name: Get version from tag - id: version - run: | - echo "VERSION=${GITHUB_REF#refs/tags/v}-1" >> $GITHUB_OUTPUT - echo "GIT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - - - name: Build Docker image with packages - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - context: . - file: ./Dockerfile.pscale-debian12 - push: false - load: true - tags: pg-roaringbitmap-builder - platforms: linux/${{ matrix.arch }} - build-args: | - VERSION=${{ steps.version.outputs.VERSION }} - GIT_SHA=${{ steps.version.outputs.GIT_SHA }} - - - name: Extract .deb packages from Docker image - run: | - # Create a temporary container to copy files from - CONTAINER_ID=$(docker create \ - --stop-timeout=1 \ - --entrypoint="" \ - pg-roaringbitmap-builder \ - tail -f /dev/null) - docker start "${CONTAINER_ID}" - - # Copy .deb files from container - docker exec ${CONTAINER_ID} sh -c 'tar -cf - /*.deb' | tar -xf - - - # Clean up the container - docker stop ${CONTAINER_ID} || true - docker rm ${CONTAINER_ID} || true - - # List the .deb files for verification - ls -la *.deb - - - name: Verify package contents - run: | - # Verify that each PostgreSQL version package contains the crucial files - for deb in postgresql-*-roaringbitmap_*.deb; do - echo "Verifying $deb..." - - # Extract the package and check contents - ar x "$deb" data.tar.xz - - # Derive the PostgreSQL major version from the package name - PG_VERSION=$(echo "$deb" | sed -n 's/postgresql-\([0-9]*\)-roaringbitmap_.*/\1/p') - - # Verify .so file exists - if ! tar -tf data.tar.xz | grep -q "./usr/lib/postgresql/${PG_VERSION}/lib/roaringbitmap.so"; then - echo "ERROR: Missing roaringbitmap.so in $deb" - exit 1 - fi - - # Verify control file exists - if ! tar -tf data.tar.xz | grep -q "./usr/share/postgresql/${PG_VERSION}/extension/roaringbitmap.control"; then - echo "ERROR: Missing roaringbitmap.control in $deb" - exit 1 - fi - - # Verify SQL file exists - if ! tar -tf data.tar.xz | grep -q "./usr/share/postgresql/${PG_VERSION}/extension/roaringbitmap--1.2.sql"; then - echo "ERROR: Missing roaringbitmap--1.2.sql in $deb" - exit 1 - fi - - echo "✓ $deb contains all crucial files" - rm -f data.tar.xz - done - echo "All packages verified successfully!" - - - name: Calculate checksums - run: | - for file in *.deb; do - sha256sum "$file" > "$file.sha256" - done - - - name: Upload artifacts - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: pg-roaringbitmap-${{ matrix.arch }} - path: | - *.deb - *.deb.sha256 - retention-days: 30 - - release: - needs: build - runs-on: ubuntu-latest - permissions: - contents: write - - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - - - name: Download all artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: packages/ - - - name: Create Release - uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 - with: - files: | - packages/**/*.deb - packages/**/*.deb.sha256 - generate_release_notes: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index d729983..14795f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # Change Log +### v1.2.0(2026-06-27) +1. Add rb_runoptimize(), to shrink binary size of bitmaps when called(#62 by @notoriousR-O-B and @smmathews-cision-us) +2. Keep rb_exsit definition for backward compatible with verion less than 1.0(#63 by @harry-leq) +3. Add postgres19 compatibility (#67 by @sibowu-aws) +4. Validate untrusted bitmaps in the recv functions(#68 by @piki) + ### v1.1.0(2025-11-10) 1. made rangeend = 0 as unlimited(in rb64_fill,rb64_clear,rb64_flip,rb64_range,rb64_range_cardinality and rb64_select) @@ -79,4 +85,4 @@ ### v0.0.3 (2018-03-31) - fork from https://github.com/zeromax007/gpdb-roaringbitmap and make roaringbitmap to be a PostgreSQL extension -- update the CRoaring to v0.2.39. \ No newline at end of file +- update the CRoaring to v0.2.39. diff --git a/Dockerfile.package b/Dockerfile.package new file mode 100644 index 0000000..e82ca23 --- /dev/null +++ b/Dockerfile.package @@ -0,0 +1,145 @@ +# syntax=docker/dockerfile:1 + +# Builds the PlanetScale pg_roaringbitmap .debs for exactly one PostgreSQL +# major against pinned PGDG packages, then tests the exact packages in a clean +# image. +# +# build: debian/ tree via dpkg-buildpackage (debian/rules skips +# dh_pgxs_test; scripts/test-package runs the suite instead) +# packages: release assets only (canonical names; pg-trixie-- on +# Debian 13) plus .sha256 sidecars +# test: fresh base image + pinned PostgreSQL runtime + scripts/test-package + +ARG BASE_IMAGE=debian:bookworm-slim@sha256:98f4b71de414932439ac6ac690d7060df1f27161073c5036a7553723881bffbe +FROM ${BASE_IMAGE} AS build + +ARG DEBIAN_CODENAME +ARG DEBIAN_VERSION +ARG PACKAGE_VERSION +ARG PG_MAJOR +ARG PG_PACKAGE_VERSION + +RUN case "$DEBIAN_VERSION:$DEBIAN_CODENAME" in \ + 12:bookworm|13:trixie) ;; \ + *) echo "unsupported Debian target: $DEBIAN_VERSION ($DEBIAN_CODENAME)" >&2; exit 1 ;; \ + esac \ + && case "$PG_MAJOR" in \ + 17|18|19) ;; \ + *) echo "unsupported PostgreSQL major: $PG_MAJOR" >&2; exit 1 ;; \ + esac \ + && case "$PG_PACKAGE_VERSION" in \ + "${PG_MAJOR}"[.~]*".pgdg${DEBIAN_VERSION}+"*) ;; \ + *) echo "PG_PACKAGE_VERSION $PG_PACKAGE_VERSION does not match PG $PG_MAJOR / Debian $DEBIAN_VERSION" >&2; exit 1 ;; \ + esac \ + && test -n "$PACKAGE_VERSION" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + debhelper \ + devscripts \ + && rm -rf /var/lib/apt/lists/* + +# PGDG: component "19" only on PG 19 rows, so stable rows cannot resolve beta +# packages. postgresql-server-dev-19 needs libpq-dev (>= 19~~), and component +# 19 carries every beta's libpq, so pin libpq to the server version there. +# Never install postgresql-all / postgresql-server-dev-all (unpinned 10-18). +RUN install -d /usr/share/postgresql-common/pgdg \ + && curl --fail --silent --show-error --output /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + && pgdg_components=main \ + && libpq_pins= \ + && if [ "$PG_MAJOR" = 19 ]; then \ + pgdg_components="main 19"; \ + libpq_pins="libpq5=${PG_PACKAGE_VERSION} libpq-dev=${PG_PACKAGE_VERSION}"; \ + fi \ + && echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${DEBIAN_CODENAME}-pgdg ${pgdg_components}" \ + > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + "postgresql-${PG_MAJOR}=${PG_PACKAGE_VERSION}" \ + "postgresql-client-${PG_MAJOR}=${PG_PACKAGE_VERSION}" \ + "postgresql-server-dev-${PG_MAJOR}=${PG_PACKAGE_VERSION}" \ + postgresql-common-dev \ + $libpq_pins \ + && rm -rf /var/lib/apt/lists/* \ + && majors=$(ls /usr/lib/postgresql | xargs) \ + && if [ "$majors" != "$PG_MAJOR" ]; then \ + echo "/usr/lib/postgresql has '$majors', expected only '$PG_MAJOR'" >&2; exit 1; \ + fi + +COPY . /src/pg_roaringbitmap +WORKDIR /src/pg_roaringbitmap + +# Restrict pg_buildext (debian/control, dh_pgxs_loop) to the one major this +# row builds. +ENV PG_SUPPORTED_VERSIONS=${PG_MAJOR} + +# -d: debian/control Build-Depends names postgresql-all/postgresql-server-dev-all, +# which are deliberately not installed; the exact dependencies are above. +RUN versions=$(pg_buildext supported-versions | xargs) \ + && if [ "$versions" != "$PG_MAJOR" ]; then \ + echo "pg_buildext supported-versions is '$versions', expected '$PG_MAJOR'" >&2; exit 1; \ + fi \ + && pg_buildext updatecontrol \ + && test "$(grep '^Package: ' debian/control)" = "Package: postgresql-${PG_MAJOR}-roaringbitmap" \ + && dch -v "$PACKAGE_VERSION" --distribution "$DEBIAN_CODENAME" "Custom PlanetScale build" \ + && dpkg-buildpackage -us -uc -b -d + +RUN deb_arch=$(dpkg --print-architecture) \ + && case "$deb_arch" in \ + amd64) asset_arch=x86_64 ;; \ + arm64) asset_arch=arm64 ;; \ + *) echo "unsupported package architecture: $deb_arch" >&2; exit 1 ;; \ + esac \ + && prefix= \ + && if [ "$DEBIAN_CODENAME" = trixie ]; then prefix="pg${PG_MAJOR}-trixie-${asset_arch}-"; fi \ + && main="postgresql-${PG_MAJOR}-roaringbitmap_${PACKAGE_VERSION}_${deb_arch}.deb" \ + && dbgsym="postgresql-${PG_MAJOR}-roaringbitmap-dbgsym_${PACKAGE_VERSION}_${deb_arch}.deb" \ + && built=$(cd /src && ls -1 ./*.deb | sed 's|^\./||' | sort | xargs) \ + && if [ "$built" != "$main $dbgsym" ] && [ "$built" != "$dbgsym $main" ]; then \ + echo "unexpected build outputs: $built" >&2; exit 1; \ + fi \ + && install -d /packages \ + && cp "/src/$main" "/packages/${prefix}${main}" \ + && cp "/src/$dbgsym" "/packages/${prefix}${dbgsym}" \ + && cd /packages \ + && for f in *.deb; do sha256sum "$f" > "$f.sha256"; done \ + && ls -l /packages + +FROM scratch AS packages +COPY --from=build /packages/ / + +# Clean image for testing the exact packages: only the pinned PostgreSQL +# runtime from PGDG, with no toolchain or build tree. postgresql-client- +# ships pg_config and pg_regress, so the regression suite runs against the +# installed .so without postgresql-server-dev-. +FROM ${BASE_IMAGE} AS test +ARG PG_MAJOR +ARG PG_PACKAGE_VERSION +ENV PG_MAJOR=${PG_MAJOR} \ + PG_PACKAGE_VERSION=${PG_PACKAGE_VERSION} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc /usr/share/postgresql-common/pgdg/ +COPY --from=build /etc/apt/sources.list.d/pgdg.list /etc/apt/sources.list.d/ + +RUN libpq_pins= \ + && if [ "$PG_MAJOR" = 19 ]; then libpq_pins="libpq5=${PG_PACKAGE_VERSION}"; fi \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + "postgresql-${PG_MAJOR}=${PG_PACKAGE_VERSION}" \ + "postgresql-client-${PG_MAJOR}=${PG_PACKAGE_VERSION}" \ + $libpq_pins \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=packages / /packages/ +COPY sql/ /src/sql/ +COPY expected/ /src/expected/ +COPY scripts/test-package /src/scripts/ +WORKDIR /src +ENTRYPOINT ["/src/scripts/test-package"] diff --git a/Dockerfile.pscale-debian12 b/Dockerfile.pscale-debian12 deleted file mode 100644 index 6b4f375..0000000 --- a/Dockerfile.pscale-debian12 +++ /dev/null @@ -1,42 +0,0 @@ -FROM postgres:18.1-bookworm AS deps - -# VERSION should be injected by the ci release workflow as a build-arg based on the git tag pushed to the repo -ARG VERSION=1.2.0-1.pscale1 -ARG GIT_SHA=unknown -ENV VERSION=${VERSION} -ENV GIT_SHA=${GIT_SHA} - -RUN apt-get -qy update && \ - apt-get install -qy --no-install-recommends \ - architecture-is-64-bit \ - build-essential \ - devscripts \ - debhelper-compat \ - lsb-release \ - postgresql-all \ - postgresql-server-dev-all - -FROM deps AS builder - -COPY . /src -WORKDIR /src - -# this modifies the /src/debian/control.in template to make entries for all supported PostgreSQL versions -RUN pg_buildext updatecontrol - -# this modifies the /src/debian/changelog file used by dpkg-buildpackage to set the version to use on the packages -RUN dch -v "${VERSION}" \ - --distribution "$(lsb_release -cs)" \ - "Custom PlanetScale build" - -# This builds .deb packages for all supported PostgreSQL versions -# and places them in ../ (relative to WORKDIR) -RUN dpkg-buildpackage -us -uc -b - -# final image simply holds the built .deb packages -# -# this is an optimization to speed up CI by avoiding the slow process of -# copying the full builder image into the local docker daemon when all we do -# is start it once to copy out the packages. -FROM postgres:18.1-bookworm -COPY --from=builder /*.deb / diff --git a/META.json b/META.json index 3551f1a..793e825 100644 --- a/META.json +++ b/META.json @@ -2,7 +2,7 @@ "name": "pg_roaringbitmap", "abstract": "A Roaring Bitmap data type", "description": "This library contains a single PostgreSQL extension, a Roaring Bitmap data type called 'roaringbitmap', along with some convenience operators and functions for constructing and handling Roaring Bitmaps.", - "version": "1.1.0", + "version": "1.2.0", "maintainer": [ "Chen Huajun " ], @@ -12,7 +12,7 @@ "abstract": "A Roaring Bitmap data type", "file": "roaringbitmap--1.1.sql", "docfile": "README.md", - "version": "1.1.0" + "version": "1.2.0" } }, "resources": { diff --git a/Makefile b/Makefile index 9ad325c..086e1da 100644 --- a/Makefile +++ b/Makefile @@ -6,8 +6,7 @@ MODULE_big = roaringbitmap OBJS = roaring_buffer_reader.o roaringbitmap.o roaring64_buffer_reader.o roaringbitmap64.o $(OBJS): override CFLAGS += -std=c11 -Wno-error=maybe-uninitialized \ - -Wno-declaration-after-statement -Wno-missing-prototypes \ - -Wno-missing-variable-declarations + -Wno-declaration-after-statement -Wno-missing-prototypes -Wno-missing-variable-declarations PG_CONFIG = pg_config diff --git a/docker-example/Dockerfile b/docker-example/Dockerfile index cd94762..a89c30c 100644 --- a/docker-example/Dockerfile +++ b/docker-example/Dockerfile @@ -1,5 +1,5 @@ FROM postgres:16.0-bookworm AS builder -ENV VERSION_TAG=1.1.0 +ENV VERSION_TAG=1.2.0 WORKDIR / RUN apt-get update && apt-get install -y curl unzip make gcc postgresql-server-dev-16 diff --git a/packaging/release-assets.txt b/packaging/release-assets.txt new file mode 100644 index 0000000..413dd08 --- /dev/null +++ b/packaging/release-assets.txt @@ -0,0 +1,15 @@ +# Release contract for planetscale/pg_roaringbitmap v releases. +# One row per build: major debian asset_arch deb_arch +# Each row ships the main and -dbgsym .deb, each with a .sha256 sidecar +# (4 files per row). Debian 12 assets use the canonical Debian names; Debian 13 +# assets add the release-only prefix pg-trixie--. +17 12 arm64 arm64 +17 12 x86_64 amd64 +18 12 arm64 arm64 +18 12 x86_64 amd64 +18 13 arm64 arm64 +18 13 x86_64 amd64 +19 12 arm64 arm64 +19 12 x86_64 amd64 +19 13 arm64 arm64 +19 13 x86_64 amd64 diff --git a/roaringbitmap.c b/roaringbitmap.c index b4fb4a1..e15847d 100644 --- a/roaringbitmap.c +++ b/roaringbitmap.c @@ -341,13 +341,20 @@ PG_FUNCTION_INFO_V1(roaringbitmap_recv); Datum roaringbitmap_recv(PG_FUNCTION_ARGS) { - Datum dd = DirectFunctionCall1(bytearecv, PG_GETARG_DATUM(0)); - bytea *serializedbytes = DatumGetByteaP(dd); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); + int nbytes; + bytea *serializedbytes; roaring_bitmap_t *r1; size_t expectedsize; const char *reason; - r1 = roaring_bitmap_portable_deserialize_safe(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); + nbytes = buf->len - buf->cursor; + if (nbytes <= 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), + errmsg("empty roaring bitmap binary data"))); + + r1 = roaring_bitmap_portable_deserialize_safe(pq_getmsgbytes(buf, nbytes), nbytes); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), diff --git a/roaringbitmap64.c b/roaringbitmap64.c index 16a1f1d..12ce7a5 100644 --- a/roaringbitmap64.c +++ b/roaringbitmap64.c @@ -213,13 +213,20 @@ PG_FUNCTION_INFO_V1(roaringbitmap64_recv); Datum roaringbitmap64_recv(PG_FUNCTION_ARGS) { - Datum dd = DirectFunctionCall1(bytearecv, PG_GETARG_DATUM(0)); - bytea *serializedbytes = DatumGetByteaP(dd); + StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); + int nbytes; + bytea *serializedbytes; roaring64_bitmap_t *r1; size_t expectedsize; const char *reason; - r1 = roaring64_bitmap_portable_deserialize_safe(VARDATA(serializedbytes), VARSIZE(serializedbytes) - VARHDRSZ); + nbytes = buf->len - buf->cursor; + if (nbytes <= 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), + errmsg("empty roaring bitmap binary data"))); + + r1 = roaring64_bitmap_portable_deserialize_safe(pq_getmsgbytes(buf, nbytes), nbytes); if (!r1) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), diff --git a/scripts/test-package b/scripts/test-package new file mode 100755 index 0000000..24ac08f --- /dev/null +++ b/scripts/test-package @@ -0,0 +1,384 @@ +#!/usr/bin/env bash +# +# Exact-package test for the pg_roaringbitmap release assets of one build row. +# Runs as root in the clean `test` stage of Dockerfile.package: +# +# 1. asset names, .sha256 sidecars, .deb metadata and exact manifests +# (expectations taken from debian/control.in and the v1.2.0-1.pscale1 +# release assets) +# 2. dpkg -i into the clean image (no extra packages may be pulled in) +# 3. initdb + start (roaringbitmap needs no preload); CREATE EXTENSION and +# behavior probes: set operations, aggregates, roaringbitmap64, a binary +# COPY round-trip through *_recv, and corrupt blobs rejected +# 4. the regression suite (sql/*.sql, as the Makefile's REGRESS) against the +# installed .so. postgresql-client- ships pg_regress, so nothing is +# built or added; the .so checksum must not change. +# +# --probes-only skips 1-2 and runs 3-4 against an existing installation +# (--bindir), as the current user unless root. This lets the probes run on a +# developer machine against a source build, without Docker. + +set -euo pipefail + +PG_MAJOR="" +DEBIAN_VERSION="" +ASSET_ARCH="" +PACKAGE_VERSION="" +PACKAGES_DIR=/packages +SRC_DIR=/src +BINDIR="" +PROBES_ONLY=false +PG_PACKAGE_VERSION=${PG_PACKAGE_VERSION:-} + +usage() { + echo "usage: scripts/test-package --pg <17|18|19> --debian <12|13> --asset-arch --version [--packages ] [--src ]" >&2 + echo " scripts/test-package --probes-only --pg <17|18|19> --bindir [--src ]" >&2 + echo " PG_PACKAGE_VERSION must name the pinned PGDG server version (package mode)" >&2 + exit 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --pg) PG_MAJOR=${2:-}; shift 2 ;; + --debian) DEBIAN_VERSION=${2:-}; shift 2 ;; + --asset-arch) ASSET_ARCH=${2:-}; shift 2 ;; + --version) PACKAGE_VERSION=${2:-}; shift 2 ;; + --packages) PACKAGES_DIR=${2:-}; shift 2 ;; + --src) SRC_DIR=${2:-}; shift 2 ;; + --bindir) BINDIR=${2:-}; shift 2 ;; + --probes-only) PROBES_ONLY=true; shift ;; + *) echo "test-package: unknown argument: $1" >&2; usage ;; + esac +done + +[[ "$PG_MAJOR" =~ ^(17|18|19)$ ]] && [[ -d "$SRC_DIR/sql" && -d "$SRC_DIR/expected" ]] || usage +if $PROBES_ONLY; then + [[ -x "$BINDIR/pg_config" ]] || usage +else + if [[ ! "$DEBIAN_VERSION" =~ ^(12|13)$ ]] || [[ ! "$ASSET_ARCH" =~ ^(x86_64|arm64)$ ]] || + [[ -z "$PACKAGE_VERSION" ]] || [[ -z "$PG_PACKAGE_VERSION" ]] || [[ ! -d "$PACKAGES_DIR" ]]; then + usage + fi + BINDIR="/usr/lib/postgresql/${PG_MAJOR}/bin" +fi + +fail() { + echo "test-package: FAIL: $*" >&2 + exit 1 +} + +expect_eq() { + local what=$1 actual=$2 expected=$3 + [[ "$actual" == "$expected" ]] || fail "$what: expected '$expected', got '$actual'" +} + +sha256() { + if command -v sha256sum >/dev/null; then sha256sum "$@"; else shasum -a 256 "$@"; fi +} + +PKG="postgresql-${PG_MAJOR}-roaringbitmap" +PKGLIBDIR=$("$BINDIR/pg_config" --pkglibdir) + +if ! $PROBES_ONLY; then + case "$ASSET_ARCH" in + x86_64) DEB_ARCH=amd64 ;; + arm64) DEB_ARCH=arm64 ;; + esac + PREFIX="" + if [[ "$DEBIAN_VERSION" == 13 ]]; then + PREFIX="pg${PG_MAJOR}-trixie-${ASSET_ARCH}-" + fi + MAIN_ASSET="${PREFIX}${PKG}_${PACKAGE_VERSION}_${DEB_ARCH}.deb" + DBGSYM_ASSET="${PREFIX}${PKG}-dbgsym_${PACKAGE_VERSION}_${DEB_ARCH}.deb" + MAIN_DEB="$PACKAGES_DIR/$MAIN_ASSET" + DBGSYM_DEB="$PACKAGES_DIR/$DBGSYM_ASSET" + expect_eq "pkglibdir" "$PKGLIBDIR" "/usr/lib/postgresql/${PG_MAJOR}/lib" + + echo "test-package: PG ${PG_MAJOR} / Debian ${DEBIAN_VERSION} / ${ASSET_ARCH} / ${PACKAGE_VERSION}" + + # --- pinned runtime ----------------------------------------------------------- + expect_eq "postgresql-${PG_MAJOR} version" \ + "$(dpkg-query -W -f='${Version}' "postgresql-${PG_MAJOR}")" "$PG_PACKAGE_VERSION" + expect_eq "/usr/lib/postgresql" "$(find /usr/lib/postgresql -mindepth 1 -maxdepth 1 -printf '%f\n' | sort | xargs)" "$PG_MAJOR" + if [[ "$PG_MAJOR" != 19 ]] && grep -Eq -- '-pgdg .*\b19\b' /etc/apt/sources.list.d/pgdg.list; then + fail "PGDG component 19 is enabled on a PG ${PG_MAJOR} row" + fi + + # --- 1. asset set, checksums -------------------------------------------------- + # Same contract as v1.2.0-1.pscale1: main + dbgsym, each with a sidecar + # holding `sha256sum ` output. + expected_assets=$(printf '%s\n' "$MAIN_ASSET" "$MAIN_ASSET.sha256" "$DBGSYM_ASSET" "$DBGSYM_ASSET.sha256" | sort) + actual_assets=$(find "$PACKAGES_DIR" -mindepth 1 -maxdepth 1 -printf '%f\n' | sort) + if [[ "$actual_assets" != "$expected_assets" ]]; then + diff <(echo "$expected_assets") <(echo "$actual_assets") >&2 || true + fail "asset set in $PACKAGES_DIR does not match" + fi + for asset in "$MAIN_ASSET" "$DBGSYM_ASSET"; do + expect_eq "$asset.sha256" "$(cat "$PACKAGES_DIR/$asset.sha256")" \ + "$(cd "$PACKAGES_DIR" && sha256sum "$asset")" + done + + # --- metadata ----------------------------------------------------------------- + # From debian/control.in; matches v1.2.0-1.pscale1 (no Breaks field there). + field() { dpkg-deb --field "$1" "$2"; } + expect_eq Package "$(field "$MAIN_DEB" Package)" "$PKG" + expect_eq Version "$(field "$MAIN_DEB" Version)" "$PACKAGE_VERSION" + expect_eq Architecture "$(field "$MAIN_DEB" Architecture)" "$DEB_ARCH" + expect_eq Source "$(field "$MAIN_DEB" Source)" pg-roaringbitmap + expect_eq Section "$(field "$MAIN_DEB" Section)" libs + expect_eq Priority "$(field "$MAIN_DEB" Priority)" optional + expect_eq Maintainer "$(field "$MAIN_DEB" Maintainer)" "PlanetScale " + expect_eq Homepage "$(field "$MAIN_DEB" Homepage)" https://github.com/planetscale/pg_roaringbitmap + depends=$(field "$MAIN_DEB" Depends) + [[ "$depends" =~ ^postgresql-${PG_MAJOR},\ libc6\ \(\>=\ [0-9.]+\)$ ]] || fail "Depends: $depends" + expect_eq Breaks "$(field "$MAIN_DEB" Breaks)" "" + + expect_eq dbgsym.Package "$(field "$DBGSYM_DEB" Package)" "$PKG-dbgsym" + expect_eq dbgsym.Version "$(field "$DBGSYM_DEB" Version)" "$PACKAGE_VERSION" + expect_eq dbgsym.Architecture "$(field "$DBGSYM_DEB" Architecture)" "$DEB_ARCH" + expect_eq dbgsym.Section "$(field "$DBGSYM_DEB" Section)" debug + expect_eq dbgsym.Depends "$(field "$DBGSYM_DEB" Depends)" "$PKG (= $PACKAGE_VERSION)" + build_id=$(field "$DBGSYM_DEB" Build-Ids) + [[ "$build_id" =~ ^[0-9a-f]{40}$ ]] || fail "dbgsym Build-Ids: $build_id" + + # --- manifests ---------------------------------------------------------------- + # Path column of `dpkg-deb --contents` (symlinks keep their "-> target"). + # changelog.gz is the upstream CHANGELOG.md (dh_installchangelogs). + manifest() { dpkg-deb --contents "$1" | awk '{ $1 = $2 = $3 = $4 = $5 = ""; sub(/^ +/, ""); print }' | sort; } + + ext=./usr/share/postgresql/${PG_MAJOR}/extension + expected_main=$(sort < ${PKG} +EOF + ) + for kind in main dbgsym; do + if [[ "$kind" == main ]]; then deb=$MAIN_DEB expected=$expected_main; else deb=$DBGSYM_DEB expected=$expected_dbgsym; fi + actual=$(manifest "$deb") + if [[ "$actual" != "$expected" ]]; then + diff <(echo "$expected") <(echo "$actual") >&2 || true + fail "$kind manifest differs" + fi + done + + # --- 2. clean install --------------------------------------------------------- + before=$(dpkg-query -W -f='${Package}\n' | sort) + dpkg -i "$MAIN_DEB" "$DBGSYM_DEB" + after=$(dpkg-query -W -f='${Package}\n' | sort) + expect_eq "packages added by install" "$(comm -13 <(echo "$before") <(echo "$after") | xargs)" "$PKG $PKG-dbgsym" + expect_eq "installed version" "$(dpkg-query -W -f='${Version}' "$PKG")" "$PACKAGE_VERSION" + dpkg --verify "$PKG" "$PKG-dbgsym" || fail "dpkg --verify" +fi + +SO="" +for suffix in .so .dylib; do + [[ -f "$PKGLIBDIR/roaringbitmap$suffix" ]] && SO="$PKGLIBDIR/roaringbitmap$suffix" && break +done +[[ -n "$SO" ]] || fail "roaringbitmap library missing in $PKGLIBDIR" +so_sha=$(sha256 "$SO") + +# --- cluster helpers ------------------------------------------------------------ +if [[ $EUID -eq 0 ]]; then + WORK=/tmp/roaringbitmap-test + RUN_AS=(runuser -u postgres --) +else + WORK=$(mktemp -d) + RUN_AS=() +fi +SOCK=$WORK/sock +PORT=55435 +PGDATA_DIR=$WORK/data +LOG=$WORK/postgres.log +export PGHOST=$SOCK PGPORT=$PORT + +as_pg_user() { ${RUN_AS[@]+"${RUN_AS[@]}"} env PATH="$BINDIR:$PATH" "$@"; } + +# mkdir_pg ...: directories owned by the user running the server +mkdir_pg() { + mkdir -p "$@" + [[ $EUID -ne 0 ]] || chown postgres:postgres "$@" +} + +psql_q() { "$BINDIR/psql" -X -q -A -t -v ON_ERROR_STOP=1 -U postgres -d postgres "$@"; } + +start_cluster() { + stop_cluster + rm -rf "$PGDATA_DIR" "$SOCK" "$LOG" + mkdir_pg "$WORK" "$SOCK" + as_pg_user initdb -D "$PGDATA_DIR" -U postgres --auth=trust --no-locale -E UTF8 >/dev/null + local opts="-c port=$PORT -c listen_addresses='' -c unix_socket_directories=$SOCK" + as_pg_user pg_ctl -D "$PGDATA_DIR" -l "$LOG" -w -o "$opts" start >/dev/null || + { cat "$LOG" >&2; fail "server did not start"; } + expect_eq "server major" "$(psql_q -c "SELECT current_setting('server_version_num')::int / 10000")" "$PG_MAJOR" +} + +stop_cluster() { + if [[ -f "$PGDATA_DIR/postmaster.pid" ]]; then + as_pg_user pg_ctl -D "$PGDATA_DIR" -m fast -w stop >/dev/null + fi + if [[ -f "$LOG" ]] && grep -E 'PANIC|TRAP:|terminated by signal|server process .* was terminated' "$LOG" >&2; then + fail "server log has crashes" + fi + return 0 +} + +cleanup() { + stop_cluster || true + [[ $EUID -eq 0 ]] || rm -rf "$WORK" +} +trap cleanup EXIT + +# probe