Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Generated schema snapshots; codegen emits CRLF on Windows
schema_versioned_docs/** text eol=lf
6 changes: 0 additions & 6 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,6 @@ jobs:
- name: Install NPM dependencies
run: npm ci

- name: Generate schema markdown docs
uses: OvertureMaps/workflows/.github/actions/generate-schema-docs@faa3fa4bbe01d85dc2e2139ff1f74ce70feb7df9 # zizmor: ignore[stale-action-refs] - main
with:
output-dir: ${{ github.workspace }}/docs/schema/reference
schema-ref: 'main'

- name: Build
run: npm run build

Expand Down
14 changes: 1 addition & 13 deletions .github/workflows/production_deploy_documentation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,8 @@ on:
push:
branches: [main]

# Allow running from the actions tab, and lets the schema repo trigger a
# rebuild against a specific released schema version/tag.
# Allow running from the actions tab.
workflow_dispatch:
inputs:
schema-ref:
description: 'Schema repo ref (branch, tag, or SHA) to generate reference docs from.'
type: string
required: true

concurrency:
group: pages
Expand Down Expand Up @@ -45,12 +39,6 @@ jobs:
- name: Install NPM dependencies
run: npm ci --prefer-dedupe

- name: Generate schema markdown docs
uses: OvertureMaps/workflows/.github/actions/generate-schema-docs@faa3fa4bbe01d85dc2e2139ff1f74ce70feb7df9 # zizmor: ignore[stale-action-refs] - main
with:
output-dir: ${{ github.workspace }}/docs/schema/reference
schema-ref: ${{ inputs['schema-ref'] || 'main' }}

- name: Build Docusaurus Pages 🔧
run: npm run build

Expand Down
17 changes: 0 additions & 17 deletions .github/workflows/staging_deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,6 @@ jobs:
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
needs: check-fork
outputs:
schema-sha: ${{ steps.schema-docs.outputs.schema-sha }}
schema-ref: ${{ steps.schema-docs.outputs.schema-ref }}
steps:
- name: Check out the main docs repo repository and build.
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -55,13 +52,6 @@ jobs:

- run: npm ci --omit=dev

- name: Generate schema markdown docs
id: schema-docs
uses: OvertureMaps/workflows/.github/actions/generate-schema-docs@faa3fa4bbe01d85dc2e2139ff1f74ce70feb7df9 # zizmor: ignore[stale-action-refs] - main
with:
output-dir: ${{ github.workspace }}/docs/schema/reference
schema-ref: 'main'

- name: Build Docusaurus website
run: npm run build
env:
Expand Down Expand Up @@ -112,13 +102,7 @@ jobs:
run: |
echo "time=$(date -u +'%b %d, %Y %H:%M UTC')" >> $GITHUB_OUTPUT
echo "short-sha=${PR_HEAD_SHA:0:7}" >> $GITHUB_OUTPUT
schema_ref="${NEEDS_BUILD_OUTPUTS_SCHEMA_REF#refs/heads/}"
schema_ref="${schema_ref#refs/tags/}"
echo "schema-ref-short=$schema_ref" >> $GITHUB_OUTPUT
echo "schema-short-sha=${NEEDS_BUILD_OUTPUTS_SCHEMA_SHA:0:7}" >> $GITHUB_OUTPUT
env:
NEEDS_BUILD_OUTPUTS_SCHEMA_REF: ${{ needs.build.outputs.schema-ref }}
NEEDS_BUILD_OUTPUTS_SCHEMA_SHA: ${{ needs.build.outputs.schema-sha }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}

- name: Comment on PR
Expand All @@ -130,7 +114,6 @@ jobs:
| | |
|-------------------------|----------------------------------------------------------------------|
| 🌍 **Preview site** | ${{ env.STAGING_URL }}${{ env.PREVIEW_PATH }}/index.html |
| 🗂️ **Schema ref** | [${{ steps.deploy-metadata.outputs.schema-ref-short }}@${{ steps.deploy-metadata.outputs.schema-short-sha }}](https://github.com/OvertureMaps/schema/tree/${{ needs.build.outputs.schema-ref }}) |
| 🕐 **Updated** | ${{ steps.deploy-metadata.outputs.time }} |
| 📝 **Commit** | [${{ steps.deploy-metadata.outputs.short-sha }}](${{ github.server_url }}/${{ github.repository }}/commit/${{ github.event.pull_request.head.sha }}) |

Expand Down
6 changes: 3 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ node_modules/
build/
static/maplibre/

# Schema reference docs are generated at build time by the
# generate-schema-docs action; see docs/schema/index.md.
docs/schema/reference/
# Scratch output of the schema doc generator: written by add-schema-version and
# by the schema repo's PR preview workflow (SCHEMA_PREVIEW=true). See README.
schema/reference/
3 changes: 2 additions & 1 deletion .markdownlintignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
# Generated by OvertureMaps/schema — do not lint
docs/schema/reference/
schema/reference/
schema_versioned_docs/
36 changes: 32 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,42 @@ This repository uses [Docusaurus](https://docusaurus.io/) to publish the documen
- `community-projects.json` - source data for all community project cards
- `og-image-cache.json` - cached `og:image` URLs for entries without an explicit `image` field (see [OG Image Cache](#og-image-cache) below)
- `docs/`: The main documentation pages available at docs.overturemaps.org/. The sidebar for these pages is manually curated in the `sidebars.js` file.
- Notice there is no `schema reference` folder. See below.
- `schema/`: Source for the schema reference overview page (`index.md`) and scratch output of the schema doc generator. See below.
- `schema_versioned_docs/`, `schema_versioned_sidebars/`, `schema_versions.json`: committed snapshots of the schema reference for each released schema tag. See below.

## Schema Reference (`docs.overturemaps.org/schema`)

The Overture schema repository [OvertureMaps/schema](https://github.com/OvertureMaps/schema) maintains the official Overture schema as Pydantic models, and the reference pages under `docs.overturemaps.org/schema` are generated directly from those models. This keeps the schema and its documentation permanently in sync.
The Overture schema repository [OvertureMaps/schema](https://github.com/OvertureMaps/schema) maintains the official Overture schema as Pydantic models, and the reference pages under `docs.overturemaps.org/schema` are generated directly from those models.

Every build (CI, PR preview, and production) runs the [`generate-schema-docs`](https://github.com/OvertureMaps/workflows/tree/main/.github/actions/generate-schema-docs) action, which generates Markdown into `docs/schema/reference/` from the schema repository's `main` branch (or a specific `schema-ref` when triggered via `workflow_dispatch`, e.g. from an `overture-schema` release). `docs/schema/reference/` is gitignored and never committed here.
The schema reference is its own [versioned Docusaurus docs instance](https://docusaurus.io/docs/versioning) (plugin id `schema`, config in `docusaurus.config.js`, sidebar in `sidebars-schema.js`), separate from the main `docs/` instance. Only released schema tags are published; there is no rolling "latest" built from `main`. The newest tag in `schema_versions.json` is served at `/schema/`, older tags at `/schema/vX.Y.Z/`, and a version dropdown appears in the navbar on schema pages only.

**If you spot a typo or error under `docs.overturemaps.org/schema`, it is not fixable in this repository.** Open an issue or PR against the docstrings/models in [OvertureMaps/schema](https://github.com/OvertureMaps/schema) instead — any change here will be overwritten on the next build.
Snapshots are generated once per tag and committed under `schema_versioned_docs/version-vX.Y.Z/`. Builds do not call the schema generator, so a production deploy only ever changes the schema pages when a new snapshot lands here.

**If you spot a typo or error under `docs.overturemaps.org/schema`, it is not fixable in this repository.** Open an issue or PR against the docstrings/models in [OvertureMaps/schema](https://github.com/OvertureMaps/schema) instead; the fix appears in the next release's snapshot. Re-snapshotting an existing tag is possible (delete its three artifacts and re-run the script below) but pointless unless the tag itself moved.

The overview page (`schema/index.md`) is copied into each snapshot when it's created. Edits to it need to be applied to the `index.md` in each `schema_versioned_docs/version-*/` too.

### Adding a schema version

Run this after a `vX.Y.Z` tag is published in [OvertureMaps/schema](https://github.com/OvertureMaps/schema/releases). It needs `git`, [`uv`](https://docs.astral.sh/uv/), and `npm install` already done.

```shell
npm run add-schema-version -- v2.0.0
```

The script (`scripts/add-schema-version.mjs`) clones the schema repo at that tag, runs its `overture-codegen` into `schema/reference/`, then runs `docusaurus docs:version:schema <tag>`, which writes:

- `schema_versioned_docs/version-<tag>/` (with links into the schema repo pinned to the tag)
- `schema_versioned_sidebars/version-<tag>-sidebars.json`
- an entry in `schema_versions.json` (kept sorted newest-first; the first entry is what `/schema/` serves)

Commit those three and open a PR. `schema/reference/` is cleaned up afterwards.

Only tags that ship the `overture-schema-codegen` package (v1.17.0 and later) can be added; earlier releases were JSON Schema and have no generator. The script refuses tags that don't match `vX.Y.Z` or are already in `schema_versions.json`.

### Schema PR previews

The schema repo's PR preview workflow checks out this repo, generates Markdown from the PR branch into `schema/reference/`, and builds with `SCHEMA_PREVIEW=true`. In that mode the `schema` instance builds only the `current` version from `schema/reference/` at `/schema/`; committed snapshots, the version dropdown, blog, and community pages are skipped.

## Developing

Expand All @@ -51,6 +78,7 @@ Now navigate to <http://localhost:3000> to see the live preview.
- `npm run serve` - Serve the built site locally
- `npm run deploy` - Deploy the site
- `npm run fetch-og` - Fetch and cache `og:image` metadata for community project entries (see [OG Image Cache](#og-image-cache) below)
- `npm run add-schema-version -- vX.Y.Z` - Snapshot the schema reference for a released schema tag (see [Adding a schema version](#adding-a-schema-version) above)
- `npm run swizzle` - Customize Docusaurus components by "ejecting" them for modification
- `npm run write-translations` - Generate translation files for internationalization
- `npm run write-heading-ids` - Auto-generate heading IDs for better linking
Expand Down
164 changes: 108 additions & 56 deletions docusaurus.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Note: type annotations allow type checking and IDEs autocompletion

const { themes } = require('prism-react-renderer');
const schemaVersions = require('./schema_versions.json');

// both modes use the dark palette — code blocks are forced dark bg in
// light mode too (see custom.css), so light-theme token colors fail contrast.
Expand Down Expand Up @@ -40,6 +41,30 @@ function getLatestOvertureRelease() {

const latestOvertureRelease = getLatestOvertureRelease();

// Schema reference docs are a separate, versioned docs instance (see README
// "Schema Reference"). Only released schema tags are published: snapshots live
// in schema_versioned_docs/, the newest is served at /schema/ and older ones at
// /schema/<tag>/. The `current` version (schema/reference, generated from a
// schema branch by the schema repo's CI) is built only for schema PR previews.
// Released versions are stable, so suppress the default "unmaintained" banner.
// The instance can't live under docs/: nested docs instances get their MDX
// compiled twice.
const [latestSchemaVersion] = schemaVersions;
const latestSchemaDocsDir = `schema_versioned_docs/version-${latestSchemaVersion}`;
const schemaVersionOptions = isSchemaPreview
? {
includeCurrentVersion: true,
onlyIncludeVersions: ['current'],
versions: { current: { label: 'preview', path: '' } },
}
: {
includeCurrentVersion: false,
lastVersion: latestSchemaVersion,
versions: Object.fromEntries(
schemaVersions.map((v) => [v, { label: v, banner: 'none', path: v === latestSchemaVersion ? '' : v }]),
),
};

/** @type {import('@docusaurus/types').Config} */
const config = {
title: 'Overture Documentation',
Expand Down Expand Up @@ -95,62 +120,82 @@ const config = {

themes: [],

plugins: isSchemaPreview
? []
: [
[
'@docusaurus/plugin-client-redirects',
{
redirects: [
{
from: '/releases',
to: '/release-calendar/',
},
],
},
],
[
'@docusaurus/plugin-content-pages',
{
id: 'community',
path: './community',
routeBasePath: 'community',
showLastUpdateTime: true,
},
],
[
'docusaurus-plugin-llms',
{
generateLLMsTxt: true,
generateLLMsFullTxt: true,
excludeImports: true,
removeDuplicateHeadings: true,
includeBlog: true,
// Order docs from most introductory to most detailed
includeOrder: [
'getting-data/**',
'guides/**',
'schema/**',
'examples/**',
'gers/**',
],
includeUnmatchedLast: true,
// Playground pages are interactive components, not useful for LLMs
ignoreFiles: ['playground/**'],
// Schema-only subset for LLMs focused on the data model
customLLMFiles: [
{
filename: 'llms-schema.txt',
includePatterns: ['schema/**'],
fullContent: true,
title: 'Overture Maps Schema Reference',
description:
'Schema reference for all Overture Maps data types: addresses, base, buildings, divisions, places, and transportation.',
},
],
},
],
],
plugins: [
[
'@docusaurus/plugin-content-docs',
/** @type {import('@docusaurus/plugin-content-docs').Options} */
({
id: 'schema',
path: 'schema',
routeBasePath: 'schema',
sidebarPath: require.resolve('./sidebars-schema.js'),
Comment thread
lowlydba marked this conversation as resolved.
showLastUpdateTime: true,
breadcrumbs: false,
...schemaVersionOptions,
}),
],
...(isSchemaPreview
? []
: [
[
'@docusaurus/plugin-client-redirects',
{
redirects: [
{
from: '/releases',
to: '/release-calendar/',
},
],
},
],
[
'@docusaurus/plugin-content-pages',
{
id: 'community',
path: './community',
routeBasePath: 'community',
showLastUpdateTime: true,
},
],
[
'docusaurus-plugin-llms',
{
generateLLMsTxt: true,
generateLLMsFullTxt: true,
// Newest schema version only; older snapshots are excluded
docsDir: [
{ path: 'docs', routeBasePath: '/', label: 'Docs' },
{ path: latestSchemaDocsDir, routeBasePath: 'schema', label: 'Schema Reference' },
],
excludeImports: true,
removeDuplicateHeadings: true,
includeBlog: true,
// Order docs from most introductory to most detailed
includeOrder: [
'getting-data/**',
'guides/**',
`${latestSchemaDocsDir}/**`,
'examples/**',
'gers/**',
],
includeUnmatchedLast: true,
// Playground pages are interactive components, not useful for LLMs
ignoreFiles: ['playground/**'],
// Schema-only subset for LLMs focused on the data model
customLLMFiles: [
{
filename: 'llms-schema.txt',
includePatterns: [`${latestSchemaDocsDir}/**`],
fullContent: true,
title: 'Overture Maps Schema Reference',
description:
'Schema reference for all Overture Maps data types: addresses, base, buildings, divisions, places, and transportation.',
},
],
},
],
]),
],

presets: [
[
Expand Down Expand Up @@ -219,6 +264,13 @@ const config = {
label: 'Community',
position: 'left',
},
{
type: 'docsVersionDropdown',
docsPluginId: 'schema',
position: 'right',
// Only shown on schema pages; see custom.css
className: 'schema-version-dropdown',
},
]
: []),
{
Expand Down
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
"private": true,
"scripts": {
"fetch-og": "node scripts/fetch-og-images.mjs",
"add-schema-version": "node scripts/add-schema-version.mjs",
"copy-maplibre-worker": "node scripts/copy-maplibre-worker.mjs",
"docusaurus": "docusaurus",
"start": "npm run copy-maplibre-worker && npm run docusaurus start",
Expand All @@ -16,8 +17,8 @@
"write-heading-ids": "docusaurus write-heading-ids",
"lint": "eslint .",
"lint:fix": "eslint . --fix",
"lint:md": "markdownlint 'docs/**/*.md' 'docs/**/*.mdx' 'blog/**/*.md' 'blog/**/*.mdx' --ignore node_modules",
"lint:md:fix": "markdownlint 'docs/**/*.md' 'docs/**/*.mdx' 'blog/**/*.md' 'blog/**/*.mdx' --ignore node_modules --fix",
"lint:md": "markdownlint 'docs/**/*.md' 'docs/**/*.mdx' 'schema/*.md' 'blog/**/*.md' 'blog/**/*.mdx' --ignore node_modules",
"lint:md:fix": "markdownlint 'docs/**/*.md' 'docs/**/*.mdx' 'schema/*.md' 'blog/**/*.md' 'blog/**/*.mdx' --ignore node_modules --fix",
"format": "prettier --write .",
"format:check": "prettier --check .",
"test": "vitest run",
Expand Down
2 changes: 1 addition & 1 deletion docs/schema/index.md → schema/index.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
id: index
slug: /schema/
slug: /
title: Schema Reference
sidebar_label: Overview
description: Pydantic schemas for Overture Maps data
Expand Down
Loading
Loading