From e7d68549f57e724fbe12e728cae8102a09fcc8d4 Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:53:03 +0200 Subject: [PATCH 1/9] Design note: setting helpers (M5 part 3c) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 85 +++++++++++++++++++++++++++++++ docs/decisions/web-interface.md | 1 + 2 files changed, 86 insertions(+) create mode 100644 docs/decisions/setting-helpers.md diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md new file mode 100644 index 0000000..4f4a554 --- /dev/null +++ b/docs/decisions/setting-helpers.md @@ -0,0 +1,85 @@ +# Setting helpers + +Status: proposed (M5 part 3c, issue #238). Talked through with txoof on 2026-10-10; the open questions at the end are still to be answered. + +## Problem + +Some plugin settings are hard to fill in by hand: +- `met_no` and `moon_phase` need latitude and longitude. Most people don't know them for their town. `met_no` also gives better temperatures with the place's altitude (issue #176; met.no calls altitude "optional but recommended", because its own height map is coarse in hilly places). +- The planned Dutch public transport plugin needs stop and station codes. Users should find them by name and pick from a list, never type codes. The NS station search needs the NS API key typed in the same form. +- `lms_client` (M7) needs the address of a music server on the home network. PaperPi can find it by asking the network. +- A later plugin may need the user to log in to an outside account (a calendar, Spotify): open a link on another device, enter a code, wait until it is confirmed, then save a token. + +v1 had a command per plugin for latitude and longitude (`met_no.get_coord`). It used OpenStreetMap's Nominatim, printed only the first match (so a town with a common name could give the wrong place without anyone noticing), had no time limit, and didn't send the contact name Nominatim's rules ask for. + +v2 so far: a setting can name a helper (`setting(..., helper="location")`), and the web code has a list of helpers that each return a piece of HTML for the settings page (`src/paperpi/web/helpers.py`). The list is still empty. With that design every new helper means new web code, and a plugin can't bring a helper of its own. + +## Options considered + +1. **Each helper is written in the web code** (the design so far). Rejected: every plugin with a special need changes the web code, and a plugin's own search (the Dutch stops) would live outside the plugin. +2. **A plugin sends its own HTML or script.** Rejected: + - It would run inside the logged-in page, where it could read and change every setting, API keys too. The pages now allow only PaperPi's own files (`default-src 'self'`), and that rule would have to be loosened. + - Each plugin's part would look different and could break on phones or when the web interface changes. + - It can't be used from the command line. + - A safer form exists: the plugin's page runs in a locked box inside the page (a "sandboxed iframe") that can't reach the rest. That is a lot of work no plugin needs yet; it stays on the "Later" list. +3. **A map to search on** (a small map library such as Leaflet, with map pictures from OpenStreetMap). Rejected by txoof on 2026-10-10: a place or stop is searched once, when the plugin is set up, so a map is too much work and upkeep for it. +4. **A helper is a plain function that returns data; PaperPi shows one standard search box and pick list for all of them.** **Chosen.** + +## Decision (proposed) + +### What a helper is + +- A helper is a Python function with a few texts for the page: + - the text above the search box, e.g. "Place name, e.g. Berlin or Morrison, Colorado"; + - the button text (default "Search"). A helper that needs no typed text (finding music servers) shows only a button, e.g. "Find servers"; + - where the answers come from, with a link, shown under the list (e.g. "Search by Open-Meteo"). The user sees that their typed text goes to an outside service, and the credit that service asks for is given; + - optional: settings that must be filled in first (the NS API key). Until they are, the box says so instead of searching. +- The function gets what the user typed and the other values in the form, checked as Save would check them. An empty secret field (an API key) gives the saved secret, as in a preview (`web-interface.md`). +- It returns a list of **choices**. Each choice has a label ("Morrison"), a detail line ("Colorado, United States, 1757 m") and the settings it fills in, e.g. `lat`, `lon` and `altitude`. +- It may also stop with a message for the user ("The NS API key was not accepted"). Any other error shows "The search failed" and the details go to the log. +- It uses `paperpi.webrequest` for web requests, like plugins do, so the same time limits and size limits apply. + +### Where helpers come from + +- **PaperPi's own helpers** are for needs many plugins share. The first is `location`. +- **A plugin's own helpers** are part of the plugin, next to its settings and layouts (e.g. a stop search in the Dutch transport plugin). +- A setting names its helper as now: `setting(..., helper="location")`. The plugin's own helpers are looked up first, then PaperPi's. A name that is found nowhere fails the plugin check, so a typing mistake is caught by the tests and not on the user's page. +- The search box is shown under the setting that names the helper. A choice may fill in several settings; PaperPi fills in only settings the plugin has, and checks the values with the plugin's settings description before they go into the form. + +### How a helper runs + +- In a new, short-lived process with a time limit, as plugin updates and previews are, so a helper that hangs or crashes can't stop the web interface or PaperPi. Time limit: 30 seconds, the same as one web request with its retry (`errors-and-time-limits.md`). +- One search at a time; a second one meanwhile is asked to wait, as with the Preview button. +- The typed text may be at most 100 characters. At most 10 choices are shown. + +### On the settings page + +- Under the field: the text, a search box and the button. After a search, the list of choices appears, each with a **Use** button. "Nothing found" comes with a hint (check the spelling, or try a larger place). +- **Use** fills in the form's fields. **Nothing is saved yet**: the user sees the values, can change them, can press Preview, and then Save. The list stays until the next search, so a wrong pick is easy to change. +- No new JavaScript library: htmx (already shipped) sends the search and puts the values in the fields. +- Typing the values by hand keeps working. When the service can't be reached, the page says so and the fields are left as they are. + +### Later (described so they fit, not built until a plugin needs them) + +- **Logging in to an outside account:** a helper may return a **step** instead of choices: a message with a link and a code ("Open example.com/device on your phone and enter ABCD-1234"). PaperPi then calls the helper again every few seconds, each call in its own process with its own time limit, until it returns a choice (the token, which fills a secret setting) or stops with a message. The rest of the structure stays the same. +- **Sandboxed plugin pages** (option 2 in its safe form), if a plugin ever needs more than a search box and a list. + +### The first helper: `location` + +- Uses the place search of **Open-Meteo** (`geocoding-api.open-meteo.com`), chosen with txoof on 2026-10-10. No key and no sign-up. It finds towns and cities (not street addresses) and returns region, country, latitude, longitude **and altitude**; Nominatim, which v1 used, returns no altitude. It understands "Morrison, Colorado"; a plain "Morrison" returns 10 towns in several countries, so region and country are shown with each choice. +- Terms: free for non-commercial use, fewer than 10,000 requests a day. The data is CC BY 4.0 and the place data is from GeoNames, so the page shows "Search by Open-Meteo, place data from GeoNames" with links next to the list. +- A choice fills in `lat` and `lon` (4 decimals, about 10 m), `altitude` (whole metres, as met.no asks) and, when it is still empty, the name shown on the screen (`place` in `met_no`). +- `met_no` gets an optional `altitude` setting (whole metres, -500 to 9000), sent to met.no when it is filled in. This closes issue #176. The example in #176 had a sign mistake: `lon = 84.31` is on the Tibetan plateau, where met.no's 4949 m is right; Atlanta is at `-84.31`. +- `moon_phase` uses it for `lat` and `lon` only; altitude doesn't matter for it. + +### Planned PRs + +1. This note, the helper structure (functions, choices, the separate process, plugin check) and the search box with its pick list, tested with a helper in the `debugging` plugin. About 500 lines. +2. The `location` helper with Open-Meteo, `altitude` for `met_no`, and both plugins using it. About 400 lines. + +## Open questions + +To be answered with txoof one at a time: +1. Does the search start with the button (and Enter), or while the user types? +2. Should helpers also work from the command line on the Pi (e.g. `paperpi find "Morrison, Colorado"`, printing the choices as lines to copy into the config file), and if so in these PRs or later? +3. Should `location` fill in the name shown on the screen when it is empty, as proposed above? diff --git a/docs/decisions/web-interface.md b/docs/decisions/web-interface.md index 4657d37..f9fed94 100644 --- a/docs/decisions/web-interface.md +++ b/docs/decisions/web-interface.md @@ -34,6 +34,7 @@ In v2 the web interface is the main way to set up and change PaperPi: plugins, s - One page handles every plugin. It reads the plugin's settings description (see `plugin-interface.md`) and builds the form from it: number field, dropdown, password field for API keys, and so on. - On Save, the values are checked with the same description. Errors are shown next to the field. A saved change applies at once (see `live-config-reload.md`). - A setting that needs more than a plain field can name an input type from a small list in the web interface, e.g. `location` (lat/lon lookup), `server_search` (find music servers), `layout_picker`. New input types are added when a plugin needs one. *Built in M5 part 3a-2:* a plugin names one with `paperpi.plugin.setting(..., helper="location")`; the web interface's list (`paperpi.web.helpers`) holds a function per name that adds its part under the field. A name it doesn't know adds nothing, so the plain field still works. The first helper is `location` (part 3c). + - *Update (M5 part 3c, 2026-10-10):* replaced by setting helpers: a helper is a function in PaperPi or in the plugin that returns choices as data, and the web interface shows one standard search box and pick list for all of them. See `setting-helpers.md`. - Plugins added later show up without changes to the web code. - *Agreed with txoof on 2026-10-08 (M5 part 3a):* the page shows the plugin's `name` first, then its own settings, then display time, refresh, layout and level; the other shared settings (time limit, alert and storage settings) are folded under "More settings", which opens by itself when one of them has an error or a value that is not the default. Each field shows its value, or the default when the file doesn't set it. - How a save works (M5 part 3a-1): only settings that changed are written. An empty field means "the default" (an empty check box: off; an empty list: nothing picked); a value changed to the default is taken out of the file and becomes the comment `# key = default` again (a value left as it is stays, also one written in the file that equals the default), so the help text above it still fits. A new setting takes the place of its `# key = ...` comment line when there is one. If any value has an error, nothing is saved; each error is shown next to its field, with what was typed (never a secret). A secret (such as an API key) is never shown on the page; leaving its field empty keeps the saved one. A setting the form can't show (a group of settings, a list of numbers) is shown as it is in the file, to be changed there. A setting written over several lines in the file is never changed by the page. From 742ffe527eea9d2c17bcb352ab751bd371249152 Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:06:03 +0200 Subject: [PATCH 2/9] Setting helpers: search starts with the button (txoof) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index 4f4a554..e23f4aa 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -54,7 +54,7 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and ### On the settings page -- Under the field: the text, a search box and the button. After a search, the list of choices appears, each with a **Use** button. "Nothing found" comes with a hint (check the spelling, or try a larger place). +- Under the field: the text, a search box and the button. The search starts with the button or the Enter key, not while the user types (txoof, 2026-10-10): one request per search, which matters for services with a daily limit such as NS. After a search, the list of choices appears, each with a **Use** button. "Nothing found" comes with a hint (check the spelling, or try a larger place). - **Use** fills in the form's fields. **Nothing is saved yet**: the user sees the values, can change them, can press Preview, and then Save. The list stays until the next search, so a wrong pick is easy to change. - No new JavaScript library: htmx (already shipped) sends the search and puts the values in the fields. - Typing the values by hand keeps working. When the service can't be reached, the page says so and the fields are left as they are. @@ -80,6 +80,6 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and ## Open questions To be answered with txoof one at a time: -1. Does the search start with the button (and Enter), or while the user types? +1. ~~Does the search start with the button (and Enter), or while the user types?~~ Button and Enter (txoof, 2026-10-10). 2. Should helpers also work from the command line on the Pi (e.g. `paperpi find "Morrison, Colorado"`, printing the choices as lines to copy into the config file), and if so in these PRs or later? 3. Should `location` fill in the name shown on the screen when it is empty, as proposed above? From 299416ac5c86746a55f74cfe130378ae2df31077 Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:11:17 +0200 Subject: [PATCH 3/9] Setting helpers: command line later, as paperpi helper . (txoof) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index e23f4aa..2a94d79 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -61,6 +61,8 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and ### Later (described so they fit, not built until a plugin needs them) +- **On the command line** (txoof, 2026-10-10: later, as its own issue): `paperpi helper . ""`, e.g. `paperpi helper met_no.location "Morrison, Colorado"`. It prints the choices with the lines to copy into the plugin's block in the config file. It names the plugin type, not a plugin `id`, so it works before the plugin is in the config file. To decide in that issue: how a helper that needs other settings first (the NS API key) gets them, e.g. from a plugin block in the config file or typed after the text. + - **Logging in to an outside account:** a helper may return a **step** instead of choices: a message with a link and a code ("Open example.com/device on your phone and enter ABCD-1234"). PaperPi then calls the helper again every few seconds, each call in its own process with its own time limit, until it returns a choice (the token, which fills a secret setting) or stops with a message. The rest of the structure stays the same. - **Sandboxed plugin pages** (option 2 in its safe form), if a plugin ever needs more than a search box and a list. @@ -81,5 +83,5 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and To be answered with txoof one at a time: 1. ~~Does the search start with the button (and Enter), or while the user types?~~ Button and Enter (txoof, 2026-10-10). -2. Should helpers also work from the command line on the Pi (e.g. `paperpi find "Morrison, Colorado"`, printing the choices as lines to copy into the config file), and if so in these PRs or later? +2. ~~Should helpers also work from the command line on the Pi, and if so in these PRs or later?~~ Later, as its own issue, in the form `paperpi helper met_no.location "Morrison, Colorado"` (txoof, 2026-10-10; see "Later"). 3. Should `location` fill in the name shown on the screen when it is empty, as proposed above? From 3dcba09ab9eeed2ac8daebd61455ec772ed31b20 Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:22:55 +0200 Subject: [PATCH 4/9] Setting helpers: choices are dictionaries of kinds (fill, show, link, note; more planned) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 84 +++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 21 deletions(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index 2a94d79..62da91c 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -1,6 +1,6 @@ # Setting helpers -Status: proposed (M5 part 3c, issue #238). Talked through with txoof on 2026-10-10; the open questions at the end are still to be answered. +Status: proposed (M5 part 3c, issue #238). Decided with txoof on 2026-10-10. ## Problem @@ -23,28 +23,68 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and - It can't be used from the command line. - A safer form exists: the plugin's page runs in a locked box inside the page (a "sandboxed iframe") that can't reach the rest. That is a lot of work no plugin needs yet; it stays on the "Later" list. 3. **A map to search on** (a small map library such as Leaflet, with map pictures from OpenStreetMap). Rejected by txoof on 2026-10-10: a place or stop is searched once, when the plugin is set up, so a map is too much work and upkeep for it. -4. **A helper is a plain function that returns data; PaperPi shows one standard search box and pick list for all of them.** **Chosen.** +4. **A helper is a plain function of the plugin that returns data (a dictionary per choice); PaperPi shows one standard search box and pick list for all of them.** **Chosen.** ## Decision (proposed) +### Who does what + +- **PaperPi owns the page:** a search box, a list of choices, and putting the chosen one into the form. It doesn't need to know what a place, a stop or a server is. +- **The plugin owns its settings:** what they mean and which ones a search changes. So the logic of a search lives in the plugin, and a new kind of plugin needs nothing new from PaperPi. +- Agreed with txoof on 2026-10-10. An earlier draft gave each choice fixed parts (a label, a detail line, values that are "set" or "suggested") guessed from the few plugins we know; a dictionary of kinds (below) fits plugins nobody has thought of yet. + ### What a helper is -- A helper is a Python function with a few texts for the page: +- A helper is a Python function of the plugin, with a few texts for the page: - the text above the search box, e.g. "Place name, e.g. Berlin or Morrison, Colorado"; - the button text (default "Search"). A helper that needs no typed text (finding music servers) shows only a button, e.g. "Find servers"; - where the answers come from, with a link, shown under the list (e.g. "Search by Open-Meteo"). The user sees that their typed text goes to an outside service, and the credit that service asks for is given; - optional: settings that must be filled in first (the NS API key). Until they are, the box says so instead of searching. -- The function gets what the user typed and the other values in the form, checked as Save would check them. An empty secret field (an API key) gives the saved secret, as in a preview (`web-interface.md`). -- It returns a list of **choices**. Each choice has a label ("Morrison"), a detail line ("Colorado, United States, 1757 m") and the settings it fills in, e.g. `lat`, `lon` and `altitude`. +- The function gets what the user typed and the form's current settings, checked as Save would check them. An empty secret field (an API key) gives the saved secret, as in a preview (`web-interface.md`). +- It returns a **result**: a list of **choices**, and optionally a `note` for the whole result (below). - It may also stop with a message for the user ("The NS API key was not accepted"). Any other error shows "The search failed" and the details go to the log. - It uses `paperpi.webrequest` for web requests, like plugins do, so the same time limits and size limits apply. +### What a choice is + +A choice is a dictionary. Each key is a **kind** that says what PaperPi does with its value. For a Dutch transport stop: + +```python +{ + "fill": {"stop_code": "3104", "stop_name": "Nieuwe Duinweg"}, + "show": {"Stop": "Nieuwe Duinweg", "Town": "Den Haag", "Lines": "9, 21", "Distance": "120 m"}, + "link": {"See on map": "https://www.openstreetmap.org/?mlat=52.09&mlon=4.29"}, +} +``` + +**Kinds built now** (all four are used by the first helper, `location`): + +| Kind | Where | What PaperPi does with it | +|---|---|---| +| `fill` | choice | Settings put into the form when the user presses **Use**. The keys are the plugin's own setting names. PaperPi checks them with the plugin's settings description, as Save does, so a wrong name or value is an error and never a broken setting. Only what is in `fill` changes, so a helper leaves a setting alone by leaving it out (e.g. the name shown on the screen, when the user has already typed one). | +| `show` | choice | Information to help the user pick: label and text, shown in the order given. The first entry is the heading of the choice. | +| `link` | choice | Web addresses the user can open in a new tab, e.g. "See on map". Only `http` and `https` addresses; others are left out and logged. | +| `note` | result | A message for the whole result, e.g. "Only the first 10 matches are shown; type more to narrow it down". | + +**Kinds planned** (same dictionary shape; each is built with the first plugin that needs it): + +| Kind | What it would do | Likely first user | +|---|---|---| +| `add` | Adds an item to a list setting instead of replacing a field (each **Use** adds one stop). | Dutch transport plugin | +| `next` | Choosing starts a second search instead of filling in fields: pick a stop, then a direction or line; pick a music server, then a player on it. | Dutch transport plugin, `lms_client` | +| `disabled` | The choice is shown but can't be used, with the reason ("No trains to Utrecht from this station"). | NS trains | +| `picture` | A small image next to the choice (a station logo, an album cover), sent inside the page like preview images. | music plugins | +| `wait` | For logging in to an outside account: show a message, a link and a code ("Open example.com/device on your phone and enter ABCD-1234"), and ask the helper again every few seconds, each call in its own process with its own time limit, until it returns a choice (e.g. a token in a secret setting) or stops with a message. | a calendar or Spotify plugin | + +- **A kind PaperPi doesn't know is ignored and logged**, so a plugin written for a newer PaperPi still works on an older one. +- **All texts are shown as plain text**, so a stop name with odd characters can't change the page. +- What happens after **Use** (such as drawing a preview at once) is the same for every plugin, so the page decides it, not each helper. + ### Where helpers come from -- **PaperPi's own helpers** are for needs many plugins share. The first is `location`. -- **A plugin's own helpers** are part of the plugin, next to its settings and layouts (e.g. a stop search in the Dutch transport plugin). -- A setting names its helper as now: `setting(..., helper="location")`. The plugin's own helpers are looked up first, then PaperPi's. A name that is found nowhere fails the plugin check, so a typing mistake is caught by the tests and not on the user's page. -- The search box is shown under the setting that names the helper. A choice may fill in several settings; PaperPi fills in only settings the plugin has, and checks the values with the plugin's settings description before they go into the form. +- Helpers are part of the plugin, next to its settings and layouts. A setting names one as now: `setting(..., helper="location")`, and the search box is shown under that setting. A name the plugin doesn't have fails the plugin check, so a typing mistake is caught by the tests and not on the user's page. +- **PaperPi offers search functions** for needs many plugins share. The first is a place search: "find places for this text" returns places with name, region, country, latitude, longitude and altitude. A plugin's helper calls it and turns each place into a choice for its own settings. +- For the common case PaperPi also has a **ready-made location helper**: the plugin only names its settings for latitude, longitude and, if it has them, altitude and the name shown on the screen. This is a shortcut built on the place search, not a separate idea. ### How a helper runs @@ -61,27 +101,29 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and ### Later (described so they fit, not built until a plugin needs them) -- **On the command line** (txoof, 2026-10-10: later, as its own issue): `paperpi helper . ""`, e.g. `paperpi helper met_no.location "Morrison, Colorado"`. It prints the choices with the lines to copy into the plugin's block in the config file. It names the plugin type, not a plugin `id`, so it works before the plugin is in the config file. To decide in that issue: how a helper that needs other settings first (the NS API key) gets them, e.g. from a plugin block in the config file or typed after the text. - -- **Logging in to an outside account:** a helper may return a **step** instead of choices: a message with a link and a code ("Open example.com/device on your phone and enter ABCD-1234"). PaperPi then calls the helper again every few seconds, each call in its own process with its own time limit, until it returns a choice (the token, which fills a secret setting) or stops with a message. The rest of the structure stays the same. +- **On the command line** (txoof, 2026-10-10: later, as its own issue): `paperpi helper . ""`, e.g. `paperpi helper met_no.location "Morrison, Colorado"`. It prints each choice's `show` and `link` texts and its `fill` values as lines to copy into the plugin's block in the config file. It names the plugin type, not a plugin `id`, so it works before the plugin is in the config file. To decide in that issue: how a helper that needs other settings first (the NS API key) gets them, e.g. from a plugin block in the config file or typed after the text. +- **The planned kinds** in the table above. - **Sandboxed plugin pages** (option 2 in its safe form), if a plugin ever needs more than a search box and a list. -### The first helper: `location` +### The first helper: `location` in `met_no` and `moon_phase` -- Uses the place search of **Open-Meteo** (`geocoding-api.open-meteo.com`), chosen with txoof on 2026-10-10. No key and no sign-up. It finds towns and cities (not street addresses) and returns region, country, latitude, longitude **and altitude**; Nominatim, which v1 used, returns no altitude. It understands "Morrison, Colorado"; a plain "Morrison" returns 10 towns in several countries, so region and country are shown with each choice. +- The place search uses **Open-Meteo** (`geocoding-api.open-meteo.com`), chosen with txoof on 2026-10-10. No key and no sign-up. It finds towns and cities (not street addresses) and returns region, country, latitude, longitude **and altitude**; Nominatim, which v1 used, returns no altitude. It understands "Morrison, Colorado"; a plain "Morrison" returns 10 towns in several countries, so region and country are shown with each choice. - Terms: free for non-commercial use, fewer than 10,000 requests a day. The data is CC BY 4.0 and the place data is from GeoNames, so the page shows "Search by Open-Meteo, place data from GeoNames" with links next to the list. -- A choice fills in `lat` and `lon` (4 decimals, about 10 m), `altitude` (whole metres, as met.no asks) and, when it is still empty, the name shown on the screen (`place` in `met_no`). +- A choice: + - `fill`: `lat` and `lon` (4 decimals, about 10 m), `altitude` (whole metres, as met.no asks), and the name shown on the screen (`place` in `met_no`) only when it is still empty, so a name the user typed ("Home", "Cabin") is kept; + - `show`: place, region, country, altitude; + - `link`: "See on map" (OpenStreetMap at that point), so the user can check the place without a map in PaperPi. - `met_no` gets an optional `altitude` setting (whole metres, -500 to 9000), sent to met.no when it is filled in. This closes issue #176. The example in #176 had a sign mistake: `lon = 84.31` is on the Tibetan plateau, where met.no's 4949 m is right; Atlanta is at `-84.31`. - `moon_phase` uses it for `lat` and `lon` only; altitude doesn't matter for it. ### Planned PRs -1. This note, the helper structure (functions, choices, the separate process, plugin check) and the search box with its pick list, tested with a helper in the `debugging` plugin. About 500 lines. -2. The `location` helper with Open-Meteo, `altitude` for `met_no`, and both plugins using it. About 400 lines. +1. This note, the helper structure (helper functions, results with the four kinds, the separate process, plugin check) and the search box with its pick list, tested with a helper in the `debugging` plugin. About 500 lines. +2. The place search with Open-Meteo, the ready-made location helper, `altitude` for `met_no`, and both plugins using it. About 400 lines. ## Open questions -To be answered with txoof one at a time: -1. ~~Does the search start with the button (and Enter), or while the user types?~~ Button and Enter (txoof, 2026-10-10). -2. ~~Should helpers also work from the command line on the Pi, and if so in these PRs or later?~~ Later, as its own issue, in the form `paperpi helper met_no.location "Morrison, Colorado"` (txoof, 2026-10-10; see "Later"). -3. Should `location` fill in the name shown on the screen when it is empty, as proposed above? +None. Answered with txoof on 2026-10-10: +1. The search starts with the button or the Enter key, not while the user types. +2. The command line comes later, as its own issue, as `paperpi helper met_no.location "Morrison, Colorado"` (see "Later"). +3. The name shown on the screen is filled in only when it is empty. This is decided by the helper (it leaves `place` out of `fill`), not by a rule in PaperPi (see "What a choice is"). From 2f64c332ddfa159faa8caadfb0722db9b1e2a178 Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:48:54 +0200 Subject: [PATCH 5/9] Setting helpers: search box and pick list on the settings page (M5 part 3c) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 2 +- docs/writing-plugins.md | 43 ++++- src/paperpi/helper.py | 149 ++++++++++++++++ src/paperpi/limits.py | 10 ++ src/paperpi/plugin.py | 45 ++++- src/paperpi/plugins/debugging/__init__.py | 52 +++++- src/paperpi/runner.py | 96 +++++++++-- src/paperpi/web/app.py | 79 ++++++++- src/paperpi/web/forms.py | 9 +- src/paperpi/web/helpers.py | 110 +++++++++--- src/paperpi/web/plugins.py | 55 +++++- src/paperpi/web/server.py | 9 +- src/paperpi/web/static/style.css | 7 + src/paperpi/web/templates/_field.html | 62 +++++++ src/paperpi/web/templates/filled.html | 13 ++ src/paperpi/web/templates/found.html | 38 +++++ src/paperpi/web/templates/helper.html | 20 +++ src/paperpi/web/templates/settings.html | 75 ++------ tests/test_helper.py | 171 +++++++++++++++++++ tests/test_web_helpers.py | 197 ++++++++++++++++++++++ tests/test_web_settings.py | 15 +- 21 files changed, 1117 insertions(+), 140 deletions(-) create mode 100644 src/paperpi/helper.py create mode 100644 src/paperpi/web/templates/_field.html create mode 100644 src/paperpi/web/templates/filled.html create mode 100644 src/paperpi/web/templates/found.html create mode 100644 src/paperpi/web/templates/helper.html create mode 100644 tests/test_helper.py create mode 100644 tests/test_web_helpers.py diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index 62da91c..bea80fc 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -118,7 +118,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit ### Planned PRs -1. This note, the helper structure (helper functions, results with the four kinds, the separate process, plugin check) and the search box with its pick list, tested with a helper in the `debugging` plugin. About 500 lines. +1. This note, the helper structure (helper functions, results with the four kinds, the separate process, plugin check) and the search box with its pick list, tested with a helper in the `debugging` plugin. About 900 lines (txoof chose one PR over two on 2026-10-10). 2. The place search with Open-Meteo, the ready-made location helper, `altitude` for `met_no`, and both plugins using it. About 400 lines. ## Open questions diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 3125e76..2c30eb2 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -80,7 +80,7 @@ Each setting is a field of the plugin's `Settings` class, with a type, a default - Every setting needs a default. - Don't use the names of the shared settings, which every `[[plugin]]` block already has: `id`, `name`, `type`, `enabled`, `level`, `display_time`, `refresh`, `time_limit`, `layout`, `alert_reminder`, `alert_max_time`. - Use `pydantic.SecretStr` as the type for API keys and passwords. PaperPi then never shows their values in error messages or logs. -- A setting the user must fill in before the plugin can work (a place, an email address, an API key) is made with `paperpi.plugin.setting(..., required=True)` instead of `Field`. It takes the same arguments as `Field`. Its default must be "not set": `None`, empty text or an empty `SecretStr` (text of only spaces also counts as not set). Until every required setting is filled in, the plugin is not shown and not counted as broken; the config check, `paperpi list` and the web interface's Active Plugins page say which settings are missing. The example block marks them "(required)". `setting` is also where other options for a single setting go: `helper="location"` names a web interface helper shown under the setting's field (the first one, a latitude and longitude lookup, comes in M5 part 3c; see `paperpi.web.helpers`). +- A setting the user must fill in before the plugin can work (a place, an email address, an API key) is made with `paperpi.plugin.setting(..., required=True)` instead of `Field`. It takes the same arguments as `Field`. Its default must be "not set": `None`, empty text or an empty `SecretStr` (text of only spaces also counts as not set). Until every required setting is filled in, the plugin is not shown and not counted as broken; the config check, `paperpi list` and the web interface's Active Plugins page say which settings are missing. The example block marks them "(required)". `setting` is also where other options for a single setting go, such as `helper=` (see "Setting helpers" below). - The web interface shows the plugin's `description` in its Plugin Library, and the help text (`description`) of every setting on the plugin's Settings page, next to its field. Write both for someone at home who is not a programmer. The settings page is built from the settings description: a `Literal` becomes a choice list, `bool` a check box, `int`/`float` a number field (with its `ge`, `gt`, `le` and `lt` limits), `str` a text field (with `max_length`), `SecretStr` a password field, and a tuple or list of a `Literal` a list to tick. A setting of another kind is shown as it is in the file, to be changed there. ```python @@ -100,6 +100,47 @@ type = "basic_clock" hours = 12 ``` +## Setting helpers + +Some settings are hard to fill in by hand, such as a place's latitude and longitude or a stop's code. A plugin can offer a **helper**: a search box under the setting on its settings page. The user types something, PaperPi shows a list of choices, and **Use** fills in the form's fields (nothing is saved until Save). The reasons are in `docs/decisions/setting-helpers.md`. + +- A helper is a `paperpi.plugin.Helper` in the plugin's `helpers`, by name. A setting names it with `setting(..., helper="")`; the search box is shown under that setting. The plugin check fails for a name the plugin doesn't have. +- Its `find(query)` gets a `Query`: `query.text` (what the user typed, at most 100 characters) and `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret). +- It returns a list of choices, or `{"choices": [...], "note": "..."}` with a message for the whole result. Each choice is a dictionary; its keys say what PaperPi does with each value: + - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone (e.g. a name the user already typed). + - `show`: label and text, shown to help the user choose; the first one is the heading. + - `link`: label and web address (`http` or `https` only), e.g. "See on map". + A kind PaperPi doesn't know is left out and logged. At most 10 choices are shown. +- `raise HelperProblem("...")` stops with a message for the user (e.g. "The API key was not accepted"). Any other error shows "The search failed"; the details go to PaperPi's log. +- Each search runs in its own process with a time limit of 30 seconds, like an update. Use `paperpi.webrequest` for web requests. +- `Helper` also takes the `prompt` above the box, the `button` text, `typed=False` for a helper that needs no typed text (only a button), `needs` (settings that must be filled in first, e.g. an API key) and `source` and `source_link` (where the answers come from, shown under them; say it when the typed text goes to an outside service). + +```python +from paperpi.plugin import Helper, HelperProblem, PluginSettings, Query, setting + + +class Settings(PluginSettings): + stop: str = setting("", helper="stops", description="Code of the stop") + stop_name: str = setting("", description="Name shown on the screen") + + +def find_stops(query: Query): + found = search_the_stop_list(query.text) # the plugin's own code + return [ + { + "fill": {"stop": s.code} | ({} if query.settings.stop_name else {"stop_name": s.name}), + "show": {"Stop": s.name, "Town": s.town}, + } + for s in found + ] + + +# In PLUGIN = Plugin(...): +# helpers={"stops": Helper(find_stops, prompt="Name of the stop, e.g. Centraal Station")}, +``` + +The `debugging` plugin has a helper to try it out: `words` turns each typed word into a choice for its `text` setting. + ## Layouts `LAYOUTS` maps names to epdlib layouts (see [epdlib's layout guide](https://github.com/txoof/epdlib/blob/main/docs/layouts.md)). The first one is the default; the user can pick another with `layout = "..."`. A layout is a dictionary, or a function that makes one from the settings, for layouts that depend on a setting. diff --git a/src/paperpi/helper.py b/src/paperpi/helper.py new file mode 100644 index 0000000..9df50ef --- /dev/null +++ b/src/paperpi/helper.py @@ -0,0 +1,149 @@ +"""Setting helpers: a search a plugin offers on its settings page, such as finding a place's +latitude and longitude from its name (see ``docs/decisions/setting-helpers.md``). + +A helper is a :class:`Helper` in the plugin's ``helpers``. A setting names it with +``setting(..., helper="")`` and the settings page shows a search box under that +setting. The helper's ``find`` gets a :class:`Query` (the typed text and the form's settings) +and returns choices: a list of dictionaries, or ``{"choices": [...], "note": "..."}``. Each +choice is a dictionary whose keys are **kinds** that say what PaperPi does with the value: + +- ``fill``: settings put into the form when the user presses Use (nothing is saved yet); +- ``show``: label and text, shown to help the user choose; the first one is the heading; +- ``link``: label and web address (``http`` or ``https``) the user can open. + +``note`` is a message for the whole result. A kind PaperPi doesn't know is left out and +logged, so a plugin made for a newer PaperPi still works. :func:`clean` checks a result. +A helper runs in its own process with a time limit (:func:`paperpi.runner.run_helper`). +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +from . import limits + +if TYPE_CHECKING: + from .plugin import PluginSettings + +#: The kinds of a choice PaperPi knows; the others are left out (see the top). +KINDS = ("fill", "show", "link") +#: The kinds of web address a helper may link to. +WEB = ("http", "https") +#: The longest text PaperPi shows from a choice. +LONGEST_TEXT = 200 + + +@dataclass(frozen=True) +class Query: + """What a helper's ``find`` gets.""" + + text: str + """What the user typed, without spaces around it (``""`` for a helper with ``typed=False``).""" + settings: PluginSettings + """The settings in the form, checked as Save checks them. An empty secret field (an API + key) gives the saved secret.""" + + +@dataclass(frozen=True) +class Helper: + """A search a plugin offers for one of its settings.""" + + find: Callable[[Query], Sequence[Mapping[str, Any]] | Mapping[str, Any]] + """Returns the choices (see the top). Raises :class:`HelperProblem` with a message for + the user; any other error shows "The search failed" and goes to the log.""" + prompt: str + """The text above the search box, e.g. "Place name, e.g. Berlin or Morrison, Colorado".""" + button: str = "Search" + typed: bool = True + """False for a helper that needs no typed text (e.g. "Find servers"): only a button.""" + source: str = "" + """Where the answers come from, shown under them, e.g. "Search by Open-Meteo". Say it + when the typed text goes to an outside service.""" + source_link: str = "" + """A web address for ``source``.""" + needs: tuple[str, ...] = () + """Settings that must be filled in before a search, e.g. an API key.""" + + +class HelperProblem(Exception): + """Raised by a helper's ``find`` to stop with a message for the user, such as "The API + key was not accepted".""" + + +@dataclass(frozen=True) +class Choice: + """One checked choice.""" + + show: tuple[tuple[str, str], ...] + """Label and text; the first is the heading.""" + fill: Mapping[str, Any] + """Setting and value.""" + links: tuple[tuple[str, str], ...] = () + """Label and web address.""" + + @property + def heading(self) -> str: + return self.show[0][1] + + +@dataclass(frozen=True) +class Found: + """A checked result.""" + + choices: tuple[Choice, ...] + note: str = "" + more: bool = False + """The helper found more than :data:`~paperpi.limits.HELPER_CHOICES`; only those are kept.""" + left_out: tuple[str, ...] = () + """What was left out (kinds PaperPi doesn't know, links that are not ``http``), to log.""" + + +def clean(result: Any, settings: Sequence[str]) -> Found: + """Check what a helper's ``find`` returned. ``settings`` are the names a ``fill`` may + use. Raises ``TypeError`` for a result that breaks the rules (a bug in the plugin).""" + note: Any = "" + left_out: list[str] = [] + if isinstance(result, Mapping): + left_out += [f"result kind {k!r}" for k in result if k not in ("choices", "note")] + note, result = result.get("note", ""), result.get("choices", ()) + if isinstance(result, str | bytes) or not isinstance(result, Sequence): + raise TypeError("a helper returns a list of choices or {'choices': [...], 'note': ...}") + if not isinstance(note, str): + raise TypeError("a result's note must be text") + choices = [] + for number, raw in enumerate(result[: limits.HELPER_CHOICES], 1): + if not isinstance(raw, Mapping): + raise TypeError(f"choice {number} is not a dictionary") + left_out += [f"choice kind {k!r}" for k in raw if k not in KINDS] + show = _pairs(raw.get("show", {}), f"choice {number}: show") + if not show: + raise TypeError(f"choice {number} has nothing to show") + fill = raw.get("fill", {}) + if not isinstance(fill, Mapping): + raise TypeError(f"choice {number}: fill must be a dictionary") + unknown = [str(k) for k in fill if k not in settings] + if unknown: + raise TypeError(f"choice {number} fills settings it doesn't have: {', '.join(unknown)}") + links = [] + for label, address in _pairs(raw.get("link", {}), f"choice {number}: link"): + if urlsplit(address).scheme in WEB: + links.append((label, address)) + else: + left_out.append(f"link {address!r} (only http and https)") + choices.append(Choice(show, dict(fill), tuple(links))) + more = len(result) > limits.HELPER_CHOICES + return Found(tuple(choices), _short(note), more, tuple(left_out)) + + +def _pairs(raw: Any, what: str) -> tuple[tuple[str, str], ...]: + if not isinstance(raw, Mapping): + raise TypeError(f"{what} must be a dictionary") + return tuple((_short(str(k)), _short(str(v))) for k, v in raw.items()) + + +def _short(text: str) -> str: + text = " ".join(text.split()) + return text if len(text) <= LONGEST_TEXT else text[: LONGEST_TEXT - 1] + "…" diff --git a/src/paperpi/limits.py b/src/paperpi/limits.py index 16c62be..d33ddd0 100644 --- a/src/paperpi/limits.py +++ b/src/paperpi/limits.py @@ -23,6 +23,16 @@ #: drawn, so the first screen update goes first. Opening the Library page starts them at once. LIBRARY_PICTURES_WAIT = 60.0 +#: Longest one search of a setting helper may take: one web request with its retry +#: (``docs/decisions/setting-helpers.md``). +HELPER_SEARCH = 30.0 + +#: Most characters the user may type into a helper's search box. +HELPER_TEXT = 100 + +#: Most choices a helper's search shows. +HELPER_CHOICES = 10 + #: How long a plugin process may take to exit after it has handed over its image. PLUGIN_EXIT = 5.0 diff --git a/src/paperpi/plugin.py b/src/paperpi/plugin.py index 364fc80..a8b1f16 100644 --- a/src/paperpi/plugin.py +++ b/src/paperpi/plugin.py @@ -19,12 +19,14 @@ from __future__ import annotations +import dataclasses import re from collections.abc import Callable, Mapping from dataclasses import dataclass from enum import StrEnum from pathlib import Path from typing import Any, Literal +from urllib.parse import urlsplit from epdlib import Layout, ScreenMode from PIL import Image @@ -32,6 +34,12 @@ from pydantic.fields import FieldInfo from . import limits +from .helper import WEB + +# Part of the plugin interface too, so plugins import them from here. +from .helper import Helper as Helper +from .helper import HelperProblem as HelperProblem +from .helper import Query as Query class State(StrEnum): @@ -116,10 +124,9 @@ def setting( with a required setting that is not set is not shown; the config check and ``paperpi list`` and the web interface's Active Plugins page say which settings it needs. - ``helper``: the name of a web interface helper for this setting, shown under its field - on the settings page, for example ``"location"`` (look up a place's latitude and - longitude, M5 part 3c). A name the web interface doesn't know is left out, so the - plain field still works. See ``paperpi.web.helpers``. + ``helper``: the name of one of the plugin's ``helpers`` (a search, such as finding a + place's latitude and longitude), shown under this setting's field on the settings page. + See :mod:`paperpi.helper`. """ extra = field.pop("json_schema_extra", None) or {} if not isinstance(extra, dict): @@ -145,7 +152,7 @@ def is_required(info: FieldInfo) -> bool: def helper_of(info: FieldInfo) -> str | None: - """The web interface helper a setting names (``setting(helper=...)``), or ``None``.""" + """The helper a setting names (``setting(helper=...)``), or ``None``.""" helper = _options(info).get("helper") return helper if isinstance(helper, str) else None @@ -351,6 +358,8 @@ class Plugin: """Suggested most megabytes in its storage folder (e.g. more for a photo album).""" storage_days: int = limits.STORAGE_DAYS """Suggested days after which its saved files are removed (0 = keep them).""" + helpers: Mapping[str, Helper] = dataclasses.field(default_factory=dict) + """Searches for the settings page, by name (see :mod:`paperpi.helper`).""" def __post_init__(self) -> None: problems = [] @@ -374,6 +383,7 @@ def __post_init__(self) -> None: ) if not self.layouts: problems.append("needs at least one layout") + problems += self._helper_problems() if not limits.SHORTEST_REFRESH <= self.refresh <= limits.LONGEST_SETTING: problems.append( f"refresh must be between {limits.SHORTEST_REFRESH:g} " @@ -386,6 +396,31 @@ def __post_init__(self) -> None: if problems: raise PluginDefinitionError(f"plugin {self.type!r}: " + "; ".join(problems)) + def _helper_problems(self) -> list[str]: + problems = [] + if not isinstance(self.helpers, Mapping): + return ["helpers must be a dictionary"] + good = isinstance(self.settings, type) and issubclass(self.settings, PluginSettings) + own = self.settings.model_fields if good else {} + for name, helper in self.helpers.items(): + if not isinstance(helper, Helper): + problems.append(f"helper {name!r} must be a Helper") + elif unknown := [k for k in helper.needs if k not in own]: + problems.append(f"helper {name!r} needs settings it doesn't have: {unknown}") + elif helper.source_link and urlsplit(helper.source_link).scheme not in WEB: + problems.append(f"helper {name!r}: source_link must start with http or https") + for key, info in own.items(): + name = helper_of(info) + if name is not None and name not in self.helpers: + problems.append(f"setting {key!r} names helper {name!r}, which it doesn't have") + return problems + + def helper(self, key: str) -> Helper | None: + """The helper the setting ``key`` names, or ``None``.""" + info = self.settings.model_fields.get(key) + name = helper_of(info) if info is not None else None + return self.helpers.get(name) if name is not None else None + @property def default_layout(self) -> str: return next(iter(self.layouts)) diff --git a/src/paperpi/plugins/debugging/__init__.py b/src/paperpi/plugins/debugging/__init__.py index 750eb9a..6ae5330 100644 --- a/src/paperpi/plugins/debugging/__init__.py +++ b/src/paperpi/plugins/debugging/__init__.py @@ -10,6 +10,11 @@ The image shows only ``text`` and the state, so updates with the same state give the same image (the screen is then not written again). + +``text`` has a setting helper, ``words`` (see :mod:`paperpi.helper`), to test the settings +page's search box: each typed word becomes a choice that fills in ``text``. Some words +misbehave instead: ``crash`` raises an error, ``hang`` hangs until the time limit, +``problem`` stops with a message for the user and ``many`` gives 12 choices. """ import time @@ -19,13 +24,26 @@ from ... import limits from ...files import write_atomic -from ...plugin import NOTHING, Context, Plugin, PluginSettings, alert, ready +from ...plugin import ( + NOTHING, + Context, + Helper, + HelperProblem, + Plugin, + PluginSettings, + Query, + alert, + ready, + setting, +) COUNT_FILE = "count" class Settings(PluginSettings): - text: str = Field("debugging", max_length=100, description="Text shown above the state") + text: str = setting( + "debugging", max_length=100, helper="words", description="Text shown above the state" + ) states: tuple[Literal["nothing", "ready", "alert"], ...] = Field( ("ready",), min_length=1, @@ -73,6 +91,29 @@ def draw(state: str, context: Context) -> dict: return {"text": context.settings.text, "state": state} +def find_words(query: Query): + """The ``words`` helper (see the top).""" + words = query.text.split() + if "crash" in words: + raise RuntimeError("crash, as asked") + if "hang" in words: + while True: # until the time limit stops this process + time.sleep(3600) + if "problem" in words: + raise HelperProblem("A problem, as asked.") + if "many" in words: + words = [f"word {n}" for n in range(1, 13)] + choices = [ + { + "fill": {"text": word}, + "show": {"Word": word, "Now": query.settings.text}, + "link": {"Example": "https://example.org/"}, + } + for word in words + ] + return {"choices": choices, "note": f"{len(words)} words."} + + BLOCK = {"type": "text", "align": "center", "padding": 0.05} LAYOUTS = { @@ -93,4 +134,11 @@ def draw(state: str, context: Context) -> dict: draw=draw, sample="ready", refresh=30, + helpers={ + "words": Helper( + find_words, + prompt="Some words; each one becomes a choice", + source="The debugging plugin", + ) + }, ) diff --git a/src/paperpi/runner.py b/src/paperpi/runner.py index b3929af..cccd6c5 100644 --- a/src/paperpi/runner.py +++ b/src/paperpi/runner.py @@ -1,4 +1,5 @@ -"""Run one plugin update in a new, short-lived process, with a time limit. +"""Run one plugin update (or one search of a setting helper) in a new, short-lived process, +with a time limit. See ``docs/decisions/plugin-interface.md``. Each update gets its own process: when it is done, the process exits and its memory is given back. A plugin that hangs or crashes is @@ -11,6 +12,7 @@ from __future__ import annotations +import contextlib import dataclasses import logging import multiprocessing @@ -19,6 +21,7 @@ import threading import time import traceback +from collections.abc import Callable, Iterator from dataclasses import dataclass from multiprocessing.connection import Connection from urllib.parse import quote, quote_plus @@ -27,7 +30,8 @@ from pydantic import SecretBytes, SecretStr from . import limits -from .plugin import Context, State +from .helper import Found, HelperProblem, Query +from .plugin import Context, PluginSettings, State log = logging.getLogger(__name__) @@ -86,14 +90,47 @@ def run_update( """ time_limit = limits.PLUGIN_UPDATE if time_limit is None else time_limit start = time.monotonic() - receiver, sender = _context.Pipe(duplex=False) # The settings travel as plain data: unpacking a settings object would import the plugin # before the child is in its own process group, where a hang can't be stopped cleanly. settings = context.settings.model_dump() bare = dataclasses.replace(context, settings=None) + args = (plugin_type, package, bare, settings, sample) + message = _run(plugin_type, _child, args, time_limit) + seconds = time.monotonic() - start + _, state, image = message + return UpdateResult(State(state), _unpack(image), seconds) + + +def run_helper( + plugin_type: str, + name: str, + text: str, + settings: PluginSettings, + *, + time_limit: float | None = None, + package: str = "paperpi.plugins", +) -> Found: + """Run one search of the plugin's helper ``name`` (see :mod:`paperpi.helper`) in its own + process, for the typed ``text`` and the form's ``settings``, and return its checked + result. + + ``time_limit`` defaults to :data:`paperpi.limits.HELPER_SEARCH`. Raises + :class:`HelperProblem` with the helper's message for the user, and otherwise the same + errors as :func:`run_update`. + """ + time_limit = limits.HELPER_SEARCH if time_limit is None else time_limit + args = (plugin_type, package, name, text, settings.model_dump()) + return _run(plugin_type, _helper_child, args, time_limit)[1] + + +def _run(plugin_type: str, target: Callable, args: tuple, time_limit: float) -> tuple: + """Run ``target(sender, *args)`` in a new plugin process and return the message it sends: + ``("ok", ...)``. Raises for the others: ``("error", reason, details)`` and + ``("problem", message)`` (a helper's message for the user).""" + receiver, sender = _context.Pipe(duplex=False) process = _context.Process( - target=_child, - args=(sender, plugin_type, package, bare, settings, sample), + target=target, + args=(sender, *args), name=f"paperpi-plugin-{plugin_type}", daemon=True, ) @@ -119,12 +156,12 @@ def run_update( sender.close() receiver.close() _stop(process, wait=finished) - seconds = time.monotonic() - start if message[0] == "error": _, reason, details = message raise PluginFailed(plugin_type, reason, details) - _, state, image = message - return UpdateResult(State(state), _unpack(image), seconds) + if message[0] == "problem": + raise HelperProblem(message[1]) + return message def stop_all() -> None: @@ -186,18 +223,37 @@ def _child( sample: bool, ) -> None: """Runs in the plugin process: one update, then send the result and exit.""" - os.setpgrp() # own process group, so the parent can stop everything this plugin starts - secrets: list[str] = [] - try: - from . import plugins + with _reporting(sender) as secrets: from .plugin import draw_update - plugin = plugins.load(plugin_type, package) - checked = plugin.settings.model_validate(settings) - secrets = [v.get_secret_value() for v in dict(checked).values() if _is_secret(v)] + plugin, checked = _load(plugin_type, package, settings, secrets) context = dataclasses.replace(context, settings=checked) state, image = draw_update(plugin, context, sample=sample) sender.send(("ok", state.value, _pack(image))) + + +def _helper_child( + sender: Connection, plugin_type: str, package: str, name: str, text: str, settings: dict +) -> None: + """Runs in the plugin process: one search of a helper, then send the result and exit.""" + with _reporting(sender) as secrets: + from .helper import clean + + plugin, checked = _load(plugin_type, package, settings, secrets) + result = plugin.helpers[name].find(Query(text, checked)) + sender.send(("ok", clean(result, list(plugin.settings.model_fields)))) + + +@contextlib.contextmanager +def _reporting(sender: Connection) -> Iterator[list[str]]: + """In the plugin process: send any error as a message, with secret settings (filled into + the list it gives) replaced with ``****``. Also closes ``sender``.""" + os.setpgrp() # own process group, so the parent can stop everything this plugin starts + secrets: list[str] = [] + try: + yield secrets + except HelperProblem as problem: + sender.send(("problem", _hide(str(problem), secrets))) except BaseException as error: # noqa: BLE001 - report everything, then exit reason = _hide(f"{type(error).__name__}: {error}", secrets) sender.send(("error", reason, _hide(traceback.format_exc(), secrets))) @@ -205,6 +261,16 @@ def _child( sender.close() +def _load(plugin_type: str, package: str, settings: dict, secrets: list[str]): + """The plugin and its checked settings; their secrets are added to ``secrets``.""" + from . import plugins + + plugin = plugins.load(plugin_type, package) + checked = plugin.settings.model_validate(settings) + secrets += [v.get_secret_value() for v in dict(checked).values() if _is_secret(v)] + return plugin, checked + + def _is_secret(value) -> bool: return isinstance(value, SecretStr | SecretBytes) diff --git a/src/paperpi/web/app.py b/src/paperpi/web/app.py index 3eacebe..392fabe 100644 --- a/src/paperpi/web/app.py +++ b/src/paperpi/web/app.py @@ -11,7 +11,9 @@ up or down, remove, settings (``/plugins//...?id=``, n = place in the file, counted from 0, and the plugin's id to check it is still there); ``/plugins//preview`` draws the settings form's - values without saving them (part of the settings page) + values without saving them; ``/plugins//search/`` runs + that setting's helper and ``/plugins//use/`` puts a + choice into the form (all three are parts of the settings page) ``/library`` Plugin Library: every plugin type with an example picture (``/library/pictures/``; ``/library//picture`` is the picture's part of the page, asked for again until it is drawn); @@ -30,6 +32,7 @@ from __future__ import annotations import ipaddress +import json import logging from collections.abc import Awaitable, Callable from pathlib import Path @@ -42,12 +45,12 @@ from fastapi.templating import Jinja2Templates from starlette.concurrency import run_in_threadpool -from .. import __version__ +from .. import __version__, limits from ..config import folder_name from .auth import COOKIE_SECONDS, Auth, new_cookie from .config_file import EditError -from .forms import Read, fields -from .helpers import helper_html +from .forms import Read, fields, form_texts +from .helpers import Searcher, SearchFailed from .password import PasswordError, password_problem from .pictures import Pictures from .plugins import FormErrors, PluginEditor, library, library_item @@ -68,17 +71,25 @@ def create_app( editor: PluginEditor | None = None, previewer: Previewer | None = None, pictures: Pictures | None = None, + searcher: Searcher | None = None, ) -> FastAPI: """The web interface, using ``auth`` for the password and log-in, ``editor`` to - change the plugins in the config file, ``previewer`` to draw previews and ``pictures`` - for the Plugin Library's example pictures (default: none).""" + change the plugins in the config file, ``previewer`` to draw previews, ``pictures`` + for the Plugin Library's example pictures (default: none) and ``searcher`` for the + settings helpers' searches.""" editor = editor or PluginEditor(auth.config_file) previewer = previewer or Previewer() pictures = pictures or Pictures(None) + searcher = searcher or Searcher() app = FastAPI(title="PaperPi", docs_url=None, redoc_url=None, openapi_url=None) app.mount("/static", StaticFiles(directory=_HERE / "static"), name="static") templates = Jinja2Templates(directory=_HERE / "templates") - templates.env.globals.update(version=__version__, auth=auth, helper_html=helper_html) + templates.env.globals.update( + version=__version__, + auth=auth, + text_limit=limits.HELPER_TEXT, + time_limit=limits.HELPER_SEARCH, + ) templates.env.filters["number"] = _number templates.env.filters["sentence"] = lambda text: text[:1].upper() + text[1:] @@ -308,6 +319,48 @@ async def preview(request: Request, index: int): return page(request, "preview.html", problem=problem) return page(request, "preview.html", drawn=drawn, job=job) + @app.post("/plugins/{index}/search/{key}", response_class=HTMLResponse) + async def search(request: Request, index: int, key: str): + # Only a part of the settings page (htmx), so every answer is a 200. + plugin_id, sent = _settings_form(await request.form()) + text = (sent.pop("_search", None) or [""])[-1] + job = None + try: + job = await run_in_threadpool(editor.search_job, index, plugin_id, sent, key, text) + found = await run_in_threadpool(searcher.find, job) + except FormErrors as error: + return page(request, "found.html", errors=error.found.errors) + except (EditError, Busy, SearchFailed) as error: + helper = job.helper if job else None + return page(request, "found.html", problem=str(error), helper=helper) + except Exception: # noqa: BLE001 - e.g. no plugin process could be started + log.exception("search of %s, %s", plugin_id, key) + problem = "The search failed. PaperPi's log has the details." + return page(request, "found.html", problem=problem) + # What Use sends back: each setting with its texts, as the form sends them. + uses = [ + json.dumps({k: form_texts(v) for k, v in choice.fill.items()}) + for choice in found.choices + ] + values = {"found": found, "helper": job.helper, "uses": uses} + return page(request, "found.html", index=index, key=key, **values) + + @app.post("/plugins/{index}/use/{key}", response_class=HTMLResponse) + async def use(request: Request, index: int, key: str): + # Only a part of the settings page (htmx), so every answer is a 200. + plugin_id, sent = _settings_form(await request.form()) + fill = _fill((sent.pop("_use", None) or [""])[-1]) + if not fill: + return page(request, "filled.html", problem="This choice can't be used.", shown=[]) + try: + shown = await run_in_threadpool(editor.filled, index, plugin_id, sent, fill) + except EditError as error: + return page(request, "filled.html", problem=str(error), shown=[]) + if not shown: + problem = "This choice has nothing the form can fill in." + return page(request, "filled.html", problem=problem, shown=[]) + return page(request, "filled.html", shown=shown) + def settings_saved(index: int, plugin_id: str, done: str, hand_edits: bool): values = {"id": plugin_id, "done": done} | ({"hand": "1"} if hand_edits else {}) return RedirectResponse(f"/plugins/{index}/settings?{urlencode(values)}", 303) @@ -391,6 +444,18 @@ def _settings_form(form) -> tuple[str, dict[str, list[str]]]: return plugin_id, {k: [str(v) for v in form.getlist(k)] for k in form if k != "id"} +def _fill(text: str) -> dict[str, list[str]]: + """The settings and texts a Use button sends (``{}`` when they are not in that form).""" + try: + fill = json.loads(text) + except ValueError: + return {} + texts = fill.values() if isinstance(fill, dict) else [None] + if not all(isinstance(v, list) and all(isinstance(t, str) for t in v) for v in texts): + return {} + return fill + + def _number(value: float) -> str: """A number as the pages show it: ``604800``, not ``604800.0``.""" return str(int(value)) if float(value).is_integer() else str(value) diff --git a/src/paperpi/web/forms.py b/src/paperpi/web/forms.py index b9d6732..a04b3fb 100644 --- a/src/paperpi/web/forms.py +++ b/src/paperpi/web/forms.py @@ -74,7 +74,7 @@ class FormField: max_length: int | None = None error: str = "" helper: str | None = None - """The web interface helper shown under the field (see :mod:`paperpi.web.helpers`).""" + """The name of the plugin's helper shown under the field (see :mod:`paperpi.helper`).""" changed: bool = False """The config file sets a value that is not the default.""" @@ -120,7 +120,7 @@ def fields( if key == "layout": kind, allowed = "choice", tuple(plugin.layouts) value = block.get(key, shown[key]) - texts = [_text(v) for v in value] if isinstance(value, list | tuple) else [_text(value)] + texts = form_texts(value) if kind == "file": texts = [example.toml_value(value) if value is not None else ""] if _holds_secret(info.annotation) and texts[0]: @@ -204,6 +204,11 @@ def read(plugin: Plugin, block: Mapping[str, Any], form: Mapping[str, Sequence[s return Read(changes) +def form_texts(value: Any) -> list[str]: + """``value`` as the texts a form sends for it (several for a list).""" + return [_text(v) for v in value] if isinstance(value, list | tuple) else [_text(value)] + + def _infos(plugin: Plugin) -> dict[str, FieldInfo]: """Every setting the form shows, with its description.""" entry = {k: v for k, v in PluginEntry.model_fields.items() if k not in NOT_IN_FORM} diff --git a/src/paperpi/web/helpers.py b/src/paperpi/web/helpers.py index 0c2c7c6..e16708b 100644 --- a/src/paperpi/web/helpers.py +++ b/src/paperpi/web/helpers.py @@ -1,31 +1,99 @@ -"""Setting helpers: an extra part of the settings page, under one setting's field, for a -setting that needs more than a plain field (``docs/decisions/web-interface.md``). - -A plugin names a helper with ``paperpi.plugin.setting(..., helper="location")``. A helper -here is a function that gets the field (:class:`paperpi.web.forms.FormField`) and returns -the HTML to show under it. A name that is not in :data:`HELPERS` (yet) shows nothing, so -the plain field still works. The first helper, ``location`` (look up a place's latitude -and longitude), comes in M5 part 3c; new ones are added when a plugin needs one. - -The page shows a helper's HTML as it is, and the field's texts come from the config file -or the form. So build the HTML with ``Markup("

{}

").format(...)`` or a template, -never with an f-string: those escape the texts (turn ``<`` into ``<`` and so on), so -typed text can't become part of the page. +"""Setting helpers on the settings page: the search box under a setting, its choices and their +Use buttons (see ``docs/decisions/setting-helpers.md`` and :mod:`paperpi.helper`). + +A search runs the plugin's helper in its own process with a time limit +(:func:`paperpi.runner.run_helper`), with the settings in the form as Save would write them. +Only one search runs at a time, as with previews: each one starts a Python process and may +wait for a web server. Use puts a choice's ``fill`` into the form's fields; nothing is saved. """ from __future__ import annotations +import logging +import threading from collections.abc import Callable +from dataclasses import dataclass + +from .. import limits +from ..config import PluginConfig +from ..helper import Found, Helper, HelperProblem +from ..plugin import is_set +from ..runner import PluginFailed, PluginTimeout, run_helper +from .preview import Busy + +log = logging.getLogger(__name__) + +#: Runs one search: ``search(plugin_type, name, text, settings, time_limit=...)``, as +#: :func:`paperpi.runner.run_helper`. Tests pass a quick fake. +Search = Callable[..., Found] + + +class SearchFailed(RuntimeError): + """The search could not be done or gave no answer; the message says why.""" + + +@dataclass(frozen=True) +class Job: + """One search: the plugin block as Save would write it, the setting and the typed text.""" + + found: PluginConfig + key: str + """The setting whose helper this is.""" + name: str + """The helper's name in the plugin's ``helpers``.""" + helper: Helper + text: str -from markupsafe import Markup -from .forms import FormField +class Searcher: + """Runs helpers' searches, one at a time.""" -#: The helpers the settings page knows, by name. -HELPERS: dict[str, Callable[[FormField], Markup]] = {} + def __init__(self, search: Search = run_helper): + self._search = search + self._lock = threading.Lock() + self._stopped = threading.Event() + def stop(self) -> None: + """PaperPi stops: start no more plugin processes for searches. (A running one is + stopped by :func:`paperpi.runner.stop_all`.)""" + self._stopped.set() -def helper_html(field: FormField) -> Markup: - """The HTML of the helper ``field`` names, or nothing.""" - helper = HELPERS.get(field.helper) if field.helper else None - return helper(field) if helper is not None else Markup("") + def find(self, job: Job) -> Found: + """Search. Raises :class:`~paperpi.web.preview.Busy` while another search runs and + :class:`SearchFailed` with a message for the user. Can take up to + :data:`~paperpi.limits.HELPER_SEARCH` seconds, so call it from a worker thread.""" + if job.helper.typed and not job.text: + raise SearchFailed("Type what to search for first.") + if len(job.text) > limits.HELPER_TEXT: + raise SearchFailed(f"Type at most {limits.HELPER_TEXT} characters.") + settings = job.found.settings + missing = [key for key in job.helper.needs if not is_set(getattr(settings, key))] + if missing: + raise SearchFailed(f"Fill in these settings first: {', '.join(missing)}.") + if not self._lock.acquire(blocking=False): + raise Busy("Another search is running. Wait a moment, then try again.") + try: + if self._stopped.is_set(): + raise SearchFailed("PaperPi is stopping, so no search can be done now.") + plugin_type = job.found.plugin.type + try: + found = self._search( + plugin_type, job.name, job.text, settings, time_limit=limits.HELPER_SEARCH + ) + except PluginTimeout: + raise SearchFailed( + f"The search took longer than {limits.HELPER_SEARCH:g} seconds. " + "Try again later, or fill in the settings by hand." + ) from None + except HelperProblem as problem: + raise SearchFailed(str(problem)) from None + except PluginFailed as error: + log.warning("search of %s, %s: %s", job.found.entry.id, job.key, error) + if error.details: + log.debug("%s", error.details) + raise SearchFailed("The search failed. PaperPi's log has the details.") from None + finally: + self._lock.release() + if found.left_out: + log.info("search of %s left out: %s", job.found.entry.id, "; ".join(found.left_out)) + return found diff --git a/src/paperpi/web/plugins.py b/src/paperpi/web/plugins.py index 42f2eeb..8174550 100644 --- a/src/paperpi/web/plugins.py +++ b/src/paperpi/web/plugins.py @@ -24,8 +24,8 @@ from pydantic import ValidationError from .. import config, example, plugins -from ..plugin import ID_LENGTH, Plugin, PluginEntry, label -from . import config_file, forms, preview +from ..plugin import ID_LENGTH, Plugin, PluginEntry, helper_of, label +from . import config_file, forms, helpers, preview #: Plugin types the Plugin Library doesn't offer: ``default`` is PaperPi's own message when #: nothing else can be shown, and ``debugging`` is for testing PaperPi. @@ -177,6 +177,50 @@ def preview_job( """What a preview of block ``index`` draws with the settings of a sent form: the block as Save would write it, and the screen. Nothing is saved. Raises :class:`FormErrors` when a value can't be used.""" + found, checked = self._as_saved(index, plugin_id, form, "drawn") + # The screen as it runs now: a new size, mode or type waits for the next start. + display = self.display or checked.display + width, height = display.layout_size + return preview.Job(found, width, height, display.screen_mode) + + def search_job( + self, index: int, plugin_id: str, form: Mapping[str, Sequence[str]], key: str, text: str + ) -> helpers.Job: + """A search with the helper of setting ``key`` of block ``index``, for the typed + ``text`` and the settings of a sent form as Save would write them. Nothing is + saved. Raises :class:`FormErrors` when a value can't be used.""" + found, _ = self._as_saved(index, plugin_id, form, "searched for") + info = found.plugin.settings.model_fields.get(key) + name = helper_of(info) if info is not None else None + if name is None: + raise config_file.EditError(f"The setting {key!r} has no search.") + return helpers.Job(found, key, name, found.plugin.helpers[name], text.strip()) + + def filled( + self, + index: int, + plugin_id: str, + form: Mapping[str, Sequence[str]], + fill: Mapping[str, Sequence[str]], + ) -> list[forms.FormField]: + """The fields of the settings form of block ``index`` after Use put a choice's + ``fill`` (setting and texts) into the sent form, each with its error if the value + can't be used. Only the plugin's own settings that the form can change are filled.""" + plugin, block = self.settings(index, plugin_id) + sent = {k: list(v) for k, v in form.items()} | {k: list(v) for k, v in fill.items()} + found = forms.read(plugin, block, sent) + shown = forms.fields(plugin, block, sent, found.errors) + return [ + f + for f in shown + if f.key in fill and f.group == "own" and f.kind not in ("secret", "file") + ] + + def _as_saved( + self, index: int, plugin_id: str, form: Mapping[str, Sequence[str]], what: str + ) -> tuple[config.PluginConfig, config.Config]: + """Block ``index`` checked with the settings of a sent form, as Save would write + them, and the whole checked file. Nothing is saved.""" path, text = config_file.read(self.path) new = _with_settings(text, index, plugin_id, form) checked = _parse(new, path.name) @@ -190,13 +234,10 @@ def preview_job( if p.line and place.start < p.line <= place.end ) raise config_file.EditError( - "This plugin can't be drawn. Its settings in the config file have a problem: " + f"This plugin can't be {what}. Its settings in the config file have a problem: " f"{problems or 'unknown'}" ) - # The screen as it runs now: a new size, mode or type waits for the next start. - display = self.display or checked.display - width, height = display.layout_size - return preview.Job(found, width, height, display.screen_mode) + return found, checked def row(self, index: int, plugin_id: str) -> Row: """The block at ``index``, if it still has the ID ``plugin_id``.""" diff --git a/src/paperpi/web/server.py b/src/paperpi/web/server.py index dbb7f91..4457a5d 100644 --- a/src/paperpi/web/server.py +++ b/src/paperpi/web/server.py @@ -18,6 +18,7 @@ from .. import config from .app import create_app from .auth import Auth +from .helpers import Searcher from .pictures import FOLDER, Pictures, Screen from .plugins import PluginEditor from .preview import Previewer @@ -42,6 +43,7 @@ def __init__( self.auth = auth self.editor = editor self.previewer = Previewer() + self.searcher = Searcher() self.pictures = pictures or Pictures(None) self.display: config.DisplaySettings | None = None """The ``[display]`` settings the screen uses now (see :meth:`use_display`).""" @@ -52,7 +54,7 @@ def __init__( """The port it listens on (useful with ``port = 0`` in tests).""" self._server = uvicorn.Server( uvicorn.Config( - create_app(auth, self.editor, self.previewer, self.pictures), + create_app(auth, self.editor, self.previewer, self.pictures, self.searcher), log_config=None, # PaperPi's own logging stays as it is log_level="warning", access_log=False, @@ -97,9 +99,10 @@ def running(self) -> bool: return self._thread.is_alive() def stop_previews(self) -> None: - """Start no more previews or library pictures: PaperPi is stopping (see - :meth:`Previewer.stop`).""" + """Start no more previews, library pictures or helper searches: PaperPi is stopping + (see :meth:`Previewer.stop`).""" self.previewer.stop() + self.searcher.stop() self.pictures.stop() def stop(self) -> None: diff --git a/src/paperpi/web/static/style.css b/src/paperpi/web/static/style.css index 67c4b1c..709bdad 100644 --- a/src/paperpi/web/static/style.css +++ b/src/paperpi/web/static/style.css @@ -40,3 +40,10 @@ code { background: #eee; padding: 0 0.2rem; } .preview { margin: 1rem 0; } .preview img { display: block; max-width: 100%; height: auto; border: 1px solid #999; } .preview figcaption { color: #555; font-size: 0.9rem; margin-top: 0.4rem; } +.helper { margin: -0.4rem 0 1rem; padding: 0.5rem 0.75rem; border-left: 4px solid #ccc; } +.helper > label { display: block; margin: 0 0 0.25rem; } +.helper .search { display: flex; gap: 0.5rem; } +.helper .search input { flex: 1; } +.found { padding-left: 1.25rem; } +.found li { margin: 0.5rem 0; } +.found button { margin-top: 0.25rem; } diff --git a/src/paperpi/web/templates/_field.html b/src/paperpi/web/templates/_field.html new file mode 100644 index 0000000..9b75e00 --- /dev/null +++ b/src/paperpi/web/templates/_field.html @@ -0,0 +1,62 @@ +{#- One field of the settings form (settings.html). With oob, htmx puts it in place of the + field with the same id (after Use in a helper's choices). -#} +{% macro field(f, oob=false) %} +
+ {% if f.kind == "check" %} + {# A browser sends nothing for a box that is off, so "false" goes first. #} + + + {% else %} + + {% if f.kind == "choice" %} + + {% elif f.kind == "choices" %} + {# A browser sends nothing for a list with nothing picked, so an empty value goes first. #} + +
+ {% for choice in f.choices %} + + {% endfor %} +
+ {% elif f.kind == "number" %} + {# A phone's number keys often have no minus, so only for numbers that can't be below 0. #} + = 0 %} + inputmode="{{ 'numeric' if f.whole else 'decimal' }}"{% endif %}> + {% elif f.kind == "text" %} + + {% elif f.kind == "secret" %} + + {% else %} +

{{ f.value or "(not set)" }} + Change this one in the config file.

+ {% endif %} + {% if f.kind not in ("secret", "file") %} +
+ {%- if f.kind == "number" and (f.minimum is not none or f.maximum is not none) %} + {%- if f.minimum is not none %}From {{ f.minimum|number }}{% endif %} + {%- if f.maximum is not none %} up to {{ f.maximum|number }}{% endif %}. {% endif %} + {%- if f.default %}Default: {{ f.default }}.{% elif f.kind != "choices" %}Empty: not set.{% endif -%} +
+ {% endif %} + {% endif %} + {% if f.error %}

{{ f.error|sentence }}

{% endif %} +
+{% endmacro %} diff --git a/src/paperpi/web/templates/filled.html b/src/paperpi/web/templates/filled.html new file mode 100644 index 0000000..689253f --- /dev/null +++ b/src/paperpi/web/templates/filled.html @@ -0,0 +1,13 @@ +{#- After Use: the message goes into

, and each filled field takes the place + of the field with its id (hx-swap-oob). -#} +{% from "_field.html" import field %} +{% if problem %} +{{ problem|sentence }} +{% elif shown|selectattr("error")|list %} +Filled in, but some values can't be used; see the messages next to + them. +{% else %} +Filled in: {{ shown|map(attribute="key")|join(", ") }}. Not saved yet: press + Save to keep it. +{% endif %} +{% for f in shown %}{{ field(f, oob=true) }}{% endfor %} diff --git a/src/paperpi/web/templates/found.html b/src/paperpi/web/templates/found.html new file mode 100644 index 0000000..fd69ef2 --- /dev/null +++ b/src/paperpi/web/templates/found.html @@ -0,0 +1,38 @@ +{#- Only a part of the settings page: htmx puts it into

. -#} +{% if errors %} +

Fix these settings first, then search again:

+
    + {% for name, message in errors.items() %} +
  • {% if name %}{{ name }}: {% endif %}{{ message }}
    • + {% endfor %} +
+{% elif problem %} +

{{ problem|sentence }}

+{% else %} +{% if found.note %}

{{ found.note }}

{% endif %} +{% if not found.choices %} +

Nothing found. Check the spelling, or try something more general.

+{% else %} +
    + {% for choice in found.choices %} +
  • + {{ choice.heading }} + {%- for label, text in choice.show[1:] %}
    {{ label }}: {{ text }}{% endfor %} + {%- for label, address in choice.links %}
    {{ label }}{% endfor %} + {%- if choice.fill %} +
    + {%- endif %} +
    • + {% endfor %} +
+{% if found.more %}

Only the first {{ found.choices|length }} are shown. Type more + to narrow the search.

{% endif %} +

+{% endif %} +{% endif %} +{% if helper and helper.source %} +

{% if helper.source_link %}{{ helper.source }}{% else %}{{ helper.source }}{% endif %}

+{% endif %} diff --git a/src/paperpi/web/templates/helper.html b/src/paperpi/web/templates/helper.html new file mode 100644 index 0000000..9ae526a --- /dev/null +++ b/src/paperpi/web/templates/helper.html @@ -0,0 +1,20 @@ +{#- A setting helper's search box, under its field (settings.html). Its input and button belong + to the form search-form- after the settings form. -#} +
+ {% if helper.typed %} + +
+ + +
+ {% else %} +

{{ helper.prompt }}

+ + {% endif %} +

Searching. This can take up to + {{ time_limit|number }} seconds.

+
+
diff --git a/src/paperpi/web/templates/settings.html b/src/paperpi/web/templates/settings.html index 7b1228a..b493b73 100644 --- a/src/paperpi/web/templates/settings.html +++ b/src/paperpi/web/templates/settings.html @@ -2,66 +2,7 @@ {% set logged_in = true %} {% block title %}Settings of {{ row.name }} · PaperPi{% endblock %} -{% macro field(f) %} -
- {% if f.kind == "check" %} - {# A browser sends nothing for a box that is off, so "false" goes first. #} - - - {% else %} - - {% if f.kind == "choice" %} - - {% elif f.kind == "choices" %} - {# A browser sends nothing for a list with nothing picked, so an empty value goes first. #} - -
- {% for choice in f.choices %} - - {% endfor %} -
- {% elif f.kind == "number" %} - {# A phone's number keys often have no minus, so only for numbers that can't be below 0. #} - = 0 %} - inputmode="{{ 'numeric' if f.whole else 'decimal' }}"{% endif %}> - {% elif f.kind == "text" %} - - {% elif f.kind == "secret" %} - - {% else %} -

{{ f.value or "(not set)" }} - Change this one in the config file.

- {% endif %} - {% if f.kind not in ("secret", "file") %} -
- {%- if f.kind == "number" and (f.minimum is not none or f.maximum is not none) %} - {%- if f.minimum is not none %}From {{ f.minimum|number }}{% endif %} - {%- if f.maximum is not none %} up to {{ f.maximum|number }}{% endif %}. {% endif %} - {%- if f.default %}Default: {{ f.default }}.{% elif f.kind != "choices" %}Empty: not set.{% endif -%} -
- {% endif %} - {% endif %} - {% if f.error %}

{{ f.error|sentence }}

{% endif %} - {{ helper_html(f) }} -
-{% endmacro %} +{% from "_field.html" import field %} {% block main %}

{{ row.name }}

@@ -79,12 +20,15 @@

{{ row.name }}

{% endif %} {% for found in row.errors %}

{{ found }}

{% endfor %} -
+ {% for f in groups.name %}{{ field(f) }}{% endfor %} {% if groups.own %}

Settings of this plugin

- {% for f in groups.own %}{{ field(f) }}{% endfor %} + {% for f in groups.own %}{{ field(f) }} + {%- set helper = plugin.helper(f.key) %}{% if helper %}{% include "helper.html" %}{% endif %} + {%- endfor %} {% endif %}

When and how it is shown

{% for f in groups.common %}{{ field(f) }}{% endfor %} @@ -103,6 +47,13 @@

When and how it is shown

minute.

+{# A form of its own for each search box, so Enter searches instead of saving. It sends the + settings form's values too. #} +{% for f in groups.own %}{% if plugin.helper(f.key) %} +
+{% endif %}{% endfor %}
Technical information diff --git a/tests/test_helper.py b/tests/test_helper.py new file mode 100644 index 0000000..6ec0c53 --- /dev/null +++ b/tests/test_helper.py @@ -0,0 +1,171 @@ +"""Setting helpers: the plugin's side (paperpi.helper, the plugin check, run_helper).""" + +import pytest +from pydantic import SecretStr + +from paperpi import limits, plugins, runner +from paperpi.helper import Found, Helper, HelperProblem, clean +from paperpi.plugin import PluginDefinitionError, PluginSettings, setting +from paperpi.runner import PluginFailed, PluginTimeout, run_helper + +from .test_plugin import make as make_plugin + +SETTINGS = ["lat", "lon", "place"] + + +def choice(**kinds): + return {"show": {"Place": "Morrison"}} | kinds + + +def test_a_list_of_choices_is_checked(): + found = clean( + [ + choice( + fill={"lat": 39.6536, "lon": -105.1911}, link={"See on map": "https://osm.org/"} + ), + {"show": {"Place": "Morrison", "Region": "Illinois"}}, + ], + SETTINGS, + ) + first, second = found.choices + assert first.heading == "Morrison" and first.fill == {"lat": 39.6536, "lon": -105.1911} + assert first.links == (("See on map", "https://osm.org/"),) + assert second.show == (("Place", "Morrison"), ("Region", "Illinois")) and not second.fill + assert found == Found(found.choices) # no note, no more, nothing left out + + +def test_a_result_can_have_a_note_and_only_the_first_choices_are_kept(): + many = [choice(fill={"place": str(n)}) for n in range(limits.HELPER_CHOICES + 2)] + found = clean({"choices": many, "note": "Many places."}, SETTINGS) + assert len(found.choices) == limits.HELPER_CHOICES and found.more + assert found.note == "Many places." + assert clean({"note": "Nothing here."}, SETTINGS) == Found((), "Nothing here.") + + +def test_unknown_kinds_and_other_links_are_left_out_and_noted(): + found = clean( + { + "choices": [choice(add={"stops": "3104"}, link={"Bad": "javascript:alert(1)"})], + "wait": 5, + }, + SETTINGS, + ) + assert found.choices[0].links == () + assert found.left_out == ( + "result kind 'wait'", + "choice kind 'add'", + "link 'javascript:alert(1)' (only http and https)", + ) + + +def test_texts_are_made_short_and_on_one_line(): + found = clean([{"show": {"Place": "Morrison\n Colorado", "Long": "x" * 500}}], SETTINGS) + (_, place), (_, long) = found.choices[0].show + assert place == "Morrison Colorado" and len(long) == 200 and long.endswith("…") + + +@pytest.mark.parametrize( + "result, message", + [ + ("Morrison", "list of choices"), + ([["Morrison"]], "choice 1 is not a dictionary"), + ([{"fill": {"lat": 1}}], "choice 1 has nothing to show"), + ([choice(fill={"altitude": 1757})], "fills settings it doesn't have: altitude"), + ([choice(fill=[1])], "fill must be a dictionary"), + ([choice(link="https://osm.org/")], "link must be a dictionary"), + ({"choices": [], "note": 5}, "note must be text"), + ], +) +def test_a_result_that_breaks_the_rules_is_a_bug_in_the_plugin(result, message): + with pytest.raises(TypeError, match=message): + clean(result, SETTINGS) + + +class PlaceSettings(PluginSettings): + lat: float | None = setting(None, helper="location") + key: SecretStr = setting(SecretStr("")) + + +def find(query): + return [] + + +@pytest.mark.parametrize( + "helpers, message", + [ + ({}, "setting 'lat' names helper 'location', which it doesn't have"), + ({"location": find}, "helper 'location' must be a Helper"), + ( + {"location": Helper(find, "Place", needs=("api_key",))}, + "needs settings it doesn't have: \\['api_key'\\]", + ), + ( + {"location": Helper(find, "Place", source_link="ftp://x")}, + "source_link must start with http or https", + ), + ([], "helpers must be a dictionary"), + ], +) +def test_the_plugin_check_catches_wrong_helpers(helpers, message): + with pytest.raises(PluginDefinitionError, match=message): + make_plugin(settings=PlaceSettings, helpers=helpers) + + +def test_a_plugin_finds_the_helper_of_a_setting(): + helper = Helper(find, "Place", needs=("key",), source_link="https://open-meteo.com/") + plugin = make_plugin(settings=PlaceSettings, helpers={"location": helper}) + assert plugin.helper("lat") is helper + assert plugin.helper("key") is None and plugin.helper("nope") is None + + +# --- In a real plugin process, with the debugging plugin's "words" helper ------------------- + + +@pytest.fixture +def words(): + return plugins.load("debugging").settings(text="now") + + +def test_run_helper_returns_the_checked_choices(words): + found = run_helper("debugging", "words", "hello world", words) + assert [c.heading for c in found.choices] == ["hello", "world"] + assert found.choices[0].fill == {"text": "hello"} + assert dict(found.choices[0].show)["Now"] == "now" # the form's settings reached it + assert found.note == "2 words." + + +def test_run_helper_reports_a_problem_an_error_and_a_hang(words): + with pytest.raises(HelperProblem, match="A problem, as asked"): + run_helper("debugging", "words", "problem", words) + with pytest.raises(PluginFailed, match="RuntimeError: crash, as asked"): + run_helper("debugging", "words", "crash", words) + with pytest.raises(PluginTimeout): + run_helper("debugging", "words", "hang", words, time_limit=1) + + +def test_run_helper_hides_secret_settings_in_its_messages(monkeypatch): + class Secret(PluginSettings): + key: SecretStr = setting(SecretStr("")) + + def leak(query): + raise HelperProblem(f"key {query.settings.key.get_secret_value()} refused") + + plugin = make_plugin(settings=Secret, helpers={"leak": Helper(leak, "Key")}) + settings = Secret(key=SecretStr("TOPSECRET")) + # In this process (the fake plugin can't be imported in another one), as the child would. + sender = Sender() + monkeypatch.setattr(plugins, "load", lambda plugin_type, package: plugin) + monkeypatch.setattr(runner.os, "setpgrp", lambda: None) + runner._helper_child(sender, "fake", "tests", "leak", "", settings.model_dump()) + assert sender.sent == [("problem", "key **** refused")] + + +class Sender: + def __init__(self): + self.sent = [] + + def send(self, message): + self.sent.append(message) + + def close(self): + pass diff --git a/tests/test_web_helpers.py b/tests/test_web_helpers.py new file mode 100644 index 0000000..db00a42 --- /dev/null +++ b/tests/test_web_helpers.py @@ -0,0 +1,197 @@ +"""Setting helpers on the settings page: the search box, the choices and Use (M5 part 3c).""" + +import html +import json +import re +import threading + +import pytest +from fastapi.testclient import TestClient + +from paperpi import config +from paperpi.helper import Found, HelperProblem, clean +from paperpi.runner import PluginFailed, PluginTimeout +from paperpi.web import auth +from paperpi.web.app import create_app +from paperpi.web.helpers import Searcher +from paperpi.web.plugins import PluginEditor + +from .test_web_settings import PI + +ID = "debugging-1a2b3c4d" +CONFIG = f"""\ +config_version = 1 +[display] +type = "virtual" + +[[plugin]] +id = "{ID}" +type = "debugging" +""" +FORM = {"id": ID, "text": "typed"} + + +class FakeSearch: + """Stands in for the plugin process: gives ``answer`` (a result, or an error to raise) + and notes what it was asked.""" + + def __init__(self, answer): + self.answer = answer + self.calls = [] + + def __call__(self, plugin_type, name, text, settings, *, time_limit): + self.calls.append((plugin_type, name, text, settings, time_limit)) + if isinstance(self.answer, Exception): + raise self.answer + return clean(self.answer, ["text", "delay"]) + + +@pytest.fixture +def cfg(tmp_path): + path = tmp_path / "paperpi.toml" + path.write_text(CONFIG) + return path + + +def make_client(cfg, search=None): + editor = PluginEditor(cfg) + editor.loaded(CONFIG) + paperpi_auth = auth.Auth(cfg, config.WebSettings(login=False)) + searcher = Searcher(search) if search else Searcher() + app = create_app(paperpi_auth, editor, searcher=searcher) + return TestClient(app, follow_redirects=False, base_url=PI) + + +def search(client, text, **form): + """The page part a search gives.""" + sent = FORM | form | {"_search": text} + return client.post("/plugins/0/search/text", data=sent).text + + +def uses(page): + """What each Use button on ``page`` sends.""" + return [json.loads(v)["_use"] for v in re.findall(r"hx-vals='([^']*)'", page)] + + +def test_the_page_has_a_search_box_with_a_form_of_its_own(cfg): + page = make_client(cfg).get("/plugins/0/settings", params={"id": ID}).text + box = page[page.index('id="helper-text"') :] + assert "Some words; each one becomes a choice" in box + # Enter in the box searches (its own form) instead of saving the settings. + assert 'name="_search" type="search" form="search-form-text"' in box + assert '
"}, + "link": {"See on map": "https://www.openstreetmap.org/?mlat=39.65"}, + }, + {"show": {"Place": "Nothing to fill"}}, + ], + "note": "2 places.", + } + ) + page = search(make_client(cfg, fake), " Morrison, Colorado ") + assert "Morrison" in page and "Region: Colorado <b>" in page + assert '") == 10 and "Only the first 10 are shown" in page + + +@pytest.mark.parametrize( + "answer, message", + [ + (HelperProblem("The API key was not accepted."), "The API key was not accepted."), + (PluginTimeout("debugging", "slow"), "The search took longer than 30 seconds"), + (PluginFailed("debugging", "RuntimeError: boom"), "The search failed. PaperPi's log"), + (RuntimeError("no process"), "The search failed. PaperPi's log"), + ], +) +def test_a_failed_search_says_why(cfg, answer, message): + page = search(make_client(cfg, FakeSearch(answer)), "x") + page = html.unescape(page) + assert message in page and "boom" not in page + + +def test_what_is_checked_before_searching(cfg): + fake = FakeSearch([]) + client = make_client(cfg, fake) + assert "Type what to search for first" in search(client, " ") + assert "at most 100 characters" in search(client, "x" * 101) + page = search(client, "x", delay="soon") + assert "Fix these settings first" in page and "delay" in page + assert "has no search" in client.post("/plugins/0/search/delay", data=FORM).text + assert "changed" in search(client, "x", id="other") + assert not fake.calls + + +def test_one_search_at_a_time(cfg): + started, go = threading.Event(), threading.Event() + + def slow(*args, **kwargs): + started.set() + go.wait(5) + return Found(()) + + client = make_client(cfg, slow) + first = threading.Thread(target=search, args=(client, "x")) + first.start() + try: + assert started.wait(5) + assert "Another search is running" in search(client, "y") + finally: + go.set() + first.join() + + +def test_use_fills_in_the_fields_without_saving(cfg): + client = make_client(cfg) + page = client.post("/plugins/0/use/text", data=FORM | {"_use": '{"text": ["Morrison"]}'}).text + assert "Filled in: text. Not saved yet" in page + field = page[page.index('
hello" in page and "Now: typed" in page + assert uses(page) == [json.dumps({"text": ["hello"]}), json.dumps({"text": ["world"]})] diff --git a/tests/test_web_settings.py b/tests/test_web_settings.py index cade852..666c77f 100644 --- a/tests/test_web_settings.py +++ b/tests/test_web_settings.py @@ -3,12 +3,11 @@ import pytest from fastapi.testclient import TestClient -from markupsafe import Markup from pydantic import Field, SecretStr from paperpi import config from paperpi.plugin import Plugin, PluginSettings, helper_of, is_required, ready, setting -from paperpi.web import auth, forms, helpers +from paperpi.web import auth from paperpi.web import plugins as web_plugins from paperpi.web.app import create_app from paperpi.web.plugins import PluginEditor @@ -149,18 +148,6 @@ def test_a_plugin_moved_meanwhile_shows_the_list(client, cfg): assert cfg.read_text() == CONFIG -def test_a_helper_is_shown_under_its_field(client, monkeypatch): - show = {"test": lambda f: Markup("

help {}

").format(f.key)} - monkeypatch.setattr(helpers, "HELPERS", show) - monkeypatch.setattr( - forms, - "helper_of", - lambda info: "test" if (info.description or "").startswith("Latitude") else "nope", - ) - page = client.get("/plugins/1/settings", params={"id": "Weather"}).text - assert page.count("

help ") == 1 and "

help lat

" in page # "nope" shows nothing - - def test_setting_takes_a_helper_name(): class Settings(PluginSettings): lat: float | None = setting(None, required=True, helper="location") From 2da68a58f2fbe2fdbe15d9303ecba0bb0720720f Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:11:53 +0200 Subject: [PATCH 6/9] Setting helpers: review fixes (secrets hidden in results, plain fill values, size limits, HelperProblem only for searches, more tests) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 12 ++-- docs/decisions/web-interface.md | 2 +- docs/writing-plugins.md | 7 +- src/paperpi/helper.py | 55 ++++++++++---- src/paperpi/limits.py | 3 + src/paperpi/plugin.py | 18 +++-- src/paperpi/runner.py | 15 ++-- src/paperpi/web/app.py | 6 +- src/paperpi/web/forms.py | 17 ++--- src/paperpi/web/plugins.py | 15 +++- tests/fake_plugins/fake/__init__.py | 18 ++++- tests/test_helper.py | 58 +++++++++------ tests/test_web_helpers.py | 108 ++++++++++++++++++++++++---- 13 files changed, 246 insertions(+), 88 deletions(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index bea80fc..e53bc04 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -1,6 +1,6 @@ # Setting helpers -Status: proposed (M5 part 3c, issue #238). Decided with txoof on 2026-10-10. +Status: decided with txoof on 2026-10-10 (M5 part 3c, issue #238). ## Problem @@ -25,7 +25,7 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and 3. **A map to search on** (a small map library such as Leaflet, with map pictures from OpenStreetMap). Rejected by txoof on 2026-10-10: a place or stop is searched once, when the plugin is set up, so a map is too much work and upkeep for it. 4. **A helper is a plain function of the plugin that returns data (a dictionary per choice); PaperPi shows one standard search box and pick list for all of them.** **Chosen.** -## Decision (proposed) +## Decision ### Who does what @@ -57,11 +57,11 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit } ``` -**Kinds built now** (all four are used by the first helper, `location`): +**Kinds built now** (three kinds for a choice, plus `note` for the whole result; the first helper, `location`, uses all of them): | Kind | Where | What PaperPi does with it | |---|---|---| -| `fill` | choice | Settings put into the form when the user presses **Use**. The keys are the plugin's own setting names. PaperPi checks them with the plugin's settings description, as Save does, so a wrong name or value is an error and never a broken setting. Only what is in `fill` changes, so a helper leaves a setting alone by leaving it out (e.g. the name shown on the screen, when the user has already typed one). | +| `fill` | choice | Settings put into the form when the user presses **Use**. The keys are the plugin's own setting names. PaperPi checks them with the plugin's settings description, as Save does, so a wrong name or value is an error and never a broken setting. Only what is in `fill` changes, so a helper leaves a setting alone by leaving it out (e.g. the name shown on the screen, when the user has already typed one). Only settings the form shows as a field: not secrets, and not settings that are changed in the file. | | `show` | choice | Information to help the user pick: label and text, shown in the order given. The first entry is the heading of the choice. | | `link` | choice | Web addresses the user can open in a new tab, e.g. "See on map". Only `http` and `https` addresses; others are left out and logged. | | `note` | result | A message for the whole result, e.g. "Only the first 10 matches are shown; type more to narrow it down". | @@ -77,7 +77,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit | `wait` | For logging in to an outside account: show a message, a link and a code ("Open example.com/device on your phone and enter ABCD-1234"), and ask the helper again every few seconds, each call in its own process with its own time limit, until it returns a choice (e.g. a token in a secret setting) or stops with a message. | a calendar or Spotify plugin | - **A kind PaperPi doesn't know is ignored and logged**, so a plugin written for a newer PaperPi still works on an older one. -- **All texts are shown as plain text**, so a stop name with odd characters can't change the page. +- **All texts are shown as plain text**, so a stop name with odd characters can't change the page. Each text is put on one line and cut to 200 characters. - What happens after **Use** (such as drawing a preview at once) is the same for every plugin, so the page decides it, not each helper. ### Where helpers come from @@ -96,7 +96,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit - Under the field: the text, a search box and the button. The search starts with the button or the Enter key, not while the user types (txoof, 2026-10-10): one request per search, which matters for services with a daily limit such as NS. After a search, the list of choices appears, each with a **Use** button. "Nothing found" comes with a hint (check the spelling, or try a larger place). - **Use** fills in the form's fields. **Nothing is saved yet**: the user sees the values, can change them, can press Preview, and then Save. The list stays until the next search, so a wrong pick is easy to change. -- No new JavaScript library: htmx (already shipped) sends the search and puts the values in the fields. +- No new JavaScript library: htmx (the small script the pages already use) sends the search and puts the values in the fields. - Typing the values by hand keeps working. When the service can't be reached, the page says so and the fields are left as they are. ### Later (described so they fit, not built until a plugin needs them) diff --git a/docs/decisions/web-interface.md b/docs/decisions/web-interface.md index f9fed94..7f36fd9 100644 --- a/docs/decisions/web-interface.md +++ b/docs/decisions/web-interface.md @@ -34,7 +34,7 @@ In v2 the web interface is the main way to set up and change PaperPi: plugins, s - One page handles every plugin. It reads the plugin's settings description (see `plugin-interface.md`) and builds the form from it: number field, dropdown, password field for API keys, and so on. - On Save, the values are checked with the same description. Errors are shown next to the field. A saved change applies at once (see `live-config-reload.md`). - A setting that needs more than a plain field can name an input type from a small list in the web interface, e.g. `location` (lat/lon lookup), `server_search` (find music servers), `layout_picker`. New input types are added when a plugin needs one. *Built in M5 part 3a-2:* a plugin names one with `paperpi.plugin.setting(..., helper="location")`; the web interface's list (`paperpi.web.helpers`) holds a function per name that adds its part under the field. A name it doesn't know adds nothing, so the plain field still works. The first helper is `location` (part 3c). - - *Update (M5 part 3c, 2026-10-10):* replaced by setting helpers: a helper is a function in PaperPi or in the plugin that returns choices as data, and the web interface shows one standard search box and pick list for all of them. See `setting-helpers.md`. + - *Update (M5 part 3c, 2026-10-10):* replaced by setting helpers: a helper is a function of the plugin that returns choices as data (PaperPi offers shared searches, such as a place search, for helpers to call), and the web interface shows one standard search box and pick list for all of them. See `setting-helpers.md`. - Plugins added later show up without changes to the web code. - *Agreed with txoof on 2026-10-08 (M5 part 3a):* the page shows the plugin's `name` first, then its own settings, then display time, refresh, layout and level; the other shared settings (time limit, alert and storage settings) are folded under "More settings", which opens by itself when one of them has an error or a value that is not the default. Each field shows its value, or the default when the file doesn't set it. - How a save works (M5 part 3a-1): only settings that changed are written. An empty field means "the default" (an empty check box: off; an empty list: nothing picked); a value changed to the default is taken out of the file and becomes the comment `# key = default` again (a value left as it is stays, also one written in the file that equals the default), so the help text above it still fits. A new setting takes the place of its `# key = ...` comment line when there is one. If any value has an error, nothing is saved; each error is shown next to its field, with what was typed (never a secret). A secret (such as an API key) is never shown on the page; leaving its field empty keeps the saved one. A setting the form can't show (a group of settings, a list of numbers) is shown as it is in the file, to be changed there. A setting written over several lines in the file is never changed by the page. diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 2c30eb2..5ce05b7 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -105,14 +105,15 @@ hours = 12 Some settings are hard to fill in by hand, such as a place's latitude and longitude or a stop's code. A plugin can offer a **helper**: a search box under the setting on its settings page. The user types something, PaperPi shows a list of choices, and **Use** fills in the form's fields (nothing is saved until Save). The reasons are in `docs/decisions/setting-helpers.md`. - A helper is a `paperpi.plugin.Helper` in the plugin's `helpers`, by name. A setting names it with `setting(..., helper="")`; the search box is shown under that setting. The plugin check fails for a name the plugin doesn't have. -- Its `find(query)` gets a `Query`: `query.text` (what the user typed, at most 100 characters) and `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret). +- Its `find(query)` gets a `Query`: `query.text` (what the user typed, without spaces around it, at most 100 characters; empty for `typed=False`) and `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret). - It returns a list of choices, or `{"choices": [...], "note": "..."}` with a message for the whole result. Each choice is a dictionary; its keys say what PaperPi does with each value: - - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone (e.g. a name the user already typed). + - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone (e.g. a name the user already typed). Only settings the form shows as a field: not secrets (`SecretStr`), and not settings that are changed in the file (such as a list of numbers). - `show`: label and text, shown to help the user choose; the first one is the heading. - `link`: label and web address (`http` or `https` only), e.g. "See on map". - A kind PaperPi doesn't know is left out and logged. At most 10 choices are shown. + A kind PaperPi doesn't know is left out and logged. At most 10 choices are shown. Each text is put on one line and cut to 200 characters. - `raise HelperProblem("...")` stops with a message for the user (e.g. "The API key was not accepted"). Any other error shows "The search failed"; the details go to PaperPi's log. - Each search runs in its own process with a time limit of 30 seconds, like an update. Use `paperpi.webrequest` for web requests. +- Secret settings (API keys) are replaced with `****` in everything a search returns, and a choice can't `fill` them. A search uses the form's values before they are saved, so never send a saved secret to a web address taken from the settings: someone could type their own address and get the key. Use only the plugin's own servers for that. - `Helper` also takes the `prompt` above the box, the `button` text, `typed=False` for a helper that needs no typed text (only a button), `needs` (settings that must be filled in first, e.g. an API key) and `source` and `source_link` (where the answers come from, shown under them; say it when the typed text goes to an outside service). ```python diff --git a/src/paperpi/helper.py b/src/paperpi/helper.py index 9df50ef..7a49e73 100644 --- a/src/paperpi/helper.py +++ b/src/paperpi/helper.py @@ -8,6 +8,7 @@ choice is a dictionary whose keys are **kinds** that say what PaperPi does with the value: - ``fill``: settings put into the form when the user presses Use (nothing is saved yet); + only settings the form shows as a field, not secrets or settings changed in the file; - ``show``: label and text, shown to help the user choose; the first one is the heading; - ``link``: label and web address (``http`` or ``https``) the user can open. @@ -34,6 +35,10 @@ WEB = ("http", "https") #: The longest text PaperPi shows from a choice. LONGEST_TEXT = 200 +#: Most ``show`` or ``link`` entries of one choice; the others are left out. +MOST_PAIRS = 10 +#: The longest text a ``fill`` may put into a field. +LONGEST_FILL = 1000 @dataclass(frozen=True) @@ -101,9 +106,10 @@ class Found: """What was left out (kinds PaperPi doesn't know, links that are not ``http``), to log.""" -def clean(result: Any, settings: Sequence[str]) -> Found: +def clean(result: Any, settings: Sequence[str], hide: Callable[[str], str] = str) -> Found: """Check what a helper's ``find`` returned. ``settings`` are the names a ``fill`` may - use. Raises ``TypeError`` for a result that breaks the rules (a bug in the plugin).""" + use. ``hide`` is applied to every text, to replace secrets (API keys) with ``****``. + Raises ``TypeError`` for a result that breaks the rules (a bug in the plugin).""" note: Any = "" left_out: list[str] = [] if isinstance(result, Mapping): @@ -118,30 +124,51 @@ def clean(result: Any, settings: Sequence[str]) -> Found: if not isinstance(raw, Mapping): raise TypeError(f"choice {number} is not a dictionary") left_out += [f"choice kind {k!r}" for k in raw if k not in KINDS] - show = _pairs(raw.get("show", {}), f"choice {number}: show") + show = _pairs(raw.get("show", {}), f"choice {number}: show", hide, left_out) if not show: raise TypeError(f"choice {number} has nothing to show") - fill = raw.get("fill", {}) - if not isinstance(fill, Mapping): - raise TypeError(f"choice {number}: fill must be a dictionary") - unknown = [str(k) for k in fill if k not in settings] - if unknown: - raise TypeError(f"choice {number} fills settings it doesn't have: {', '.join(unknown)}") + fill = _fill(raw.get("fill", {}), f"choice {number}", settings, hide) links = [] - for label, address in _pairs(raw.get("link", {}), f"choice {number}: link"): + for label, address in _pairs(raw.get("link", {}), f"choice {number}: link", hide, left_out): if urlsplit(address).scheme in WEB: links.append((label, address)) else: left_out.append(f"link {address!r} (only http and https)") - choices.append(Choice(show, dict(fill), tuple(links))) + choices.append(Choice(show, fill, tuple(links))) more = len(result) > limits.HELPER_CHOICES - return Found(tuple(choices), _short(note), more, tuple(left_out)) + return Found(tuple(choices), _short(hide(note)), more, tuple(left_out)) -def _pairs(raw: Any, what: str) -> tuple[tuple[str, str], ...]: +def _pairs( + raw: Any, what: str, hide: Callable[[str], str], left_out: list[str] +) -> tuple[tuple[str, str], ...]: if not isinstance(raw, Mapping): raise TypeError(f"{what} must be a dictionary") - return tuple((_short(str(k)), _short(str(v))) for k, v in raw.items()) + if len(raw) > MOST_PAIRS: + left_out.append(f"{what}: {len(raw) - MOST_PAIRS} entries over {MOST_PAIRS}") + pairs = list(raw.items())[:MOST_PAIRS] + return tuple((_short(hide(str(k))), _short(hide(str(v)))) for k, v in pairs) + + +def _fill( + raw: Any, what: str, settings: Sequence[str], hide: Callable[[str], str] +) -> dict[str, Any]: + """A choice's ``fill``: plain values (text, numbers, true or false, or lists of them).""" + if not isinstance(raw, Mapping): + raise TypeError(f"{what}: fill must be a dictionary") + unknown = [str(k) for k in raw if k not in settings] + if unknown: + raise TypeError(f"{what} fills settings it can't fill: {', '.join(unknown)}") + fill = {} + for key, value in raw.items(): + items = value if isinstance(value, list | tuple) else [value] + if not all(isinstance(v, str | int | float | bool) for v in items): + raise TypeError(f"{what}: fill {key} must be text, a number, true or false, or a list") + if any(isinstance(v, str) and len(v) > LONGEST_FILL for v in items): + raise TypeError(f"{what}: fill {key} is longer than {LONGEST_FILL} characters") + texts = [hide(v) if isinstance(v, str) else v for v in items] + fill[key] = texts if isinstance(value, list | tuple) else texts[0] + return fill def _short(text: str) -> str: diff --git a/src/paperpi/limits.py b/src/paperpi/limits.py index d33ddd0..5f13b3c 100644 --- a/src/paperpi/limits.py +++ b/src/paperpi/limits.py @@ -33,6 +33,9 @@ #: Most choices a helper's search shows. HELPER_CHOICES = 10 +#: Most characters a Use button may send (one choice's ``fill``, as JSON). +HELPER_USE = 20_000 + #: How long a plugin process may take to exit after it has handed over its image. PLUGIN_EXIT = 5.0 diff --git a/src/paperpi/plugin.py b/src/paperpi/plugin.py index a8b1f16..713359e 100644 --- a/src/paperpi/plugin.py +++ b/src/paperpi/plugin.py @@ -25,12 +25,12 @@ from dataclasses import dataclass from enum import StrEnum from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, get_args from urllib.parse import urlsplit from epdlib import Layout, ScreenMode from PIL import Image -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, SecretBytes, SecretStr, field_validator from pydantic.fields import FieldInfo from . import limits @@ -157,6 +157,15 @@ def helper_of(info: FieldInfo) -> str | None: return helper if isinstance(helper, str) else None +def holds_secret(annotation: Any) -> bool: + """True when a setting of this type is, or holds, a secret (``SecretStr``).""" + if annotation in (SecretStr, SecretBytes): + return True + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return any(holds_secret(f.annotation) for f in annotation.model_fields.values()) + return any(holds_secret(a) for a in get_args(annotation)) + + def is_set(value: Any) -> bool: """False for "not set": ``None``, empty text (also only spaces) or an empty secret.""" if hasattr(value, "get_secret_value"): @@ -400,8 +409,9 @@ def _helper_problems(self) -> list[str]: problems = [] if not isinstance(self.helpers, Mapping): return ["helpers must be a dictionary"] - good = isinstance(self.settings, type) and issubclass(self.settings, PluginSettings) - own = self.settings.model_fields if good else {} + if not (isinstance(self.settings, type) and issubclass(self.settings, PluginSettings)): + return [] # already a problem of its own + own = self.settings.model_fields for name, helper in self.helpers.items(): if not isinstance(helper, Helper): problems.append(f"helper {name!r} must be a Helper") diff --git a/src/paperpi/runner.py b/src/paperpi/runner.py index cccd6c5..dd637c7 100644 --- a/src/paperpi/runner.py +++ b/src/paperpi/runner.py @@ -31,7 +31,7 @@ from . import limits from .helper import Found, HelperProblem, Query -from .plugin import Context, PluginSettings, State +from .plugin import Context, PluginSettings, State, holds_secret log = logging.getLogger(__name__) @@ -240,8 +240,15 @@ def _helper_child( from .helper import clean plugin, checked = _load(plugin_type, package, settings, secrets) - result = plugin.helpers[name].find(Query(text, checked)) - sender.send(("ok", clean(result, list(plugin.settings.model_fields)))) + try: + result = plugin.helpers[name].find(Query(text, checked)) + except HelperProblem as problem: # only here: in an update it is an ordinary error + sender.send(("problem", _hide(str(problem), secrets))) + return + # A secret (an API key) can't be filled in, and is never shown or logged. + fields = plugin.settings.model_fields.items() + fillable = [key for key, info in fields if not holds_secret(info.annotation)] + sender.send(("ok", clean(result, fillable, lambda text: _hide(text, secrets)))) @contextlib.contextmanager @@ -252,8 +259,6 @@ def _reporting(sender: Connection) -> Iterator[list[str]]: secrets: list[str] = [] try: yield secrets - except HelperProblem as problem: - sender.send(("problem", _hide(str(problem), secrets))) except BaseException as error: # noqa: BLE001 - report everything, then exit reason = _hide(f"{type(error).__name__}: {error}", secrets) sender.send(("error", reason, _hide(traceback.format_exc(), secrets))) diff --git a/src/paperpi/web/app.py b/src/paperpi/web/app.py index 392fabe..3f6ad35 100644 --- a/src/paperpi/web/app.py +++ b/src/paperpi/web/app.py @@ -353,7 +353,7 @@ async def use(request: Request, index: int, key: str): if not fill: return page(request, "filled.html", problem="This choice can't be used.", shown=[]) try: - shown = await run_in_threadpool(editor.filled, index, plugin_id, sent, fill) + shown = await run_in_threadpool(editor.filled, index, plugin_id, sent, key, fill) except EditError as error: return page(request, "filled.html", problem=str(error), shown=[]) if not shown: @@ -446,9 +446,11 @@ def _settings_form(form) -> tuple[str, dict[str, list[str]]]: def _fill(text: str) -> dict[str, list[str]]: """The settings and texts a Use button sends (``{}`` when they are not in that form).""" + if len(text) > limits.HELPER_USE: + return {} try: fill = json.loads(text) - except ValueError: + except (ValueError, RecursionError): # RecursionError: nested very deeply return {} texts = fill.values() if isinstance(fill, dict) else [None] if not all(isinstance(v, list) and all(isinstance(t, str) for t in v) for v in texts): diff --git a/src/paperpi/web/forms.py b/src/paperpi/web/forms.py index a04b3fb..d5c1d77 100644 --- a/src/paperpi/web/forms.py +++ b/src/paperpi/web/forms.py @@ -27,11 +27,11 @@ from dataclasses import dataclass, field from typing import Any, Literal -from pydantic import BaseModel, SecretBytes, SecretStr, ValidationError +from pydantic import BaseModel, SecretStr, ValidationError from pydantic.fields import FieldInfo from .. import example -from ..plugin import Plugin, PluginEntry, helper_of, is_required +from ..plugin import Plugin, PluginEntry, helper_of, holds_secret, is_required #: The shared settings shown right after the plugin's own ones; the others are folded away #: under "More settings" (agreed with txoof, M5 part 3a). @@ -123,7 +123,7 @@ def fields( texts = form_texts(value) if kind == "file": texts = [example.toml_value(value) if value is not None else ""] - if _holds_secret(info.annotation) and texts[0]: + if holds_secret(info.annotation) and texts[0]: texts = ["(set; it holds a secret, so it is not shown)"] if sent is not None and key in sent and kind not in ("secret", "file"): # Without the empty value the page sends for a check box or list. @@ -271,19 +271,10 @@ def _default_text(kind: Kind, default: Any, info: FieldInfo) -> str: if kind == "choices": return ", ".join(_text(v) for v in default) or "none" if kind == "file": - return "" if _holds_secret(info.annotation) else example.toml_value(default) + return "" if holds_secret(info.annotation) else example.toml_value(default) return _text(default) -def _holds_secret(annotation: Any) -> bool: - """True when a setting of this type is, or holds, a secret (``SecretStr``).""" - if annotation in (SecretStr, SecretBytes): - return True - if isinstance(annotation, type) and issubclass(annotation, BaseModel): - return any(_holds_secret(f.annotation) for f in annotation.model_fields.values()) - return any(_holds_secret(a) for a in typing.get_args(annotation)) - - def _text(value: Any) -> str: """``value`` as the form shows it.""" if value is None: diff --git a/src/paperpi/web/plugins.py b/src/paperpi/web/plugins.py index 8174550..51cd275 100644 --- a/src/paperpi/web/plugins.py +++ b/src/paperpi/web/plugins.py @@ -194,19 +194,28 @@ def search_job( name = helper_of(info) if info is not None else None if name is None: raise config_file.EditError(f"The setting {key!r} has no search.") - return helpers.Job(found, key, name, found.plugin.helpers[name], text.strip()) + helper = found.plugin.helpers[name] + # A helper without a search box gets no text, also from a form sent by hand. + text = text.strip() if helper.typed else "" + return helpers.Job(found, key, name, helper, text) def filled( self, index: int, plugin_id: str, form: Mapping[str, Sequence[str]], + key: str, fill: Mapping[str, Sequence[str]], ) -> list[forms.FormField]: - """The fields of the settings form of block ``index`` after Use put a choice's + """The fields of the settings form of block ``index`` after Use, in the choices of + the helper of setting ``key``, put a choice's ``fill`` (setting and texts) into the sent form, each with its error if the value - can't be used. Only the plugin's own settings that the form can change are filled.""" + can't be used. Only the plugin's own settings that the form can change are filled; + ``fill`` was checked by :func:`paperpi.helper.clean` when the choice was made, and is + checked again here like typed values, as a browser can send anything.""" plugin, block = self.settings(index, plugin_id) + if plugin.helper(key) is None: + raise config_file.EditError(f"The setting {key!r} has no search.") sent = {k: list(v) for k, v in form.items()} | {k: list(v) for k, v in fill.items()} found = forms.read(plugin, block, sent) shown = forms.fields(plugin, block, sent, found.errors) diff --git a/tests/fake_plugins/fake/__init__.py b/tests/fake_plugins/fake/__init__.py index 963ecf5..d764ca1 100644 --- a/tests/fake_plugins/fake/__init__.py +++ b/tests/fake_plugins/fake/__init__.py @@ -12,11 +12,11 @@ from pydantic import SecretStr -from paperpi.plugin import NOTHING, Plugin, PluginSettings, alert, ready +from paperpi.plugin import NOTHING, Helper, HelperProblem, Plugin, PluginSettings, alert, ready Act = Literal[ "ok", "nothing", "alert", "raise", "hang", "exit", "crash", "kill", "child", "leave_child", - "secret", "wrong", + "secret", "wrong", "problem", ] # fmt: skip @@ -53,9 +53,22 @@ def fetch(context): raise ValueError(f"server refused key {context.settings.key.get_secret_value()}") if act == "wrong": return "not a Fetched" + if act == "problem": # meant for helpers; in an update it is an ordinary error + raise HelperProblem("only for searches") return ready("OK") +def leak(query): + """A helper that puts the API key everywhere it can.""" + key = query.settings.key.get_secret_value() + if query.text == "fill": + return [{"show": {"Key": "x"}, "fill": {"key": key}}] + return { + "choices": [{"show": {"Key": key}, "link": {key: f"https://example.org/?key={key}"}}], + "note": f"key {key}", + } + + def draw(text, context): return {"text": text} @@ -69,4 +82,5 @@ def draw(text, context): draw=draw, sample="SAMPLE", refresh=30, + helpers={"leak": Helper(leak, "Anything")}, ) diff --git a/tests/test_helper.py b/tests/test_helper.py index 6ec0c53..2b8163c 100644 --- a/tests/test_helper.py +++ b/tests/test_helper.py @@ -3,10 +3,10 @@ import pytest from pydantic import SecretStr -from paperpi import limits, plugins, runner +from paperpi import limits, plugins from paperpi.helper import Found, Helper, HelperProblem, clean from paperpi.plugin import PluginDefinitionError, PluginSettings, setting -from paperpi.runner import PluginFailed, PluginTimeout, run_helper +from paperpi.runner import PluginFailed, PluginTimeout, run_helper, run_update from .test_plugin import make as make_plugin @@ -58,6 +58,19 @@ def test_unknown_kinds_and_other_links_are_left_out_and_noted(): ) +def test_too_many_entries_are_left_out_and_secrets_are_hidden_everywhere(): + show = {f"Label {n}": "KEY" for n in range(12)} + found = clean( + {"choices": [choice(show=show, fill={"place": "KEY"})], "note": "the KEY"}, + SETTINGS, + hide=lambda text: text.replace("KEY", "****"), + ) + (only,) = found.choices + assert len(only.show) == 10 and {text for _, text in only.show} == {"****"} + assert only.fill == {"place": "****"} and found.note == "the ****" + assert found.left_out == ("choice 1: show: 2 entries over 10",) + + def test_texts_are_made_short_and_on_one_line(): found = clean([{"show": {"Place": "Morrison\n Colorado", "Long": "x" * 500}}], SETTINGS) (_, place), (_, long) = found.choices[0].show @@ -70,10 +83,12 @@ def test_texts_are_made_short_and_on_one_line(): ("Morrison", "list of choices"), ([["Morrison"]], "choice 1 is not a dictionary"), ([{"fill": {"lat": 1}}], "choice 1 has nothing to show"), - ([choice(fill={"altitude": 1757})], "fills settings it doesn't have: altitude"), + ([choice(fill={"altitude": 1757})], "fills settings it can't fill: altitude"), ([choice(fill=[1])], "fill must be a dictionary"), ([choice(link="https://osm.org/")], "link must be a dictionary"), ({"choices": [], "note": 5}, "note must be text"), + ([choice(fill={"lat": {"x": 1}})], "fill lat must be text, a number"), + ([choice(fill={"place": "x" * 1001})], "longer than 1000 characters"), ], ) def test_a_result_that_breaks_the_rules_is_a_bug_in_the_plugin(result, message): @@ -143,29 +158,26 @@ def test_run_helper_reports_a_problem_an_error_and_a_hang(words): run_helper("debugging", "words", "hang", words, time_limit=1) -def test_run_helper_hides_secret_settings_in_its_messages(monkeypatch): - class Secret(PluginSettings): - key: SecretStr = setting(SecretStr("")) +# --- With the runner tests' fake plugin, whose "leak" helper shows its API key --------------- + +FAKES = "tests.fake_plugins" - def leak(query): - raise HelperProblem(f"key {query.settings.key.get_secret_value()} refused") - plugin = make_plugin(settings=Secret, helpers={"leak": Helper(leak, "Key")}) - settings = Secret(key=SecretStr("TOPSECRET")) - # In this process (the fake plugin can't be imported in another one), as the child would. - sender = Sender() - monkeypatch.setattr(plugins, "load", lambda plugin_type, package: plugin) - monkeypatch.setattr(runner.os, "setpgrp", lambda: None) - runner._helper_child(sender, "fake", "tests", "leak", "", settings.model_dump()) - assert sender.sent == [("problem", "key **** refused")] +def test_a_search_never_shows_or_fills_an_api_key(): + from .fake_plugins.fake import Settings + keyed = Settings(key=SecretStr("TOP/SECRET")) + found = run_helper("fake", "leak", "", keyed, package=FAKES) + (choice,) = found.choices + assert choice.show == (("Key", "****"),) + assert choice.links == (("****", "https://example.org/?key=****"),) + assert found.note == "key ****" + with pytest.raises(PluginFailed, match="fills settings it can't fill: key"): + run_helper("fake", "leak", "fill", keyed, package=FAKES) -class Sender: - def __init__(self): - self.sent = [] - def send(self, message): - self.sent.append(message) +def test_helper_problem_in_an_update_is_an_ordinary_failure(tmp_path): + from .test_runner import context - def close(self): - pass + with pytest.raises(PluginFailed, match="HelperProblem: only for searches"): + run_update("fake", context(tmp_path, "problem"), package=FAKES) diff --git a/tests/test_web_helpers.py b/tests/test_web_helpers.py index db00a42..eff4e8c 100644 --- a/tests/test_web_helpers.py +++ b/tests/test_web_helpers.py @@ -7,11 +7,14 @@ import pytest from fastapi.testclient import TestClient +from pydantic import SecretStr from paperpi import config -from paperpi.helper import Found, HelperProblem, clean +from paperpi.helper import Found, Helper, HelperProblem, clean +from paperpi.plugin import Plugin, PluginSettings, ready, setting from paperpi.runner import PluginFailed, PluginTimeout from paperpi.web import auth +from paperpi.web import plugins as web_plugins from paperpi.web.app import create_app from paperpi.web.helpers import Searcher from paperpi.web.plugins import PluginEditor @@ -53,11 +56,11 @@ def cfg(tmp_path): return path -def make_client(cfg, search=None): +def make_client(cfg, search=None, searcher=None): editor = PluginEditor(cfg) - editor.loaded(CONFIG) + editor.loaded(cfg.read_text()) paperpi_auth = auth.Auth(cfg, config.WebSettings(login=False)) - searcher = Searcher(search) if search else Searcher() + searcher = searcher or (Searcher(search) if search else Searcher()) app = create_app(paperpi_auth, editor, searcher=searcher) return TestClient(app, follow_redirects=False, base_url=PI) @@ -138,7 +141,7 @@ def test_what_is_checked_before_searching(cfg): page = search(client, "x", delay="soon") assert "Fix these settings first" in page and "delay" in page assert "has no search" in client.post("/plugins/0/search/delay", data=FORM).text - assert "changed" in search(client, "x", id="other") + assert "The config file changed after this page was opened" in search(client, "x", id="o") assert not fake.calls @@ -180,18 +183,99 @@ def test_use_shows_values_that_cant_be_used(cfg): @pytest.mark.parametrize( - "use", - ["not json", "[1]", '{"text": "Morrison"}', '{"id": ["other"]}', '{"nope": ["x"]}'], + "key, use, message", + [ + ("text", "not json", "This choice can't be used"), + ("text", "[1]", "This choice can't be used"), + ("text", '{"text": "Morrison"}', "This choice can't be used"), + ("text", "[" * 200_000, "This choice can't be used"), # too long, and nested too deeply + ("text", '{"id": ["other"]}', "nothing the form can fill in"), + ("text", '{"refresh": ["60"]}', "nothing the form can fill in"), # a shared setting + ("delay", '{"text": ["x"]}', "The setting 'delay' has no search"), + ], +) +def test_use_refuses_what_a_use_button_doesnt_send(cfg, key, use, message): + sent = FORM | {"_use": use} + page = html.unescape(make_client(cfg).post(f"/plugins/0/use/{key}", data=sent).text) + assert message in page and "Filled in" not in page + + +def test_no_search_once_paperpi_stops(cfg): + fake = FakeSearch([]) + searcher = Searcher(fake) + client = make_client(cfg, searcher=searcher) + searcher.stop() + assert "PaperPi is stopping" in search(client, "x") and not fake.calls + + +class FinderSettings(PluginSettings): + key: SecretStr = setting(SecretStr(""), description="API key") + server: str = setting("", helper="servers", description="Music server") + + +FINDER = Plugin( + type="finder", + description="Finds servers", + settings=FinderSettings, + layouts={"one": {"column": [{"name": "t", "type": "text"}]}}, + fetch=lambda context: ready(None), + draw=lambda data, context: {}, + sample=None, + refresh=60, + helpers={ + "servers": Helper( + lambda query: [], "Finds the servers", "Find servers", typed=False, needs=("key",) + ) + }, ) -def test_use_refuses_what_a_use_button_doesnt_send(cfg, use): - page = html.unescape( - make_client(cfg).post("/plugins/0/use/text", data=FORM | {"_use": use}).text + + +@pytest.fixture +def finder(cfg, monkeypatch): + """A plugin block (place 1) whose helper needs no typed text but an API key first.""" + real_load, real_available = web_plugins.plugins.load, web_plugins.plugins.available + monkeypatch.setattr(web_plugins.plugins, "available", lambda: [*real_available(), "finder"]) + monkeypatch.setattr( + web_plugins.plugins, "load", lambda t: FINDER if t == "finder" else real_load(t) ) - assert "can't be used" in page or "nothing the form can fill in" in page - assert "Filled in" not in page + cfg.write_text(CONFIG + '\n[[plugin]]\nid = "finder-1"\ntype = "finder"\n') + return cfg + + +def find_servers(client, **form): + sent = {"id": "finder-1", "_search": "typed by hand"} | form + return html.unescape(client.post("/plugins/1/search/server", data=sent).text) + + +def test_a_helper_without_typed_text_has_only_a_button(finder): + page = make_client(finder).get("/plugins/1/settings", params={"id": "finder-1"}).text + box = page[page.index('id="helper-server"') :] + assert "

Finds the servers

" in box and ">Find servers" in box + assert 'name="_search"' not in box + + +def test_settings_a_helper_needs_come_first(finder): + fake = FakeSearch([]) + client = make_client(finder, fake) + assert "Fill in these settings first: key." in find_servers(client) + assert not fake.calls + find_servers(client, key="TYPED") + ((_, name, text, settings, _),) = fake.calls + assert (name, text, settings.key.get_secret_value()) == ("servers", "", "TYPED") + # An empty secret field keeps the saved key, which is enough. + finder.write_text(finder.read_text() + 'key = "SAVED"\n') + find_servers(client) + assert fake.calls[-1][3].key.get_secret_value() == "SAVED" def test_a_real_search_in_a_plugin_process(cfg): page = search(make_client(cfg), "hello world") assert "hello" in page and "Now: typed" in page assert uses(page) == [json.dumps({"text": ["hello"]}), json.dumps({"text": ["world"]})] + + +def test_what_a_search_left_out_goes_to_the_log(cfg, caplog): + fake = FakeSearch([{"show": {"Word": "x"}, "wait": 5}]) + with caplog.at_level("INFO", logger="paperpi.web.helpers"): + search(make_client(cfg, fake), "x") + assert f"search of {ID} left out: choice kind 'wait'" in caplog.text From 8abd0b69ae2cba7812c64e642cbec9464ed66acb Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:02:50 +0200 Subject: [PATCH 7/9] Setting helpers: the search box comes before the fields it fills (txoof) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 4 ++-- docs/writing-plugins.md | 4 ++-- src/paperpi/helper.py | 4 ++-- src/paperpi/plugin.py | 2 +- src/paperpi/web/static/style.css | 2 +- src/paperpi/web/templates/helper.html | 2 +- src/paperpi/web/templates/settings.html | 5 ++++- tests/test_web_helpers.py | 2 ++ 8 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index e53bc04..8628ee6 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -82,7 +82,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit ### Where helpers come from -- Helpers are part of the plugin, next to its settings and layouts. A setting names one as now: `setting(..., helper="location")`, and the search box is shown under that setting. A name the plugin doesn't have fails the plugin check, so a typing mistake is caught by the tests and not on the user's page. +- Helpers are part of the plugin, next to its settings and layouts. A setting names one as now: `setting(..., helper="location")`, and the search box is shown just before that setting, so name the first of the settings it fills. A name the plugin doesn't have fails the plugin check, so a typing mistake is caught by the tests and not on the user's page. - **PaperPi offers search functions** for needs many plugins share. The first is a place search: "find places for this text" returns places with name, region, country, latitude, longitude and altitude. A plugin's helper calls it and turns each place into a choice for its own settings. - For the common case PaperPi also has a **ready-made location helper**: the plugin only names its settings for latitude, longitude and, if it has them, altitude and the name shown on the screen. This is a shortcut built on the place search, not a separate idea. @@ -94,7 +94,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit ### On the settings page -- Under the field: the text, a search box and the button. The search starts with the button or the Enter key, not while the user types (txoof, 2026-10-10): one request per search, which matters for services with a daily limit such as NS. After a search, the list of choices appears, each with a **Use** button. "Nothing found" comes with a hint (check the spelling, or try a larger place). +- Just before the setting's field: the text, a search box and the button (txoof, 2026-10-10: the search comes first, then the fields it fills). The search starts with the button or the Enter key, not while the user types (txoof, 2026-10-10): one request per search, which matters for services with a daily limit such as NS. After a search, the list of choices appears, each with a **Use** button. "Nothing found" comes with a hint (check the spelling, or try a larger place). - **Use** fills in the form's fields. **Nothing is saved yet**: the user sees the values, can change them, can press Preview, and then Save. The list stays until the next search, so a wrong pick is easy to change. - No new JavaScript library: htmx (the small script the pages already use) sends the search and puts the values in the fields. - Typing the values by hand keeps working. When the service can't be reached, the page says so and the fields are left as they are. diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 5ce05b7..9884133 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -102,9 +102,9 @@ hours = 12 ## Setting helpers -Some settings are hard to fill in by hand, such as a place's latitude and longitude or a stop's code. A plugin can offer a **helper**: a search box under the setting on its settings page. The user types something, PaperPi shows a list of choices, and **Use** fills in the form's fields (nothing is saved until Save). The reasons are in `docs/decisions/setting-helpers.md`. +Some settings are hard to fill in by hand, such as a place's latitude and longitude or a stop's code. A plugin can offer a **helper**: a search box on its settings page, just before the setting it fills. The user types something, PaperPi shows a list of choices, and **Use** fills in the form's fields (nothing is saved until Save). The reasons are in `docs/decisions/setting-helpers.md`. -- A helper is a `paperpi.plugin.Helper` in the plugin's `helpers`, by name. A setting names it with `setting(..., helper="")`; the search box is shown under that setting. The plugin check fails for a name the plugin doesn't have. +- A helper is a `paperpi.plugin.Helper` in the plugin's `helpers`, by name. A setting names it with `setting(..., helper="")`; the search box is shown just before that setting, so name the first of the settings it fills. The plugin check fails for a name the plugin doesn't have. - Its `find(query)` gets a `Query`: `query.text` (what the user typed, without spaces around it, at most 100 characters; empty for `typed=False`) and `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret). - It returns a list of choices, or `{"choices": [...], "note": "..."}` with a message for the whole result. Each choice is a dictionary; its keys say what PaperPi does with each value: - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone (e.g. a name the user already typed). Only settings the form shows as a field: not secrets (`SecretStr`), and not settings that are changed in the file (such as a list of numbers). diff --git a/src/paperpi/helper.py b/src/paperpi/helper.py index 7a49e73..f1030a5 100644 --- a/src/paperpi/helper.py +++ b/src/paperpi/helper.py @@ -2,8 +2,8 @@ latitude and longitude from its name (see ``docs/decisions/setting-helpers.md``). A helper is a :class:`Helper` in the plugin's ``helpers``. A setting names it with -``setting(..., helper="")`` and the settings page shows a search box under that -setting. The helper's ``find`` gets a :class:`Query` (the typed text and the form's settings) +``setting(..., helper="")`` and the settings page shows a search box just before +that setting. The helper's ``find`` gets a :class:`Query` (the typed text and the form's settings) and returns choices: a list of dictionaries, or ``{"choices": [...], "note": "..."}``. Each choice is a dictionary whose keys are **kinds** that say what PaperPi does with the value: diff --git a/src/paperpi/plugin.py b/src/paperpi/plugin.py index 713359e..547c520 100644 --- a/src/paperpi/plugin.py +++ b/src/paperpi/plugin.py @@ -125,7 +125,7 @@ def setting( ``paperpi list`` and the web interface's Active Plugins page say which settings it needs. ``helper``: the name of one of the plugin's ``helpers`` (a search, such as finding a - place's latitude and longitude), shown under this setting's field on the settings page. + place's latitude and longitude), shown just before this setting's field on the settings page. See :mod:`paperpi.helper`. """ extra = field.pop("json_schema_extra", None) or {} diff --git a/src/paperpi/web/static/style.css b/src/paperpi/web/static/style.css index 709bdad..3a161d6 100644 --- a/src/paperpi/web/static/style.css +++ b/src/paperpi/web/static/style.css @@ -40,7 +40,7 @@ code { background: #eee; padding: 0 0.2rem; } .preview { margin: 1rem 0; } .preview img { display: block; max-width: 100%; height: auto; border: 1px solid #999; } .preview figcaption { color: #555; font-size: 0.9rem; margin-top: 0.4rem; } -.helper { margin: -0.4rem 0 1rem; padding: 0.5rem 0.75rem; border-left: 4px solid #ccc; } +.helper { margin: 0.9rem 0; padding: 0.5rem 0.75rem; border-left: 4px solid #ccc; background: #f6f6f6; } .helper > label { display: block; margin: 0 0 0.25rem; } .helper .search { display: flex; gap: 0.5rem; } .helper .search input { flex: 1; } diff --git a/src/paperpi/web/templates/helper.html b/src/paperpi/web/templates/helper.html index 9ae526a..8170397 100644 --- a/src/paperpi/web/templates/helper.html +++ b/src/paperpi/web/templates/helper.html @@ -1,4 +1,4 @@ -{#- A setting helper's search box, under its field (settings.html). Its input and button belong +{#- A setting helper's search box, just before its setting's field (settings.html). Its input and button belong to the form search-form- after the settings form. -#}
{% if helper.typed %} diff --git a/src/paperpi/web/templates/settings.html b/src/paperpi/web/templates/settings.html index b493b73..773fee5 100644 --- a/src/paperpi/web/templates/settings.html +++ b/src/paperpi/web/templates/settings.html @@ -26,8 +26,11 @@

{{ row.name }}

{% for f in groups.name %}{{ field(f) }}{% endfor %} {% if groups.own %}

Settings of this plugin

- {% for f in groups.own %}{{ field(f) }} + {# A search box comes before the setting that names its helper: it fills that setting + and often the ones after it (latitude, longitude, ...). #} + {% for f in groups.own %} {%- set helper = plugin.helper(f.key) %}{% if helper %}{% include "helper.html" %}{% endif %} + {{- field(f) }} {%- endfor %} {% endif %}

When and how it is shown

diff --git a/tests/test_web_helpers.py b/tests/test_web_helpers.py index eff4e8c..f35ab38 100644 --- a/tests/test_web_helpers.py +++ b/tests/test_web_helpers.py @@ -85,6 +85,8 @@ def test_the_page_has_a_search_box_with_a_form_of_its_own(cfg): assert ' Date: Sat, 10 Oct 2026 22:08:16 +0200 Subject: [PATCH 8/9] Setting helpers: Use always replaces the name shown on the screen (txoof) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 6 +++--- docs/writing-plugins.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index 8628ee6..429c75d 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -61,7 +61,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit | Kind | Where | What PaperPi does with it | |---|---|---| -| `fill` | choice | Settings put into the form when the user presses **Use**. The keys are the plugin's own setting names. PaperPi checks them with the plugin's settings description, as Save does, so a wrong name or value is an error and never a broken setting. Only what is in `fill` changes, so a helper leaves a setting alone by leaving it out (e.g. the name shown on the screen, when the user has already typed one). Only settings the form shows as a field: not secrets, and not settings that are changed in the file. | +| `fill` | choice | Settings put into the form when the user presses **Use**. The keys are the plugin's own setting names. PaperPi checks them with the plugin's settings description, as Save does, so a wrong name or value is an error and never a broken setting. Only what is in `fill` changes, so a helper leaves a setting alone by leaving it out. Only settings the form shows as a field: not secrets, and not settings that are changed in the file. | | `show` | choice | Information to help the user pick: label and text, shown in the order given. The first entry is the heading of the choice. | | `link` | choice | Web addresses the user can open in a new tab, e.g. "See on map". Only `http` and `https` addresses; others are left out and logged. | | `note` | result | A message for the whole result, e.g. "Only the first 10 matches are shown; type more to narrow it down". | @@ -110,7 +110,7 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit - The place search uses **Open-Meteo** (`geocoding-api.open-meteo.com`), chosen with txoof on 2026-10-10. No key and no sign-up. It finds towns and cities (not street addresses) and returns region, country, latitude, longitude **and altitude**; Nominatim, which v1 used, returns no altitude. It understands "Morrison, Colorado"; a plain "Morrison" returns 10 towns in several countries, so region and country are shown with each choice. - Terms: free for non-commercial use, fewer than 10,000 requests a day. The data is CC BY 4.0 and the place data is from GeoNames, so the page shows "Search by Open-Meteo, place data from GeoNames" with links next to the list. - A choice: - - `fill`: `lat` and `lon` (4 decimals, about 10 m), `altitude` (whole metres, as met.no asks), and the name shown on the screen (`place` in `met_no`) only when it is still empty, so a name the user typed ("Home", "Cabin") is kept; + - `fill`: `lat` and `lon` (4 decimals, about 10 m), `altitude` (whole metres, as met.no asks), and the name shown on the screen (`place` in `met_no`). The name is always replaced: choosing a place is the point of the search, and the user can change the name afterwards (txoof, 2026-10-10, after trying the demo; the first plan filled it only while empty, so it was set once and never again); - `show`: place, region, country, altitude; - `link`: "See on map" (OpenStreetMap at that point), so the user can check the place without a map in PaperPi. - `met_no` gets an optional `altitude` setting (whole metres, -500 to 9000), sent to met.no when it is filled in. This closes issue #176. The example in #176 had a sign mistake: `lon = 84.31` is on the Tibetan plateau, where met.no's 4949 m is right; Atlanta is at `-84.31`. @@ -126,4 +126,4 @@ A choice is a dictionary. Each key is a **kind** that says what PaperPi does wit None. Answered with txoof on 2026-10-10: 1. The search starts with the button or the Enter key, not while the user types. 2. The command line comes later, as its own issue, as `paperpi helper met_no.location "Morrison, Colorado"` (see "Later"). -3. The name shown on the screen is filled in only when it is empty. This is decided by the helper (it leaves `place` out of `fill`), not by a rule in PaperPi (see "What a choice is"). +3. The name shown on the screen is always replaced by **Use** (changed after the demo: "only while it is empty" filled it once and never again). Either way it is decided by the helper (what it puts in `fill`), not by a rule in PaperPi (see "What a choice is"). diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 9884133..6385530 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -107,7 +107,7 @@ Some settings are hard to fill in by hand, such as a place's latitude and longit - A helper is a `paperpi.plugin.Helper` in the plugin's `helpers`, by name. A setting names it with `setting(..., helper="")`; the search box is shown just before that setting, so name the first of the settings it fills. The plugin check fails for a name the plugin doesn't have. - Its `find(query)` gets a `Query`: `query.text` (what the user typed, without spaces around it, at most 100 characters; empty for `typed=False`) and `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret). - It returns a list of choices, or `{"choices": [...], "note": "..."}` with a message for the whole result. Each choice is a dictionary; its keys say what PaperPi does with each value: - - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone (e.g. a name the user already typed). Only settings the form shows as a field: not secrets (`SecretStr`), and not settings that are changed in the file (such as a list of numbers). + - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone. Only settings the form shows as a field: not secrets (`SecretStr`), and not settings that are changed in the file (such as a list of numbers). - `show`: label and text, shown to help the user choose; the first one is the heading. - `link`: label and web address (`http` or `https` only), e.g. "See on map". A kind PaperPi doesn't know is left out and logged. At most 10 choices are shown. Each text is put on one line and cut to 200 characters. @@ -129,7 +129,7 @@ def find_stops(query: Query): found = search_the_stop_list(query.text) # the plugin's own code return [ { - "fill": {"stop": s.code} | ({} if query.settings.stop_name else {"stop_name": s.name}), + "fill": {"stop": s.code, "stop_name": s.name}, "show": {"Stop": s.name, "Town": s.town}, } for s in found From 5f38d9ff1d01b2cb2aee169a0539048c185a68d2 Mon Sep 17 00:00:00 2001 From: txoof-bot <337660340+txoof-bot@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:11:17 +0200 Subject: [PATCH 9/9] Setting helpers: a search gets the browser's language (txoof: Den Haag) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/setting-helpers.md | 2 +- docs/writing-plugins.md | 2 +- src/paperpi/helper.py | 4 ++++ src/paperpi/plugins/debugging/__init__.py | 2 +- src/paperpi/runner.py | 17 +++++++++++----- src/paperpi/web/app.py | 7 +++++-- src/paperpi/web/helpers.py | 22 ++++++++++++++++++--- src/paperpi/web/plugins.py | 10 ++++++++-- tests/test_web_helpers.py | 24 ++++++++++++++++++++++- 9 files changed, 74 insertions(+), 16 deletions(-) diff --git a/docs/decisions/setting-helpers.md b/docs/decisions/setting-helpers.md index 429c75d..9c9bac8 100644 --- a/docs/decisions/setting-helpers.md +++ b/docs/decisions/setting-helpers.md @@ -40,7 +40,7 @@ v2 so far: a setting can name a helper (`setting(..., helper="location")`), and - the button text (default "Search"). A helper that needs no typed text (finding music servers) shows only a button, e.g. "Find servers"; - where the answers come from, with a link, shown under the list (e.g. "Search by Open-Meteo"). The user sees that their typed text goes to an outside service, and the credit that service asks for is given; - optional: settings that must be filled in first (the NS API key). Until they are, the box says so instead of searching. -- The function gets what the user typed and the form's current settings, checked as Save would check them. An empty secret field (an API key) gives the saved secret, as in a preview (`web-interface.md`). +- The function gets what the user typed, the form's current settings, checked as Save would check them, and the language the user's browser asks for first (e.g. `nl`). An empty secret field (an API key) gives the saved secret, as in a preview (`web-interface.md`). The language was added after the demo (txoof, 2026-10-10): Open-Meteo finds "Den Haag" only when asked in Dutch. - It returns a **result**: a list of **choices**, and optionally a `note` for the whole result (below). - It may also stop with a message for the user ("The NS API key was not accepted"). Any other error shows "The search failed" and the details go to the log. - It uses `paperpi.webrequest` for web requests, like plugins do, so the same time limits and size limits apply. diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 6385530..260011a 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -105,7 +105,7 @@ hours = 12 Some settings are hard to fill in by hand, such as a place's latitude and longitude or a stop's code. A plugin can offer a **helper**: a search box on its settings page, just before the setting it fills. The user types something, PaperPi shows a list of choices, and **Use** fills in the form's fields (nothing is saved until Save). The reasons are in `docs/decisions/setting-helpers.md`. - A helper is a `paperpi.plugin.Helper` in the plugin's `helpers`, by name. A setting names it with `setting(..., helper="")`; the search box is shown just before that setting, so name the first of the settings it fills. The plugin check fails for a name the plugin doesn't have. -- Its `find(query)` gets a `Query`: `query.text` (what the user typed, without spaces around it, at most 100 characters; empty for `typed=False`) and `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret). +- Its `find(query)` gets a `Query`: `query.text` (what the user typed, without spaces around it, at most 100 characters; empty for `typed=False`), `query.settings` (the form's settings, checked as Save checks them; an empty secret field gives the saved secret) and `query.language` (the language the user's browser asks for first, e.g. `nl`; `en` when not known), for services that find names only in the language they are asked in. - It returns a list of choices, or `{"choices": [...], "note": "..."}` with a message for the whole result. Each choice is a dictionary; its keys say what PaperPi does with each value: - `fill`: the plugin's own settings and their values, put into the form by **Use**. Leave a setting out to leave it alone. Only settings the form shows as a field: not secrets (`SecretStr`), and not settings that are changed in the file (such as a list of numbers). - `show`: label and text, shown to help the user choose; the first one is the heading. diff --git a/src/paperpi/helper.py b/src/paperpi/helper.py index f1030a5..682469d 100644 --- a/src/paperpi/helper.py +++ b/src/paperpi/helper.py @@ -50,6 +50,10 @@ class Query: settings: PluginSettings """The settings in the form, checked as Save checks them. An empty secret field (an API key) gives the saved secret.""" + language: str = "en" + """The language the user's browser asks for first, as a short lowercase code such as + ``nl`` or ``de`` (``en`` when it is not known). For services that find names only in + the language they are asked in.""" @dataclass(frozen=True) diff --git a/src/paperpi/plugins/debugging/__init__.py b/src/paperpi/plugins/debugging/__init__.py index 6ae5330..e5d6aeb 100644 --- a/src/paperpi/plugins/debugging/__init__.py +++ b/src/paperpi/plugins/debugging/__init__.py @@ -106,7 +106,7 @@ def find_words(query: Query): choices = [ { "fill": {"text": word}, - "show": {"Word": word, "Now": query.settings.text}, + "show": {"Word": word, "Now": query.settings.text, "Language": query.language}, "link": {"Example": "https://example.org/"}, } for word in words diff --git a/src/paperpi/runner.py b/src/paperpi/runner.py index dd637c7..f61d68a 100644 --- a/src/paperpi/runner.py +++ b/src/paperpi/runner.py @@ -108,18 +108,19 @@ def run_helper( settings: PluginSettings, *, time_limit: float | None = None, + language: str = "en", package: str = "paperpi.plugins", ) -> Found: """Run one search of the plugin's helper ``name`` (see :mod:`paperpi.helper`) in its own - process, for the typed ``text`` and the form's ``settings``, and return its checked - result. + process, for the typed ``text``, the form's ``settings`` and the user's ``language`` + (see :class:`~paperpi.helper.Query`), and return its checked result. ``time_limit`` defaults to :data:`paperpi.limits.HELPER_SEARCH`. Raises :class:`HelperProblem` with the helper's message for the user, and otherwise the same errors as :func:`run_update`. """ time_limit = limits.HELPER_SEARCH if time_limit is None else time_limit - args = (plugin_type, package, name, text, settings.model_dump()) + args = (plugin_type, package, name, text, settings.model_dump(), language) return _run(plugin_type, _helper_child, args, time_limit)[1] @@ -233,7 +234,13 @@ def _child( def _helper_child( - sender: Connection, plugin_type: str, package: str, name: str, text: str, settings: dict + sender: Connection, + plugin_type: str, + package: str, + name: str, + text: str, + settings: dict, + language: str, ) -> None: """Runs in the plugin process: one search of a helper, then send the result and exit.""" with _reporting(sender) as secrets: @@ -241,7 +248,7 @@ def _helper_child( plugin, checked = _load(plugin_type, package, settings, secrets) try: - result = plugin.helpers[name].find(Query(text, checked)) + result = plugin.helpers[name].find(Query(text, checked, language)) except HelperProblem as problem: # only here: in an update it is an ordinary error sender.send(("problem", _hide(str(problem), secrets))) return diff --git a/src/paperpi/web/app.py b/src/paperpi/web/app.py index 3f6ad35..d50f96b 100644 --- a/src/paperpi/web/app.py +++ b/src/paperpi/web/app.py @@ -50,7 +50,7 @@ from .auth import COOKIE_SECONDS, Auth, new_cookie from .config_file import EditError from .forms import Read, fields, form_texts -from .helpers import Searcher, SearchFailed +from .helpers import Searcher, SearchFailed, language from .password import PasswordError, password_problem from .pictures import Pictures from .plugins import FormErrors, PluginEditor, library, library_item @@ -326,7 +326,10 @@ async def search(request: Request, index: int, key: str): text = (sent.pop("_search", None) or [""])[-1] job = None try: - job = await run_in_threadpool(editor.search_job, index, plugin_id, sent, key, text) + spoken = language(request.headers.get("accept-language")) + job = await run_in_threadpool( + editor.search_job, index, plugin_id, sent, key, text, spoken + ) found = await run_in_threadpool(searcher.find, job) except FormErrors as error: return page(request, "found.html", errors=error.found.errors) diff --git a/src/paperpi/web/helpers.py b/src/paperpi/web/helpers.py index e16708b..c29ef4a 100644 --- a/src/paperpi/web/helpers.py +++ b/src/paperpi/web/helpers.py @@ -23,11 +23,20 @@ log = logging.getLogger(__name__) -#: Runs one search: ``search(plugin_type, name, text, settings, time_limit=...)``, as -#: :func:`paperpi.runner.run_helper`. Tests pass a quick fake. +#: Runs one search: ``search(plugin_type, name, text, settings, time_limit=..., language=...)``, +#: as :func:`paperpi.runner.run_helper`. Tests pass a quick fake. Search = Callable[..., Found] +def language(accept_language: str | None) -> str: + """The language a browser asks for first, from its ``Accept-Language`` header (e.g. + ``nl-NL,nl;q=0.9,en;q=0.8`` gives ``nl``), or ``en``. Only 2 or 3 letters are taken, so + the header can't put anything else into a helper's web request.""" + first = (accept_language or "").split(",")[0].split(";")[0].strip() + code = first.split("-")[0].lower() + return code if code.isascii() and code.isalpha() and 2 <= len(code) <= 3 else "en" + + class SearchFailed(RuntimeError): """The search could not be done or gave no answer; the message says why.""" @@ -43,6 +52,8 @@ class Job: """The helper's name in the plugin's ``helpers``.""" helper: Helper text: str + language: str = "en" + """The language the browser asks for first (see :func:`language`).""" class Searcher: @@ -78,7 +89,12 @@ def find(self, job: Job) -> Found: plugin_type = job.found.plugin.type try: found = self._search( - plugin_type, job.name, job.text, settings, time_limit=limits.HELPER_SEARCH + plugin_type, + job.name, + job.text, + settings, + time_limit=limits.HELPER_SEARCH, + language=job.language, ) except PluginTimeout: raise SearchFailed( diff --git a/src/paperpi/web/plugins.py b/src/paperpi/web/plugins.py index 51cd275..13d110c 100644 --- a/src/paperpi/web/plugins.py +++ b/src/paperpi/web/plugins.py @@ -184,7 +184,13 @@ def preview_job( return preview.Job(found, width, height, display.screen_mode) def search_job( - self, index: int, plugin_id: str, form: Mapping[str, Sequence[str]], key: str, text: str + self, + index: int, + plugin_id: str, + form: Mapping[str, Sequence[str]], + key: str, + text: str, + language: str = "en", ) -> helpers.Job: """A search with the helper of setting ``key`` of block ``index``, for the typed ``text`` and the settings of a sent form as Save would write them. Nothing is @@ -197,7 +203,7 @@ def search_job( helper = found.plugin.helpers[name] # A helper without a search box gets no text, also from a form sent by hand. text = text.strip() if helper.typed else "" - return helpers.Job(found, key, name, helper, text) + return helpers.Job(found, key, name, helper, text, language) def filled( self, diff --git a/tests/test_web_helpers.py b/tests/test_web_helpers.py index f35ab38..892f988 100644 --- a/tests/test_web_helpers.py +++ b/tests/test_web_helpers.py @@ -42,8 +42,9 @@ def __init__(self, answer): self.answer = answer self.calls = [] - def __call__(self, plugin_type, name, text, settings, *, time_limit): + def __call__(self, plugin_type, name, text, settings, *, time_limit, language): self.calls.append((plugin_type, name, text, settings, time_limit)) + self.language = language if isinstance(self.answer, Exception): raise self.answer return clean(self.answer, ["text", "delay"]) @@ -273,6 +274,7 @@ def test_settings_a_helper_needs_come_first(finder): def test_a_real_search_in_a_plugin_process(cfg): page = search(make_client(cfg), "hello world") assert "hello" in page and "Now: typed" in page + assert "Language: en" in page assert uses(page) == [json.dumps({"text": ["hello"]}), json.dumps({"text": ["world"]})] @@ -281,3 +283,23 @@ def test_what_a_search_left_out_goes_to_the_log(cfg, caplog): with caplog.at_level("INFO", logger="paperpi.web.helpers"): search(make_client(cfg, fake), "x") assert f"search of {ID} left out: choice kind 'wait'" in caplog.text + + +@pytest.mark.parametrize( + "header, spoken", + [ + ("nl-NL,nl;q=0.9,en;q=0.8", "nl"), + ("de", "de"), + ("EN-gb", "en"), + (None, "en"), + ("", "en"), + ("*", "en"), + ("x&q=1", "en"), # only letters reach a helper + ], +) +def test_the_browsers_language_goes_to_the_helper(cfg, header, spoken): + fake = FakeSearch([]) + client = make_client(cfg, fake) + headers = {"Accept-Language": header} if header is not None else {} + client.post("/plugins/0/search/text", data=FORM | {"_search": "x"}, headers=headers) + assert fake.language == spoken