'}`,
-)
-if (!forceWasi) {
- nativeBinding = requireNative()
- __dbg(`requireNative() returned ${nativeBinding ? 'a binding' : 'nothing'}`)
-} else {
- __dbg('skipping the native chain: WASI was forced')
-}
-
-if (!nativeBinding || forceWasi) {
- __dbg('entering the WASI fallback chain')
- let wasiBinding = null
- let wasiBindingLoaded = false
- const wasiBindingErrors = __dbgTracedArray('wasiBindingErrors')
- const __napiWasiResolveCandidate = (specifier, isPackage, localArtifacts) => {
- try {
- require.resolve(specifier)
- } catch (resolveError) {
- if (!resolveError || resolveError.code !== 'MODULE_NOT_FOUND') {
- throw resolveError
- }
- if (isPackage) {
- try {
- require.resolve(specifier + '/package.json')
- } catch (packageError) {
- if (packageError && packageError.code === 'MODULE_NOT_FOUND') {
- return resolveError
- }
- // An exports restriction proves the package exists even when its
- // package.json is not public. Preserve the root resolution failure.
- throw resolveError
- }
- // The package exists but its main/export target is broken.
- throw resolveError
- }
- return resolveError
- }
- if (localArtifacts) {
- let artifactError = null
- for (let i = 0; i < localArtifacts.length; i++) {
- try {
- require.resolve(localArtifacts[i])
- return null
- } catch (resolveError) {
- if (!resolveError || resolveError.code !== 'MODULE_NOT_FOUND') {
- throw resolveError
- }
- artifactError = resolveError
- }
- }
- return artifactError
- }
- return null
- }
- if (!wasiBindingLoaded && (!__napiWasiFlavorRequested || __napiWasiFlavor === 'wasm32-wasi')) {
- let candidateError = null
- let candidateFailed = false
- try {
- candidateError = __napiWasiResolveCandidate('./codechat-editor-client.wasi.cjs', false, ['./codechat-editor-client.wasm32-wasi.debug.wasm', './codechat-editor-client.wasm32-wasi.wasm'])
- candidateFailed = candidateError !== null
- if (!candidateFailed) {
- __dbgAttempt('WASI local artifact', './codechat-editor-client.wasi.cjs')
- wasiBinding = __dbgOk(
- './codechat-editor-client.wasi.cjs',
- require('./codechat-editor-client.wasi.cjs'),
- )
- nativeBinding = wasiBinding
- __napiLoadedBindingTarget = 'wasm32-wasi'
- wasiBindingLoaded = true
- }
- } catch (err) {
- candidateError = err
- candidateFailed = true
- }
- if (candidateFailed) {
- wasiBindingErrors.push(candidateError)
- loadErrors.push(candidateError)
- }
- }
- if (!wasiBindingLoaded && (!__napiWasiFlavorRequested || __napiWasiFlavor === 'wasm32-wasi')) {
- let candidateError = null
- let candidateFailed = false
- try {
- candidateError = __napiWasiResolveCandidate('codechat-editor-client-wasm32-wasi', true, undefined)
- candidateFailed = candidateError !== null
- if (!candidateFailed) {
- if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
- const bindingPackageVersion = require('codechat-editor-client-wasm32-wasi/package.json').version
- if (bindingPackageVersion !== '0.2.5') {
- throw new Error(`WASI binding package version mismatch, expected 0.2.5 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
- }
- }
- __dbgAttempt('WASI package', 'codechat-editor-client-wasm32-wasi')
- wasiBinding = __dbgOk(
- 'codechat-editor-client-wasm32-wasi',
- require('codechat-editor-client-wasm32-wasi'),
- )
- nativeBinding = wasiBinding
- __napiLoadedBindingTarget = 'wasm32-wasi'
- wasiBindingLoaded = true
- }
- } catch (err) {
- candidateError = err
- candidateFailed = true
- }
- if (candidateFailed) {
- wasiBindingErrors.push(candidateError)
- loadErrors.push(candidateError)
- }
- }
- if (
- !wasiBindingLoaded &&
- forceWasi &&
- !forceWasiError &&
- !__napiWasiFlavorRequested
- ) {
- __dbg('WASI was preferred but not found; retrying the native chain')
- nativeBinding = requireNative()
- }
- if ((forceWasiError || __napiWasiFlavorRequested) && !wasiBindingLoaded) {
- const error = new Error(
- __napiWasiFlavorRequested
- ? 'WASI binding for flavor "' + __napiWasiFlavor + '" not found'
- : 'WASI binding not found and NAPI_RS_FORCE_WASI is set to error',
- )
- __dbgChain('WASI load failed', wasiBindingErrors)
- __dbg(`THROWING: ${error.message}`)
- error.cause = createLoadErrorChain(wasiBindingErrors)
- throw error
- }
-}
-
-__dbg(
- `binding resolution finished: ${nativeBinding ? 'SUCCESS' : 'FAILURE'}, target=${__napiLoadedBindingTarget}`,
-)
-if (!nativeBinding) {
- if (loadErrors.length > 0) {
- // Lead with a diagnosis of the Windows failure where one can be reached:
- // it names the cause, while the message below names only the symptom.
- const windowsDiagnosis = __napiDiagnoseWindowsLoadFailure(loadErrors)
- if (windowsDiagnosis !== null) {
- __dbg(`diagnosis: ${windowsDiagnosis}`)
- }
- const error = new Error(
- windowsDiagnosis === null
- ? `Cannot find native binding. ` +
- `npm has a bug related to optional dependencies (https://github.com/npm/cli/issues/4828). ` +
- 'Please try `npm i` again after removing both package-lock.json and node_modules directory.'
- : windowsDiagnosis,
- )
- // assign instead of the `new Error(message, { cause })` options form,
- // which Node < 16.9 silently ignores
- __dbgChain('every load attempt failed', loadErrors)
- __dbg(`THROWING: ${error.message}`)
- error.cause = createLoadErrorChain(loadErrors)
- throw error
- }
- __dbg('THROWING: Failed to load native binding, with no attempt recorded')
- throw new Error(`Failed to load native binding`)
-}
-
-function __napiStampBindingTarget(exportsObject, target) {
- if (
- Object.prototype.hasOwnProperty.call(exportsObject, '__napiBindingTarget')
- ) {
- if (exportsObject.__napiBindingTarget === target) {
- // Already ours: the root entry aliases the object it loaded, so a WASI
- // fallback candidate — or a `NAPI_RS_NATIVE_LIBRARY_PATH` override that
- // is a generated loader — arrives already stamped with this same value.
- return target
- }
- const error = new Error(
- '`__napiBindingTarget` is reserved by the generated binding loader, but the loaded binding already exports it. Rename the export, e.g. #[napi(js_name = "...")].',
- )
- error.code = 'ERR_NAPI_BINDING_TARGET_CONFLICT'
- throw error
- }
- if (!Object.isExtensible(exportsObject)) {
- // A `#[napi(module_exports)]` hook may seal or freeze this object
- // (`Object::seal` / `Object::freeze`). Reporting the artifact is metadata,
- // never a reason to fail an otherwise successful load, so the stamp is
- // skipped. What a consumer still sees then follows the entry point: the
- // browser and deferred loaders declare `__napiBindingTarget` at module
- // level and go on reporting it, while the CommonJS entries hand back this
- // very object as `module.exports`, so there the value is absent.
- return target
- }
- try {
- // [[Define]], not [[Set]]: an ordinary assignment walks the prototype
- // chain, so an inherited accessor could swallow the value or throw and
- // fail an otherwise successful load. The descriptor is what a successful
- // assignment would have produced.
- Object.defineProperty(exportsObject, '__napiBindingTarget', {
- configurable: true,
- enumerable: true,
- value: target,
- writable: true,
- })
- } catch {
- // Same rule as the non-extensible skip above: reporting the artifact is
- // metadata, never a reason to fail an otherwise successful load. An exotic
- // object (a Proxy whose defineProperty trap refuses) is skipped, not
- // thrown over.
- }
- // The CommonJS loaders assign this return value so `cjs-module-lexer` — and
- // therefore Node's CJS -> ESM named export detection — can see
- // `__napiBindingTarget` statically.
- return target
-}
-// Stamp before the alias, not after. The guard only reads `nativeBinding`
-// (`hasOwnProperty` plus a comparison), which is safe against any addon
-// accessor; an assignment is not, because a `#[napi(module_exports)]` hook can
-// expose a getter reporting this very value and a setter that throws. So the
-// assignment lands on the loader's own `module.exports`, still the original
-// object here, and the alias below replaces it.
-//
-// The assignment is what keeps the marker a statically visible CommonJS export:
-// `cjs-module-lexer` is Node's CJS -> ESM named export detection, it cannot see
-// a bare call, and the later `module.exports = nativeBinding` does not undo the
-// detection. The assignment itself always succeeds — its target is this
-// loader's own, still extensible `module.exports` — and the alias below then
-// discards the value it wrote. What a consumer reads is whatever the guard put
-// on `nativeBinding`, so on a frozen binding, where the guard skips, the
-// linked import resolves to `undefined`.
-module.exports.__napiBindingTarget = __napiStampBindingTarget(nativeBinding, __napiLoadedBindingTarget)
-module.exports = nativeBinding
-module.exports.CodeChatEditorServer = nativeBinding.CodeChatEditorServer
-module.exports.initServer = nativeBinding.initServer
-
-__dbg(
- `exports check: CodeChatEditorServer=${typeof nativeBinding.CodeChatEditorServer} initServer=${typeof nativeBinding.initServer}`,
-)
-__dbg('==== binding load finished ====')
diff --git a/extensions/standalone/src/filewatcher.rs b/extensions/standalone/src/filewatcher.rs
index 82cca999..08fc4de8 100644
--- a/extensions/standalone/src/filewatcher.rs
+++ b/extensions/standalone/src/filewatcher.rs
@@ -447,7 +447,7 @@ fn processing_task(
let (watcher_tx, mut watcher_rx) = mpsc::channel(10);
// Watch this file. Use the debouncer, to avoid multiple notifications
// for the same file. This approach returns a result of either a working
- // debouncer or any errors that occurred. The debouncer's scope needs
+ // debouncer or any errors that occurred. The debouncer's scope needs to
// live as long as this connection does; dropping it early means losing
// file change notifications.
let Ok(mut debounced_watcher) = new_debouncer(
@@ -710,6 +710,7 @@ fn processing_task(
EditorMessageContents::OpenUrl(_) |
EditorMessageContents::Capture(_) |
EditorMessageContents::ClientHtml(_) |
+ EditorMessageContents::UpdateToc(_) |
EditorMessageContents::RequestClose => {
let err = ResultErrTypes::ClientIllegalMessage;
error!("{err:?}");
@@ -786,8 +787,8 @@ mod tests {
};
use code_chat_editor::{
processing::{
- CodeChatForWeb, CodeMirror, CodeMirrorDiffable, SourceFileMetadata,
- source_to_codechat_for_web,
+ CodeChatForWeb, CodeMirror, CodeMirrorDiff, CodeMirrorDiffable, SourceFileMetadata,
+ StringDiff, source_to_codechat_for_web,
},
webserver::{
EditorMessage, EditorMessageContents, INITIAL_CLIENT_MESSAGE_ID,
@@ -1157,7 +1158,9 @@ mod tests {
let mut s = fs::read_to_string(&file_path).unwrap();
assert_eq!(s, "testing()");
- // 8. Change this file and verify that this produces an update.
+ // 8. Change this file and verify that this produces an update. Since
+ // the Client has the file's full text, this is a diff against the
+ // contents and version it sent in step 7.
//
// Message ids: IDE - 1->2, Server - 3, Client - 7.
s.push_str("123");
@@ -1187,9 +1190,14 @@ mod tests {
metadata: SourceFileMetadata {
mode: "python".to_string(),
},
- source: CodeMirrorDiffable::Plain(CodeMirror {
- doc: "testing()123".to_string(),
+ source: CodeMirrorDiffable::Diff(CodeMirrorDiff {
+ doc: vec![StringDiff {
+ from: 0,
+ to: Some(9),
+ insert: "testing()123".to_string(),
+ }],
doc_blocks: vec![],
+ version: 2.0,
}),
version: msg.1.contents.as_ref().unwrap().version,
}),
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 4e3f2513..92ec62ec 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -57,16 +57,19 @@ htmd = { git = "https://github.com/bjones1/htmd.git", branch = "dom-interface",
# This must match the version of `markup5ever_rcdom`.
html5ever = "0.39"
htmlize = "1.0.6"
+ignore = "0.4"
imara-diff = { version = "0.2", features = [] }
indoc = "2.0.5"
log.workspace = true
log4rs = "1.3"
+lru = "0.18.5"
markup5ever_rcdom = "0.39"
mime = "0.3.17"
mime_guess = "2.0.5"
minify-html = { git = "https://github.com/bjones1/minify-html.git", branch = "dev", version = "0.18.1" }
minreq = { version = "3", features = ["https-rustls-probe", "proxy"] }
normalize-line-endings = "0.3.0"
+notify-debouncer-full = "0.7.0"
path-slash = "0.2.1"
pest = "2.7.14"
pest_derive = "2.7.14"
diff --git a/server/src/ide/vscode/tests.rs b/server/src/ide/vscode/tests.rs
index f0553cfd..a7063a1c 100644
--- a/server/src/ide/vscode/tests.rs
+++ b/server/src/ide/vscode/tests.rs
@@ -59,7 +59,7 @@ use crate::{
use crate::{
processing::{
CodeChatForWeb, CodeMirror, CodeMirrorDiff, CodeMirrorDiffable, CodeMirrorDocBlock,
- CodeMirrorDocBlockTransaction, SourceFileMetadata, StringDiff,
+ CodeMirrorDocBlockTransaction, CodeMirrorDocBlockUpdate, SourceFileMetadata, StringDiff,
},
webserver::{ResultOkTypes, UpdateMessageContents, drop_leading_slash},
};
@@ -1442,6 +1442,673 @@ async fn test_vscode_ide_websocket9() {
temp_dir.close().unwrap();
}
+/// Verify that the Server re-translates the current file when a change to
+/// another file on disk outdates it. The current file cross-references an id
+/// which no file defines until the test rewrites the TOC on disk to define it.
+/// The project's watcher reports this change, so its consumer re-translates the
+/// TOC, outdating the current file.
+#[actix_web::test]
+async fn test_vscode_ide_websocket10() {
+ let connection_id = "test-connection-id10";
+ let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await;
+ open_client(&mut ws_ide).await;
+
+ // Load the current file, following the protocol in
+ // `test_vscode_ide_websocket4`.
+ let file_path_temp = fs::canonicalize(test_dir.join("test.py")).unwrap();
+ let file_path = simplified(&file_path_temp);
+ let file_path_str = file_path.to_str().unwrap().to_string();
+ send_message(
+ &mut ws_client,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::CurrentFile(
+ format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}",
+ file_path.to_slash().unwrap()
+ ),
+ None,
+ ),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_ide).await,
+ EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::CurrentFile(file_path_str.clone(), Some(true))
+ }
+ );
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void))
+ }
+ );
+
+ // Fetch the current file, which the IDE doesn't have, so the Server reads
+ // it from disk.
+ let test_dir_thread = test_dir.clone();
+ let join_handle = thread::spawn(move || {
+ assert_eq!(
+ minreq::get(format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}/test.py",
+ test_dir_thread.to_slash().unwrap(),
+ ))
+ .send()
+ .unwrap()
+ .status_code,
+ 200
+ );
+ });
+ let em = read_message(&mut ws_ide).await;
+ cast!(em.message, EditorMessageContents::LoadFile, a, b);
+ assert_eq!(em.id, INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT);
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))),
+ },
+ )
+ .await;
+ join_handle.join().unwrap();
+
+ // No file defines `foo` yet, so the cross-reference can't be resolved.
+ // Lacking a version from the IDE, the Server keeps its initial version.
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Update(UpdateMessageContents {
+ file_path: file_path_str.clone(),
+ cursor_position: None,
+ scroll_position: None,
+ is_re_translation: false,
+ contents: Some(CodeChatForWeb {
+ metadata: SourceFileMetadata {
+ mode: "python".to_string(),
+ },
+ source: CodeMirrorDiffable::Plain(CodeMirror {
+ doc: "\n".to_string(),
+ doc_blocks: vec![CodeMirrorDocBlock {
+ from: 0,
+ to: 1,
+ indent: String::new(),
+ delimiter: "#".to_string(),
+ contents: r#"See id "foo" not found."#.to_string(),
+ }],
+ }),
+ version: 0.0,
+ }),
+ }),
+ }
+ );
+ send_message(
+ &mut ws_client,
+ &EditorMessage {
+ id: INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_ide).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ }
+ );
+
+ // Define `foo` by rewriting the TOC on disk. After its debounce (2 s,
+ // checked every 0.5 s, so within `read_message`'s timeout), the watcher
+ // reports this change, so the consumer re-translates the TOC.
+ //
+ // The consumer's initial scan of the project may reach the TOC on either
+ // side of this write; either way, only one commit defines `foo`, outdating
+ // the current file once. A scan before the write finds no ids in the TOC.
+ // A scan after it commits the new TOC, which makes the watcher's report a
+ // no-op: the TOC's metadata then matches the cache's, so the consumer skips
+ // it.
+ fs::write(test_dir.join("toc.md"), "
Foo
\n").unwrap();
+
+ // Translating the TOC outdates the current file, which the Server
+ // re-translates, sending the Client the resolved cross-reference. The
+ // source is unchanged, so the version is too.
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 3.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Update(UpdateMessageContents {
+ file_path: file_path_str.clone(),
+ cursor_position: None,
+ scroll_position: None,
+ is_re_translation: true,
+ contents: Some(CodeChatForWeb {
+ metadata: SourceFileMetadata {
+ mode: "python".to_string(),
+ },
+ source: CodeMirrorDiffable::Diff(CodeMirrorDiff {
+ doc: vec![],
+ doc_blocks: vec![CodeMirrorDocBlockTransaction::Update(
+ CodeMirrorDocBlockUpdate {
+ from: 0,
+ from_new: None,
+ to: None,
+ indent: None,
+ delimiter: None,
+ contents: vec![StringDiff {
+ from: 0,
+ to: Some(97),
+ insert: "See Foo.".to_string(),
+ }],
+ }
+ )],
+ version: 0.0,
+ }),
+ version: 0.0,
+ }),
+ }),
+ }
+ );
+ send_message(
+ &mut ws_client,
+ &EditorMessage {
+ id: INITIAL_MESSAGE_ID + 3.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_ide).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 3.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ }
+ );
+
+ check_logger_errors(0);
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+}
+
+/// Have the Client acknowledge a message from the Server, then verify that the
+/// Server forwards the acknowledgement to the IDE.
+async fn acknowledge(
+ // The Client's websocket.
+ ws_client: &mut WebSocketStream,
+ // The IDE's websocket.
+ ws_ide: &mut WebSocketStream,
+ // The id of the message to acknowledge.
+ id: f64,
+) {
+ let result = EditorMessage {
+ id,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ };
+ send_message(ws_client, &result).await;
+ assert_eq!(read_message(ws_ide).await, result);
+}
+
+/// Fetch the TOC as the Client's sidebar does. The IDE reports that it doesn't
+/// have the TOC, so the Server reads it from disk.
+async fn fetch_sidebar_toc(
+ // The IDE's websocket.
+ ws_ide: &mut WebSocketStream,
+ // The connection whose Client fetches the TOC.
+ connection_id: &str,
+ // The directory containing the TOC.
+ test_dir: &Path,
+ // The id of the `LoadFile` message the Server sends to the IDE.
+ load_file_id: f64,
+ // The body of the response: the page showing the TOC.
+) -> String {
+ let url = format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}/toc.md?mode=toc",
+ test_dir.to_slash().unwrap()
+ );
+ let join_handle = thread::spawn(move || {
+ let response = minreq::get(url).send().unwrap();
+ assert_eq!(response.status_code, 200);
+ response.as_str().unwrap().to_string()
+ });
+ let em = read_message(ws_ide).await;
+ let (_path, is_current) = cast!(
+ em.message,
+ EditorMessageContents::LoadFile,
+ path,
+ is_current
+ );
+ assert!(!is_current);
+ assert_eq!(em.id, load_file_id);
+ send_message(
+ ws_ide,
+ &EditorMessage {
+ id: load_file_id,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))),
+ },
+ )
+ .await;
+ join_handle.join().unwrap()
+}
+
+/// Build an `Update` carrying the full text of a file, as the IDE sends it.
+fn ide_update(
+ // The file's path.
+ file_path: &str,
+ // The file's lexer.
+ mode: &str,
+ // The file's new text.
+ doc: &str,
+ // The file's new version.
+ version: f64,
+ // The `Update` message.
+) -> EditorMessageContents {
+ EditorMessageContents::Update(UpdateMessageContents {
+ file_path: file_path.to_string(),
+ cursor_position: None,
+ scroll_position: None,
+ is_re_translation: false,
+ contents: Some(CodeChatForWeb {
+ metadata: SourceFileMetadata {
+ mode: mode.to_string(),
+ },
+ source: CodeMirrorDiffable::Plain(CodeMirror {
+ doc: doc.to_string(),
+ doc_blocks: vec![],
+ }),
+ version,
+ }),
+ })
+}
+
+/// Verify that the Server refreshes the TOC in the Client's sidebar when its
+/// project's cache reports the TOC outdated -- the TOC cross-references a
+/// heading in the current file, which the IDE then renames -- or changed on
+/// disk.
+#[actix_web::test]
+async fn test_vscode_ide_websocket11() {
+ let connection_id = "test-connection-id11";
+ let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await;
+ open_client(&mut ws_ide).await;
+
+ // Load the current file, following the protocol in
+ // `test_vscode_ide_websocket10`.
+ let file_path_temp = fs::canonicalize(test_dir.join("test.py")).unwrap();
+ let file_path = simplified(&file_path_temp);
+ let file_path_str = file_path.to_str().unwrap().to_string();
+ send_message(
+ &mut ws_client,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::CurrentFile(
+ format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}",
+ file_path.to_slash().unwrap()
+ ),
+ None,
+ ),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_ide).await,
+ EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::CurrentFile(file_path_str.clone(), Some(true))
+ }
+ );
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void))
+ }
+ );
+ let test_dir_thread = test_dir.clone();
+ let join_handle = thread::spawn(move || {
+ assert_eq!(
+ minreq::get(format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}/test.py",
+ test_dir_thread.to_slash().unwrap(),
+ ))
+ .send()
+ .unwrap()
+ .status_code,
+ 200
+ );
+ });
+ let em = read_message(&mut ws_ide).await;
+ cast!(em.message, EditorMessageContents::LoadFile, a, b);
+ assert_eq!(em.id, INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT);
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))),
+ },
+ )
+ .await;
+ join_handle.join().unwrap();
+ let em = read_message(&mut ws_client).await;
+ cast!(em.message, EditorMessageContents::Update);
+ assert_eq!(em.id, INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT);
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+
+ // The sidebar's TOC shows the heading's text.
+ let toc_page = fetch_sidebar_toc(
+ &mut ws_ide,
+ connection_id,
+ &test_dir,
+ INITIAL_MESSAGE_ID + 3.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+ assert_contains!(toc_page, "Foo");
+
+ // Rename the heading in the IDE. The Client receives the edit...
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_IDE_MESSAGE_ID,
+ message: ide_update(&file_path_str, "python", "# Bar
\n", 1.0),
+ },
+ )
+ .await;
+ let em = read_message(&mut ws_client).await;
+ cast!(em.message, EditorMessageContents::Update);
+ assert_eq!(em.id, INITIAL_IDE_MESSAGE_ID);
+ acknowledge(&mut ws_client, &mut ws_ide, INITIAL_IDE_MESSAGE_ID).await;
+
+ // ...then, since the edit outdated the TOC, the TOC's new rendering.
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 4.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::UpdateToc(
+ ""
+ .to_string()
+ ),
+ }
+ );
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_MESSAGE_ID + 4.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+
+ // Change the TOC on disk. Once the cache's watcher reports this, the Server
+ // re-loads the TOC, asking the IDE for it first...
+ fs::write(
+ test_dir.join("toc.md"),
+ "* \n* More\n",
+ )
+ .unwrap();
+ let em = read_message(&mut ws_ide).await;
+ let (path, is_current) = cast!(
+ em.message,
+ EditorMessageContents::LoadFile,
+ path,
+ is_current
+ );
+ assert_eq!(path.file_name().unwrap(), "toc.md");
+ assert!(!is_current);
+ assert_eq!(em.id, INITIAL_MESSAGE_ID + 5.0 * MESSAGE_ID_INCREMENT);
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_MESSAGE_ID + 5.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))),
+ },
+ )
+ .await;
+ // ...then, since the IDE doesn't have it, from disk.
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 6.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::UpdateToc(
+ ""
+ .to_string()
+ ),
+ }
+ );
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_MESSAGE_ID + 6.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+
+ check_logger_errors(0);
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+}
+
+/// Verify that editing the TOC, from either the IDE or the Client, refreshes
+/// the TOC in the Client's sidebar, unless the edit leaves its rendering
+/// unchanged.
+#[actix_web::test]
+async fn test_vscode_ide_websocket12() {
+ let connection_id = "test-connection-id12";
+ let (temp_dir, test_dir, mut ws_ide, mut ws_client) = prep_test!(connection_id).await;
+ open_client(&mut ws_ide).await;
+
+ // Make the TOC the current file, following the protocol in
+ // `test_vscode_ide_websocket10`.
+ let file_path_temp = fs::canonicalize(test_dir.join("toc.md")).unwrap();
+ let file_path = simplified(&file_path_temp);
+ let file_path_str = file_path.to_str().unwrap().to_string();
+ send_message(
+ &mut ws_client,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::CurrentFile(
+ format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}",
+ file_path.to_slash().unwrap()
+ ),
+ None,
+ ),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_ide).await,
+ EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::CurrentFile(file_path_str.clone(), Some(true))
+ }
+ );
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void))
+ }
+ );
+ let test_dir_thread = test_dir.clone();
+ let join_handle = thread::spawn(move || {
+ assert_eq!(
+ minreq::get(format!(
+ "http://localhost:{IP_PORT}/vsc/fs/{connection_id}/{}/toc.md",
+ test_dir_thread.to_slash().unwrap(),
+ ))
+ .send()
+ .unwrap()
+ .status_code,
+ 200
+ );
+ });
+ let em = read_message(&mut ws_ide).await;
+ let (_path, is_current) = cast!(
+ em.message,
+ EditorMessageContents::LoadFile,
+ path,
+ is_current
+ );
+ assert!(is_current);
+ assert_eq!(em.id, INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT);
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::LoadFile(None))),
+ },
+ )
+ .await;
+ join_handle.join().unwrap();
+ let em = read_message(&mut ws_client).await;
+ cast!(em.message, EditorMessageContents::Update);
+ assert_eq!(em.id, INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT);
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_MESSAGE_ID + 2.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+ let toc_page = fetch_sidebar_toc(
+ &mut ws_ide,
+ connection_id,
+ &test_dir,
+ INITIAL_MESSAGE_ID + 3.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+ assert_contains!(toc_page, "Contents
");
+
+ // Edit the TOC in the IDE. The Client receives the edit, then the sidebar's
+ // new rendering of it.
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_IDE_MESSAGE_ID,
+ message: ide_update(&file_path_str, "markdown", "# Contents\n\nMore\n", 1.0),
+ },
+ )
+ .await;
+ let em = read_message(&mut ws_client).await;
+ cast!(em.message, EditorMessageContents::Update);
+ assert_eq!(em.id, INITIAL_IDE_MESSAGE_ID);
+ acknowledge(&mut ws_client, &mut ws_ide, INITIAL_IDE_MESSAGE_ID).await;
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 4.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::UpdateToc("Contents
More".to_string()),
+ }
+ );
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_MESSAGE_ID + 4.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+
+ // An edit which doesn't change the TOC's rendering doesn't refresh the
+ // sidebar; otherwise, an `UpdateToc` would precede the one checked below.
+ send_message(
+ &mut ws_ide,
+ &EditorMessage {
+ id: INITIAL_IDE_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ message: ide_update(&file_path_str, "markdown", "# Contents\n\nMore\n\n", 2.0),
+ },
+ )
+ .await;
+ let em = read_message(&mut ws_client).await;
+ cast!(em.message, EditorMessageContents::Update);
+ assert_eq!(em.id, INITIAL_IDE_MESSAGE_ID + MESSAGE_ID_INCREMENT);
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_IDE_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ )
+ .await;
+
+ // Edit the TOC in the Client. The sidebar receives the new rendering, while
+ // the IDE receives the edit.
+ send_message(
+ &mut ws_client,
+ &EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Update(UpdateMessageContents {
+ file_path: file_path_str.clone(),
+ cursor_position: None,
+ scroll_position: None,
+ is_re_translation: false,
+ contents: Some(CodeChatForWeb {
+ metadata: SourceFileMetadata {
+ mode: "markdown".to_string(),
+ },
+ source: CodeMirrorDiffable::Plain(CodeMirror {
+ doc: "
Contents
More and more
".to_string(),
+ doc_blocks: vec![],
+ }),
+ version: 3.0,
+ }),
+ }),
+ },
+ )
+ .await;
+ assert_eq!(
+ read_message(&mut ws_client).await,
+ EditorMessage {
+ id: INITIAL_MESSAGE_ID + 5.0 * MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::UpdateToc(
+ "Contents
More and more".to_string()
+ ),
+ }
+ );
+ let em = read_message(&mut ws_ide).await;
+ cast!(em.message, EditorMessageContents::Update);
+ assert_eq!(em.id, INITIAL_CLIENT_MESSAGE_ID + MESSAGE_ID_INCREMENT);
+ let result = EditorMessage {
+ id: INITIAL_CLIENT_MESSAGE_ID + MESSAGE_ID_INCREMENT,
+ message: EditorMessageContents::Result(Ok(ResultOkTypes::Void)),
+ };
+ send_message(&mut ws_ide, &result).await;
+ assert_eq!(read_message(&mut ws_client).await, result);
+ acknowledge(
+ &mut ws_client,
+ &mut ws_ide,
+ INITIAL_MESSAGE_ID + 5.0 * MESSAGE_ID_INCREMENT,
+ )
+ .await;
+
+ check_logger_errors(0);
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+}
+
#[test]
fn test_find_eoltypes() {
assert_eq!(
diff --git a/server/src/lexer.rs b/server/src/lexer.rs
index 3ec01072..8c19da65 100644
--- a/server/src/lexer.rs
+++ b/server/src/lexer.rs
@@ -704,7 +704,7 @@ pub fn source_lexer(
let mut append_code_doc_block = |indent: &str, delimiter: &str, contents: &str| {
// Don't append empty entries.
if delimiter.is_empty() && contents.is_empty() {
- assert!(indent.is_empty());
+ assert_eq!(indent, "");
return;
}
// Define a line as any characters up to an including a newline. If the
diff --git a/server/src/processing.rs b/server/src/processing.rs
index be899c5e..bf23dbcc 100644
--- a/server/src/processing.rs
+++ b/server/src/processing.rs
@@ -79,10 +79,10 @@ use crate::{
CodeDocBlock, DocBlock, LEXERS, LanguageLexerCompiled, source_lexer,
supported_languages::MARKDOWN_MODE,
},
- processing::cache::{FileFacts, FragmentFact, IdResolution, TargetFact},
+ processing::cache::{FileFacts, FileStamp, FragmentFact, IdResolution, TargetFact},
webserver::canonicalize,
};
-use cache::{Cache, CacheMap};
+use cache::{Cache, CacheMap, get_or_insert_cache};
// Data structures
// ---------------
@@ -330,7 +330,7 @@ const DOC_BLOCK_SEPARATOR_REMOVE_FENCE: &str = r"
const DOC_BLOCK_SEPARATOR_MENDED_FENCE: &str =
"\n$1\n";
// The value of an `id` attribute which requests that the cache assign an id;
-// see `Auto-assignment of ids` in `cache.rs`.
+// see `Auto-assignment of ids` in the cache spec.
const AUTO_ID: &str = "*";
// The column at which to word wrap doc blocks.
const WORD_WRAP_COLUMN: usize = 80;
@@ -415,6 +415,33 @@ pub fn find_path_to_toc(file_path: &Path) -> Option {
}
}
+/// Return the cache of the project containing a file, creating it -- and the
+/// threads which keep it current -- if this project has none.
+pub fn project_cache(
+ // The map of project caches.
+ cache_map: &CacheMap,
+ // A file in the project.
+ file_path: &Path,
+ // The path to the project's TOC, relative to the directory containing
+ // `file_path`, as returned by `find_path_to_toc`.
+ path_to_toc: &Path,
+) -> Arc> {
+ // The project root is the directory containing `toc.md`. Canonicalize it so
+ // that every file in a project keys the same cache, and different projects
+ // key different caches.
+ let toc_path = file_path
+ .parent()
+ .unwrap_or_else(|| Path::new("."))
+ .join(path_to_toc);
+ let root = toc_path
+ .parent()
+ .map_or_else(PathBuf::new, Path::to_path_buf);
+ let root = canonicalize(&root).unwrap_or(root);
+ get_or_insert_cache(&mut cache_map.lock().unwrap(), &root, || {
+ Cache::new_project(root.clone())
+ })
+}
+
#[derive(Debug, thiserror::Error)]
pub enum CodechatForWebToSourceError {
#[error("invalid lexer {0}")]
@@ -913,14 +940,9 @@ pub fn source_to_codechat_for_web(
let cache = if let Some(project_cache) = cache {
project_cache
} else {
- // A non-project file uses a throwaway cache whose "project" is the
- // file's directory: only references within this file resolve.
- Arc::new(Mutex::new(Cache::new(
- file_path
- .parent()
- .unwrap_or_else(|| Path::new(""))
- .to_path_buf(),
- )))
+ // A non-project file uses a throwaway cache whose "project" is the file
+ // itself: only references within this file resolve.
+ Arc::new(Mutex::new(Cache::new(file_path.to_path_buf())))
};
let code_doc_block_arr;
let codechat_for_web = CodeChatForWeb {
@@ -1197,7 +1219,7 @@ static CLEAN_TARGET_HTML: LazyLock = LazyLock::new(|| {
b
});
-// Clean HTML for storage as a `Fragment`'s content, per the spec in `cache.rs`:
+// Clean HTML for storage as a `Fragment`'s content, per the cache spec:
// like `AMMONIA_OPTIONS`, but `id` and `data-gather` attributes are stripped
// (they aren't in any allowlist) to prevent duplicate ids when the content is
// rendered in a gather list, and `` elements are removed *along with
@@ -1269,26 +1291,9 @@ pub fn source_to_codechat_for_web_string(
// file by searching the current directory, then all its parents, for a file
// named `toc.md`.
let path_to_toc = find_path_to_toc(file_path);
- let cache: Option>> = path_to_toc.as_ref().map(|path_to_toc| {
- // The project root is the directory containing `toc.md`. `path_to_toc`
- // is relative to the file's directory; canonicalize the combination so
- // that every file in a project keys the same cache, and different
- // projects key different caches.
- let toc_path = file_path
- .parent()
- .unwrap_or_else(|| Path::new("."))
- .join(path_to_toc);
- let root = toc_path
- .parent()
- .map_or_else(PathBuf::new, Path::to_path_buf);
- let root = canonicalize(&root).unwrap_or(root);
- cache
- .lock()
- .unwrap()
- .entry(root.clone())
- .or_insert_with(|| Arc::new(Mutex::new(Cache::new(root.clone()))))
- .clone()
- });
+ let cache = path_to_toc
+ .as_ref()
+ .map(|path_to_toc| project_cache(cache, file_path, path_to_toc));
Ok((
match source_to_codechat_for_web(file_contents, file_path, version, is_toc, cache) {
@@ -1441,28 +1446,33 @@ fn get_dom_body(document: &Rc) -> Rc {
document.children.borrow()[0].children.borrow()[1].clone()
}
-// HTML produced from Markdown needs additional processing, termed hydration:
-//
-// * Transform math, Mermaid, GraphViz, etc. nodes.
-// * Hydrate cross-references, fragments, and gather elements from the project
-// cache.
-// * (Eventually) record document structure information.
-// * (Eventually) fill in autocomplete fields.
-//
-// Hydration is layered (see the `Design` section in `cache.rs`); the layers
-// which depend on fragment contents (gather lists) can only run after doc
-// blocks are finalized, so hydration is split into phases:
-//
-// 1. `hydrate_dom`: collect facts from the DOM, commit them to the cache, then
-// patch `` contents and `` backlinks.
-// 2. `store_fragment_contents`: once each doc block's hydrated HTML is known,
-// store each fragment's content in the cache.
-// 3. `hydrate_gathers`: insert each gather element's list of fragment contents,
-// now all present in the cache.
-//
-// This function runs all three phases for a Markdown document, where phase 2
-// needs no doc block data (fragments aren't allowed in Markdown documents, so
-// every fragment's content is an error message).
+/// hydrate_html
+///
+/// HTML produced from Markdown needs additional processing, termed hydration:
+///
+/// * Transform math, Mermaid, GraphViz, etc. nodes.
+/// * Hydrate cross-references, fragments, and gather elements from the project
+/// cache.
+/// * (Eventually) record document structure information.
+/// * (Eventually) fill in autocomplete fields.
+///
+/// Hydration is layered (see ); the layers
+/// which depend on fragment contents (gather lists) can only run after doc
+/// blocks are finalized, so hydration is split into phases:
+///
+/// 1. `hydrate_dom`: collect facts from the DOM, commit them to the cache, then
+/// patch `` contents and `` backlinks.
+/// 2. `store_fragment_contents`: once each doc block's hydrated HTML is known,
+/// store each fragment's content in the cache.
+/// 3. `hydrate_gathers`: insert each gather element's list of fragment contents,
+/// now all present in the cache.
+///
+/// Phases 1 and 2 update the cache, and each update may outdate other files;
+/// `Cache::mark_outdated` sees that every outdated file is re-rendered.
+///
+/// This function runs all three phases for a Markdown document, where phase 2
+/// needs no doc block data (fragments aren't allowed in Markdown documents, so
+/// every fragment's content is an error message).
fn hydrate_html(html: &str, file: &Path, cache: &Arc>) -> io::Result {
let (dom, walk_context) = hydrate_dom(html, file, cache, true)?;
store_fragment_contents(&walk_context, None, cache);
@@ -1488,20 +1498,22 @@ fn hydrate_dom(
// The parsed, patched DOM plus the walk results needed by later phases.
) -> io::Result<(Rc, WalkContext)> {
let dom = html_to_dom(html, None)?;
- // Read the file's metadata and canonicalize its path before taking the
- // cache lock, so that no I/O happens while the lock is held. The metadata
- // captures the state of the file whose content is being processed; the
- // canonicalization satisfies the cache's requirement that all paths are
- // canonicalized and absolute. (Canonicalization fails for files which don't
- // exist on disk -- such as tests -- in which case the path is used as-is.)
- let metadata = file.metadata().ok();
+ // Stamp the file and canonicalize its path before taking the cache lock, so
+ // that no I/O happens while the lock is held. The stamp captures the state
+ // of the file whose content is being processed (the cache's consumer, which
+ // read that content from the file itself, replaces this with the stamp it
+ // took beforehand; see `scan_file` in `cache.rs`); the canonicalization
+ // satisfies the cache's requirement that all paths are canonicalized and
+ // absolute. (Canonicalization fails for files which don't exist on disk --
+ // such as tests -- in which case the path is used as-is.)
+ let stamp = file.metadata().ok().as_ref().map(FileStamp::new);
let path = canonicalize(file).unwrap_or_else(|_| file.to_path_buf());
// ### Assign requested ids
//
// Replace each `id="*"` with a freshly generated id before facts are
// collected, so that everything downstream sees a real id. These ids reach
// the Client but are deliberately kept out of the cache until this file is
- // written and re-read; see `Auto-assignment of ids` in `cache.rs`.
+ // written and re-read; see `Auto-assignment of ids` in the cache spec.
let mut auto_ids: HashSet = HashSet::new();
{
let cache_guard = cache.lock().unwrap();
@@ -1512,7 +1524,8 @@ fn hydrate_dom(
// Walk the DOM, collecting all cacheable facts -- targets,
// cross-references, fragments, and gather elements -- without touching the
// cache. This keeps the non-`Send` DOM types out of the cache and off its
- // critical section; see the design discussion in `cache.rs`.
+ // critical section. The walk skips the content the cache generates (see
+ // `hydrating_walk_node`), so hydration never feeds back into the facts.
let mut walk_context = WalkContext {
facts: FileFacts::default(),
path,
@@ -1532,22 +1545,20 @@ fn hydrate_dom(
// diffs them against the file's previous state: added, deleted, and
// modified targets and fragments mark the files which depend on them as
// outdated, this file is linked to every id it references, and duplicate
- // ids are recorded.
+ // ids are recorded. The commit also sees that each outdated file is
+ // re-rendered; see `Cache::mark_outdated`.
let mut cache_guard = cache.lock().unwrap();
- // TODO: schedule reprocessing for each file in `commit.outdated`.
- let _commit = cache_guard.commit_file(
+ cache_guard.commit_file(
&walk_context.path,
- metadata,
+ stamp,
mem::take(&mut walk_context.facts),
);
// ### Patch cross-references
//
// Layer 2: each ``'s content depends only on its target's inner HTML,
// which the commit just placed in the cache.
- for (id, node) in &walk_context.xrefs {
- set_attr(node, "contenteditable", "false");
- let children = xref_content(&cache_guard, id, &walk_context.path)?;
- set_element_children(node, children);
+ for node in &walk_context.xrefs {
+ hydrate_xref(&cache_guard, node, &walk_context.path)?;
}
// ### Patch fragment backlinks
//
@@ -1583,9 +1594,9 @@ fn hydrate_dom(
}
/// Replace every `id="*"` -- a request that the cache assign an id -- with a
-/// freshly generated one, recording the ids assigned. Per the spec in
-/// `cache.rs`, these ids are placed in the HTML but not recorded in the cache;
-/// the fact collection walk uses the recorded set to skip them.
+/// freshly generated one, recording the ids assigned. Per the cache spec, these
+/// ids are placed in the HTML but not recorded in the cache; the fact
+/// collection walk uses the recorded set to skip them.
fn assign_auto_ids(
// The node whose descendants are scanned.
node: &Rc,
@@ -1615,23 +1626,86 @@ fn assign_auto_ids(
}
}
+/// Hydrate one ``: mark it non-editable, then replace its contents with a
+/// link to its target or an error message.
+fn hydrate_xref(
+ // The cache, already locked by the caller.
+ cache: &Cache,
+ // The `` element.
+ node: &Rc,
+ // The canonicalized path of the file the `` is rendered in, from
+ // which its link leads.
+ path: &Path,
+ // An error only if the target's stored inner HTML can't be parsed.
+) -> io::Result<()> {
+ set_attr(node, "contenteditable", "false");
+ let children = xref_content(cache, node, path)?;
+ set_element_children(node, children);
+ Ok(())
+}
+
+/// Re-hydrate every `` within the given nodes, at any depth, which
+/// hydrated to a link; see `hydrate_gathers` for why gathered content needs
+/// this. Only a link depends on the file it's rendered in, so one which
+/// hydrated to an error message is left as it is. This matters for an ``
+/// with an `id`: cleaning a fragment's content strips every `id`, so only the
+/// fragment's own file can detect that error.
+fn hydrate_xrefs_within(
+ // The cache, already locked by the caller.
+ cache: &Cache,
+ // The nodes to search.
+ nodes: &[Rc],
+ // The canonicalized path of the file these nodes are rendered in.
+ path: &Path,
+ // An error only if a target's stored inner HTML can't be parsed.
+) -> io::Result<()> {
+ for node in nodes {
+ if get_node_tag_name(node) == Some("xref") {
+ let is_error = node
+ .children
+ .borrow()
+ .iter()
+ .any(|child| has_class(child, "cc-error"));
+ if !is_error {
+ hydrate_xref(cache, node, path)?;
+ }
+ } else {
+ hydrate_xrefs_within(cache, &node.children.borrow(), path)?;
+ }
+ }
+ Ok(())
+}
+
/// The content of one hydrated ``: a link to its target, or an error
/// message.
fn xref_content(
// The cache, already locked by the caller.
cache: &Cache,
- // The id the cross-reference names.
- id: &str,
- // The canonicalized path of the file containing the cross-reference.
+ // The `` element.
+ node: &Rc,
+ // The canonicalized path of the file the cross-reference is rendered in.
path: &Path,
// The nodes to place inside the `` element.
) -> io::Result>> {
- if !is_css_identifier(id) {
+ // An `` is never a target, so an `id` on it -- even `id="*"`, which
+ // `assign_auto_ids` leaves alone -- would mislead its reader into thinking
+ // it can be referenced.
+ if get_attr_value(node, "id").is_some() {
+ return Ok(vec![error_span(
+ "a cross-reference may not have an \"id\" attribute",
+ )]);
+ }
+ let Some(id) = get_attr_value(node, "ref") else {
+ return Ok(vec![error_span(
+ "a cross-reference requires a \"ref\" attribute",
+ )]);
+ };
+ if !is_css_identifier(&id) {
return Ok(vec![error_span(&format!(
"\"{id}\" is not a valid CSS identifier"
))]);
}
- Ok(match cache.resolve_id(id) {
+ Ok(match cache.resolve_id(&id) {
IdResolution::Target {
path: target_path,
target,
@@ -1698,8 +1772,14 @@ fn store_fragment_contents(
cache: &Arc>,
) {
let mut cache_guard = cache.lock().unwrap();
+ // The ids whose content this pass has stored. Duplicates within this file
+ // share one entry in the cache, which holds the first in document order
+ // (see `Cache::commit_file`); storing each duplicate's content in turn
+ // would instead change that entry twice per pass, outdating every file
+ // which gathers the id each time this file is processed.
+ let mut stored_ids: HashSet<&str> = HashSet::new();
for fragment in &walk_context.fragments {
- if !fragment.cached {
+ if !fragment.cached || !stored_ids.insert(&fragment.id) {
continue;
}
let content = if let Some(message) = &fragment.error {
@@ -1711,26 +1791,24 @@ fn store_fragment_contents(
// `error` set during the walk.
error_html("fragments are not allowed in Markdown documents")
};
- // TODO: schedule reprocessing for each outdated file returned here.
- let _outdated =
- cache_guard.update_fragment_content(&walk_context.path, &fragment.id, content);
+ cache_guard.update_fragment_content(&walk_context.path, &fragment.id, content);
}
}
/// Render one fragment's content: the hydrated HTML of the code/doc blocks it
-/// encloses, cleaned per the spec in `cache.rs`. Per that spec, the rendering
+/// encloses, cleaned per the cache spec. Per that spec, the rendering
/// reproduces the layout of the source it came from: each doc block carries the
-/// indent it had there, each line of a code block and the first line of each doc
-/// block are preceded by their line numbers, and equal indents in the source
-/// line up in the rendering.
+/// indent it had there, each line of a code block and the first line of each
+/// doc block are preceded by their line numbers, and equal indents in the
+/// source line up in the rendering.
///
/// ### Why a doc block's indent and a code block are ``s
///
/// The whitespace in those two *is* the layout, so everything which rewrites
-/// this HTML on its way to the screen must be told to leave it alone -- and both
-/// TinyMCE and the minifier key whitespace sensitivity on the tag name, with no
-/// class- or selector-based equivalent. (`CodeChat-doc-indent` needs no such
-/// treatment: the Client builds it outside the TinyMCE editable region, so
+/// this HTML on its way to the screen must be told to leave it alone -- and
+/// both TinyMCE and the minifier key whitespace sensitivity on the tag name,
+/// with no class- or selector-based equivalent. (`CodeChat-doc-indent` needs no
+/// such treatment: the Client builds it outside the TinyMCE editable region, so
/// nothing rewrites it.) Of the tag names both tools treat as
/// whitespace-sensitive by default, `pre` is the one which fits, and each tool
/// extends that treatment to everything nested inside it, covering the line
@@ -1811,8 +1889,8 @@ fn render_fragment_content(
// to line up with the code around it. Only the doc block's
// first line is numbered -- the block's remaining source
// lines have no fixed correspondence to the lines its
- // rendered contents occupy, since the comment delimiters are
- // gone and the text reflows.
+ // rendered contents occupy, since the comment delimiters
+ // are gone and the text reflows.
content.push_str(
"",
);
@@ -1912,7 +1990,17 @@ fn hydrate_gathers(
new_text(":"),
],
));
- items.extend(parse_html_fragment(&fragment.content)?);
+ // The fragment's content was hydrated in the fragment's
+ // file, so each cross-reference within it links from
+ // there; re-hydrate them to link from this file instead.
+ // This file needn't depend on their targets: the
+ // fragment's stored content embeds everything about a
+ // target which this re-hydration reads, so any change to
+ // a target which alters it also changes that content,
+ // which outdates this file.
+ let content = parse_html_fragment(&fragment.content)?;
+ hydrate_xrefs_within(&cache_guard, &content, &walk_context.path)?;
+ items.extend(content);
}
IdResolution::Target { .. } => items.push(error_span(&format!(
"id \"{id}\" names a target, not a fragment"
@@ -2175,9 +2263,9 @@ struct WalkContext {
/// These are excluded from `facts`: they belong to the cache only after the
/// file carrying them is written and re-read.
auto_ids: HashSet,
- /// Each cross-reference found: its destination id and its `` node,
- /// kept so the generated contents can be patched after the cache commit.
- xrefs: Vec<(String, Rc)>,
+ /// Each cross-reference's `` node, kept so the generated contents
+ /// can be patched after the cache commit.
+ xrefs: Vec>,
/// Each fragment found, with everything the later hydration phases need.
fragments: Vec,
/// DOM for all gather elements, i.e. targets with a non-empty
@@ -2200,7 +2288,7 @@ struct WalkContext {
struct FragmentHydration {
/// The fragment's DOM node.
node: Rc,
- /// The fragment's id.
+ /// The fragment's id; empty if it has none.
id: String,
/// The index of the fragment's first code/doc block; see
/// `FragmentFact::doc_block_start_index`.
@@ -2209,13 +2297,14 @@ struct FragmentHydration {
/// number of blocks in the document; see
/// `FragmentFact::code_doc_block_end_index`.
end: usize,
- /// When set, the fragment is in an error state detected during the walk
- /// (fragment in a Markdown document, or an unparsable `following`
- /// attribute): the message replaces both the hydrated tag content and the
- /// cached fragment content.
+ /// When set, the fragment is in an error state detected during the walk (a
+ /// missing or invalid id, a fragment in a Markdown document, or an
+ /// unparsable `following` attribute): the message replaces both the
+ /// hydrated tag content and the cached fragment content.
error: Option,
/// False when the fragment wasn't recorded in `FileFacts` (its id is
- /// invalid), so no content may be stored for it.
+ /// missing, invalid, or auto-assigned during this pass), so no content may
+ /// be stored for it.
cached: bool,
}
@@ -2288,18 +2377,18 @@ fn hydrating_walk_node(node: &Rc, mut walk_context: WalkContext) -> io::Re
} else if tag_name == "xref" {
// A cross reference: record the referenced id as a fact, and
// keep the node so its generated contents can be filled in
- // after the cache commit. Note that this element doesn't allow
- // an `id` attribute, so it's never a target.
+ // after the cache commit -- with an error message, if it lacks
+ // a `ref`. Note that this element is never a target; an `id`
+ // attribute on it is an error, reported by `xref_content`.
if let Some(ref_id) = get_attr_value(child, "ref") {
- walk_context.facts.xrefs.push(ref_id.clone());
- walk_context.xrefs.push((ref_id, child.clone()));
+ walk_context.facts.xrefs.push(ref_id);
}
+ walk_context.xrefs.push(child.clone());
} else if tag_name == "fragment" {
- // A fragment; without an id it's meaningless, so it's ignored.
- // Its content can't be determined yet -- doc blocks aren't
- // finalized during the walk -- so the caller stores it later
- // via `Cache::update_fragment_content`.
- if let Some(id) = id {
+ // A fragment. Its content can't be determined yet -- doc blocks
+ // aren't finalized during the walk -- so the caller stores it
+ // later via `Cache::update_fragment_content`.
+ if let Some(id) = id.filter(|id| !id.is_empty()) {
// The `following` attribute selects how many code/doc
// blocks after the current doc block the fragment encloses;
// the default is 1.
@@ -2339,8 +2428,8 @@ fn hydrating_walk_node(node: &Rc, mut walk_context: WalkContext) -> io::Re
walk_context.facts.fragments.push(FragmentFact {
id: id.clone(),
line: 0,
- doc_block_start_index: walk_context.doc_block_index,
- code_doc_block_end_index: walk_context
+ _doc_block_start_index: walk_context.doc_block_index,
+ _code_doc_block_end_index: walk_context
.doc_block_index
.saturating_add(following),
});
@@ -2353,6 +2442,18 @@ fn hydrating_walk_node(node: &Rc, mut walk_context: WalkContext) -> io::Re
error,
cached,
});
+ } else {
+ // A fragment without an id: display an error requesting the
+ // missing id, per the cache spec. Lacking an id, it
+ // can't be cached.
+ walk_context.fragments.push(FragmentHydration {
+ node: child.clone(),
+ id: String::new(),
+ start: walk_context.doc_block_index,
+ end: walk_context.doc_block_index,
+ error: Some("a fragment requires an \"id\" attribute".to_string()),
+ cached: false,
+ });
}
} else if let Some(id) = id
&& !id.is_empty()
@@ -2380,7 +2481,7 @@ fn hydrating_walk_node(node: &Rc, mut walk_context: WalkContext) -> io::Re
id,
// Clean the inner HTML for storage; note that it's
// captured before any hydration of descendants, per
- // the spec in `cache.rs`.
+ // the cache spec.
inner_html: CLEAN_TARGET_HTML
.clean(&node_inner_html(child)?)
.to_string(),
@@ -2388,7 +2489,7 @@ fn hydrating_walk_node(node: &Rc, mut walk_context: WalkContext) -> io::Re
// pulldown-cmark HTML writer preserves them; see
// the TODO in `cache.rs`.
line: 0,
- doc_block_index: walk_context.doc_block_index,
+ _doc_block_index: walk_context.doc_block_index,
gather_ids,
});
}
@@ -2403,7 +2504,7 @@ fn hydrating_walk_node(node: &Rc, mut walk_context: WalkContext) -> io::Re
}
} else if get_attr_value(child, "data-gather").is_some() {
// `data-gather` on an element without an id: display an error
- // requesting the missing id, per the spec in `cache.rs`.
+ // requesting the missing id, per the cache spec.
walk_context.gathers_error.push((
child.clone(),
"a gather element requires an \"id\" attribute".to_string(),
diff --git a/server/src/processing/cache-spec.md b/server/src/processing/cache-spec.md
index 0906ea59..a0dc014a 100644
--- a/server/src/processing/cache-spec.md
+++ b/server/src/processing/cache-spec.md
@@ -2,11 +2,11 @@ Cache spec
==========
The cache stores the location (file name and ID) and contents (inner HTML or
-code/doc blocks for fragments) of a target. Targets are HTML elements (excluding
-``s) with an ID.
+code/doc blocks for fragments) of targets and fragments. Targets are HTML
+elements (excluding ``s and ``s) with an ID.
The goal of the cache is to support cross-references and gather elements, and to
-ensure that all IDs are unique within a project. This means that
+assist in ensuring that all IDs are unique within a project. This means that
cross-references and gather elements persist across moving or renaming files,
since the IDs will be found in the cache.
@@ -17,7 +17,7 @@ do not.
The cache reflects data read directly from disk/IDE; content edited in the
Client does not update the cache until it's written to disk/IDE, at which point
-the cache much re-process this unknown file. Since the Client is designed around
+the cache must re-process the file. Since the Client is designed around
an autosave principle which updates disk/IDE regularly, there's little gap
between the two.
@@ -28,35 +28,41 @@ Cross references
----------------
A `` is a cross reference to a `Target` or a gather
-element. The `id` specifies the destination; the cache then hydrates the
-contents based on the location and contents of the target of the provided `id`
-to e.g. `Inner HTML from target`; this hydrated form
is only present in the Client.
Details:
-* This element does not allow an `id` attribute.
+* This element allows no attribute except for `ref`. Since an `` is
+ never a `Target`, an `id` attribute on it -- including `id="*"` -- would
+ mislead, so it replaces the contents with an error message.
* The inner HTML is always taken before cache hydration, to prevent circular
- dependencies:
+ dependencies (see also ):
```html
- See
- See
+ See
+ See
```
Including cache hydration would cause the inner HTML to be updated each time
the file is processed, outdating the other file.
-* If the `id` referred to isn't found or refers to a duplicate id, the inner
- text is instead an appropriate error message.
+* If the `ref` attribute is missing, the `id` referred to isn't found, refers to
+ a duplicate id, or is an invalid CSS identifier, the inner text is instead an
+ appropriate error message.
* If the cross reference is to a gather element, the text is the gather
element's inner HTML, not the gathered code/doc blocks.
+* Cross references to fragments are not allowed; this would create circular
+ dependencies (see ).
+
Fragments and gather elements
-----------------------------
@@ -68,7 +74,7 @@ contenteditable="false">See Bazzy things, Zappy things`. A fragment's
content by default includes the contents of the current doc block and the
contents of the following code/doc block; fragments are not allowed in Markdown
-documents (in the case, the fragment contents consist of an error message).
+documents (in this case, the fragment contents consist of an error message).
Fragments may include the `following` attribute to enclose a specific number of
the following code/doc blocks; for example, `` includes the current doc block along with the next 3
@@ -80,28 +86,31 @@ Details:
element list of contents will simply include an error message.
* Fragments do support indirection: gather element A includes contents from
fragment B, which contains a cross reference to target C. Changes to target C
- makes B and A outdated.
-* If a gather element refers to an `id` that is a `Target` or a `GatherElement`,
- not a `Fragment`, the resulting output for this in the list of fragments is an
- error message.
+ make B and A outdated.
+* If a gather element refers to an `id` that is a `Target`, not a `Fragment`,
+ the resulting output for this in the list of fragments is an error message.
* If a gather element refers to an `id` that wasn't found or is a duplicate, its
contents will be replaced by an error message.
* Fragments store an HTML rendering of the code and doc blocks they contain,
excluding the content produced by hydrating the `` tags, to avoid
- duplication and circular dependencies. See layer 4 under `Design`: this
- exclusion is what keeps a gather element and the fragments it lists from
- outdating each other forever. The HTML rendering of a fragment reproduces the
- layout of the source it came from: each doc block includes its indent, each
- line of a code block is preceded by that line's number, and the two are
- aligned -- a doc block and a line of code indented equally in the source begin
- in the same column, with the line numbers in a gutter of their own to the left
- of both.
+ duplication and circular dependencies. Cross-references in this rendering
+ are hydrated in the fragment's file, so their links lead from there; a
+ gather element re-hydrates each which hydrated to a link so that it leads
+ from the gather element's file instead. An error message reads the same
+ anywhere, so it's kept as is. See layer 4 under
+ : this exclusion is what keeps a gather
+ element and the fragments it lists from outdating each other forever. The HTML
+ rendering of a fragment reproduces the layout of the source it came from: each
+ doc block includes its indent, each line of a code block or the first line of
+ a doc block is preceded by that line's number, and the two are aligned -- a
+ doc block and a line of code indented equally in the source begin in the same
+ column, with the line numbers in a gutter of their own to the left of both.
* The backlinks a `` hydrates to are derived from the gather elements
which list it: for each such element, its containing file, its `id`, and its
inner HTML (the link text). A fragment's rendered output therefore depends on
the gather elements which reference it -- the reverse of the direction in
- which references are written; see `Design`.
-* A `` is valid; it contains only the current doc block.
+ which references are written; see .
+* A `` is valid; it contains only the current doc block.
* The `following` attribute is clamped if it would exceed the number of code/doc
blocks in the document.
* If the `following` value cannot be parsed to a whole number, an error message
@@ -109,10 +118,12 @@ Details:
* A gather element, as a type of `Target`, requires an `id`. A `data-gather`
attribute on an element without an `id` produces an error message which
requests the missing `id`.
+* Likewise, a fragment requires an id, producing an error message when the `id`
+ is missing.
* All ids must be valid
- [CSS identifiers](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/es/ident)
+ [CSS identifiers](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Values/ident)
per
- [MDN recommendations](https://developer.mozilla.org/en-US/docs/Web/HTML/rence/Global_attributes/id).
+ [MDN recommendations](https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Global_attributes/id).
Invalid ids produce error messages in the hydrated tag content.
Example hydration of the gather tag `Bazzy
@@ -132,37 +143,47 @@ things
`:
```
-Search
-------
+Search (future work)
+--------------------
-The cache supports searching the (cleaned) inner HTML of all `Target`s and
-gather elements; search does not include `Fragment` contents.
+The cache will support searching the (cleaned) inner HTML of all `Target`s and
+gather elements; search will not include `Fragment` contents.
-### Auto-assignment of ids
+Auto-assignment of ids
+----------------------
If `id="*"` on either a fragment, target, or gather element, the cache replaces
-this with an random autogenerated `id` placed in the resulting HTML contents,
-but this new `id` is not yet recorded in the cache. The file is then marked as
-`Unknown` when it is saved; when re-read, this `id` is then incorporated into
-the cache. This helps avoid cases where the cache and file contents become
-unsynchronized: if the `id` is placed in the cache before the write and the
-write fails, or if the file is never written (it was being scanned, but not
-actively edited, so the file contents wasn't written back).
+this with a random autogenerated `id` placed in the resulting HTML contents, but
+this new `id` is not yet recorded in the cache. It enters the cache only when
+source containing it is next translated: the Server translates each edit the
+Client saves back to the Client, to check for changes, which commits the edit;
+otherwise, the consumer reads the file once it's written to disk. This helps
+avoid cases where the cache and file contents become unsynchronized: if the `id`
+is placed in the cache before the write and the write fails, or if the file is
+never written (it was being scanned, but not actively edited, so the file
+contents weren't written back).
+
+TODO: an `id="*"` typed in the IDE reaches the source only when the Client sends
+back the HTML containing its replacement, which it does only once the user edits
+in the Client; until then, each translation of the IDE's text assigns a
+different id. Resolve this by sending the replacement to the IDE when a
+translation assigns ids.
The autogenerated id must be a valid
-[CSS identifier](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/es/ident).
+[CSS identifier](https://developer.mozilla.org/en-US/docs/Web/CSS/Reference/Values/ident).
Autogenerated ids must be checked to make sure they don't collide with an
existing id.
-Design
-------
+Cache design
The cache is a single plain-data structure, shared as an `Arc>`;
all consistency comes from that one lock, so no per-item locking (and therefore
no lock ordering) is needed. Items refer to each other by key -- files by path,
targets and fragments by id -- rather than by `Arc`/`Weak` pointers. This keeps
the structure acyclic, `Send`, and (in the future) serializable, and avoids
-garbage-collecting stale weak references.
+garbage-collecting stale weak references. Note that the watcher, consumer tx
+channel, and connections struct cache members aren't serializable, but can be
+re-created after deserialization.
Cached state always converges by design. Each layer depends only on
lower-numbered layers:
@@ -176,44 +197,57 @@ lower-numbered layers:
`id`, and inner HTML. Critically, this output is *excluded* from
`Fragment::content` (layer 3); were it included, a gather element and each
fragment it lists would outdate one another forever.
-5. Gather-list hydration depends on layers 1 and 3.
-
-Nothing reads layers 2, 4, or 5, so propagation terminates.
-
-Updating the cache is a two-phase process:
-
-1. Collect: while walking a file's DOM, record all cacheable facts
- (`FileFacts`) -- targets, cross-references, fragments, and gather elements --
- without touching the cache. This keeps the non-`Send` DOM types out of the
- cache and off its critical section. Fact collection ignores content expanded
- by the cache (the contents of `xref` and `fragment` tags; the content
- following a gather tag).
-2. Commit: `Cache::commit_file` applies the facts in one transaction, diffing
- them against the file's previous state to compute the set of files outdated
- by these changes.
-
-Item 2 requires the cache to track dependencies of an item, so that files
-containing these dependencies can be Outdated. Dependencies are tracked at file
-granularity, on the id rather than on the item defining it:
-`IdEntry::dependents` is the set of files whose rendered output depends on this
-id, by any means -- an `` or a `data-gather` list naming it. One
-uniform set suffices in all three `IdState`s because the typical action taken on
-a dependent is marking its containing file outdated (fragments also used their
-dependents to generate "See x" links).
-
-Item 2 also requires the cache to define what constitutes a difference which
-would trigger outdating dependencies. The state which is checked for differences
-is:
+5. Gather-list hydration depends on layers 1 and 3. It also re-hydrates each
+ `` within a fragment's content (layer 2) which hydrated to a link, so
+ that it links from the gathering file. That needs no dependency of its own: everything about a target which
+ this reads is embedded in the fragment's content, so any change to it also
+ changes that content, which outdates the gathering file.
+
+Only layers 1 and 3 are cached, and layer 3 depends only on layer 1 through
+layer 2. So a change travels at most two steps (C → B → A), which matches the
+indirection example, ensuring that propagation terminates.
+
+Supporting cross-references, fragments, and gather elements requires the cache
+to track dependents of an id, so that files containing these dependencies can be
+marked `Outdated` (see below); these are sent to the `Cache::consumer_tx`
+channel for processing. Dependencies are tracked at file granularity, on the id
+rather than on the item defining it: `IdEntry::dependents` is the set of files
+whose rendered output depends on this id, by any means -- an ``
+or a `data-gather` list naming it. One uniform set suffices in all three
+`IdState`s because the typical action taken on a dependent is marking this file
+outdated (fragments also use their dependents to generate "See x" links).
+
+Updating dependencies requires a definition of what constitutes a difference
+which would trigger outdating dependencies. The state which is checked for
+differences is:
* Target/gather element: type (a target/gather element), path of the containing
- file, id, IdState, inner HTML, and gather\_ids.
+ file, id, IdState, and inner HTML.
* Fragment: type (a fragment), path of the containing file, id, IdState, and
contents.
+A gather element's `gather_ids` are absent from this list because no dependent
+reads them: a cross-reference to a gather element renders only its inner HTML.
+The fragments a gather element lists do depend on them, through the reverse
+edge described below.
+
This also makes indirection (gather A includes fragment B, whose content
cross-references target C) work without extra machinery: a change to C outdates
B's file; reprocessing B's file changes B's content, which outdates A's file.
+Updating the cache is a multi-phase process; see
+. One output from this process (see
+) is a list of outdated files. For each
+outdated file, mark its status as `Outdated` (see
+) then:
+
+1. If the outdated file is the project TOC, then send `TocOutdated` to all
+ connections.
+2. If the outdated file is the current file for one or more connections, then
+ send `CurrentFileOutdated` to those connections.
+3. If neither 1 or 2 is true, the `Rescan(file)` message is sent to itself via
+ the consumer TX channel for later processing.
+
### Gather elements: the reverse edge
`dependents` alone is not enough for gather elements, because the reference runs
@@ -223,11 +257,11 @@ lists, and each of those fragments renders a *backlink* to the gather element
therefore not merely bookkeeping -- it is observable output in the fragment's
own file.
-This is why the outgoing references of gather elements cannot be maintained by
-the unlink-all/relink-all round trip used for cross-references (see
-`commit_file`): that round trip destroys the very information a diff would need.
-`commit_file` instead diffs `Target::gather_ids` explicitly, and applies this
-rule:
+This is why changes to the outgoing references of gather elements cannot be
+determined by the unlink-all/relink-all round trip used for cross-references
+(see `commit_file`): that round trip destroys the very information a diff would
+need. `commit_file` instead diffs `Target::gather_ids` explicitly, and applies
+this rule:
> When a gather element `G` is committed, outdate the file defining each id in
> the symmetric difference of `G`'s old and new `gather_ids`; if `G`'s inner
@@ -242,6 +276,8 @@ Propagation terminates here for the reason given under layer 4: rebuilding a
fragment's file updates its backlinks, but backlinks are excluded from
`Fragment::content`, so nothing further is outdated.
+### Missing and duplicate ids
+
An id referenced before (or without) being defined has an `IdEntry` whose state
is `IdState::Missing`; `dependents` holds the files waiting on it. When the id
later appears, the state transition marks those files outdated, and they simply
@@ -265,21 +301,182 @@ HTML stored as `Target` inner HTML or in `Fragment` contents requires cleaning:
* Inner HTML must allow only the
[permitted content for an `` element](https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/a#technical_summary).
-Watcher and walker
-------------------
-
-Need a watcher/walker (WW). Each time a project is found, the code must ask for
-the cache for this project path. The WW maintains a connection id -> project
-path mapping. If the current request doesn't change the mapping, simply return
-the existing cache. If the mapping changes, then update the WW thingy. The map
-must also be updated when a connection is closed (to remove an existing
-mapping).
-
-The WW thingy is another map from project path -> (cache, vec of project paths
-that are contained with this path, including the same path \[when multiple
-connections edit the same project\]). Operations:
-
-* Insert: to insert a new project path, walk all existing top-level project
- paths. If the new project path is contained within any of these, add it to the
- appropriate list. Otherwise, add a new entry.
-* Delete: find the path my looking fir
+Cache population and maintenance
+--------------------------------
+
+The cache must contain information about all renderable files in the project.
+The following design employs a file watcher to update the cache when files
+change, and a file walker to populate the cache initially and responds to
+directory changes.
+
+The `AppState` structure keeps an LRU map of project path -> cache (note that
+the cache includes a watcher and walker). Adding/removing connections resizes
+the LRU size to (connections \* `LRU_ENTRIES_PER_CONNECTION`) + 1; however, the
+cache for all current connections may not be dropped during an LRU eviction.
+This LRU-based approach seeks to balance cache memory consumption with the time
+required to load the cache for an uncached project.
+
+The cache fully supports multiple connections to the same project. It does not
+walk symlinks, since these could lead to two cache entries for the same file.
+There's no explicit support for nested projects.
+
+The cache initially registers a watcher with the provided project path and a
+consumer thread which receives results from the walker, along with a message
+channel connecting them.
+
+### Watcher/walker
+
+The watcher/walker thread monitors the root directory of the project and all
+children for changes using the
+[notify-debouncer-full](https://docs.rs/notify-debouncer-full/latest/notify_debouncer_full/)
+crate. At startup:
+
+1. It registers the watcher, to ensure that any changes during the walk are
+ still reported.
+ * The watcher sends file names of files that were created, modified, or
+ deleted and the path of created or deleted directories to the walker
+ encoded as `WatcherPathState`s via the walker channel; renames are treated
+ as a delete of the `from` path followed by a create of the `to` path.
+ * If the watcher encounters lost events (`EventKind::Other` with
+ `Flag::Rescan`), it requests a re-walk of the entire project.
+2. Even if the watcher fails, it then runs a filesystem walker loop rooted at
+ the project path, initially sending it the root directory to walk and
+ providing it the consumer channel and a second walker channel.
+
+A closed channel signals the watcher/walker thread to exit. Note that this
+approach means extraneous watcher events from ignored directories, such as
+`.git`, and additional use of OS resources such as inotify watches.
+
+#### Walker
+
+The walker is based on the [ignore](https://docs.rs/ignore/latest/ignore/)
+crate, allowing users to exclude files from the project. After the initial
+traverse of the project, it keeps
+an [IncrementalIgnore](https://docs.rs/ignore/latest/ignore/struct.IncrementalIgnore.html)
+for checking files in the `WatcherPathState`s received from the watcher.
+
+* If the path is a deleted file or directory and is not ignored by the walker,
+ it's sent directly to the consumer thread.
+* If the path refers to any of the walker's ignore files (per the
+ [ignore docs](https://docs.rs/ignore/latest/ignore/struct.WalkBuilder.html),
+ this is `.gitignore`, `.git/info/exclude`, and `.ignore`):
+ * The walker and incremental ignores are reloaded.
+ * For `.gitignore` and `.ignore`, the path containing this file is re-walked;
+ for `.git/info/exclude`, the entire tree is re-walked. For simplicity, this
+ occurs even outside a Git repository, or in subrepos, etc. TODO: also have
+ the watcher monitor global ignore files.
+* If the path is a file and is not ignored by the walker, it's sent to the
+ consumer thread.
+* If the path is a directory, it's walked; any files found are sent to the
+ consumer thread.
+
+Before performing a walk, the walker sends a `WalkStarted` to the consumer
+thread via the `Cache::consumer_tx` channel; when the walk is completed, it
+sends a `WalkFinished`.
+
+After processing a path, the walker then loops to wait for the next path. The
+thread ends when either channel is closed (indicating the cache was dropped). It
+ignores any errors produced during the walk, assuming that files it can't walk
+aren't usable by the IDE.
+
+### Consumer
+
+A consumer reads paths from the receiver paired to the `Cache::consumer_tx`
+channel. It holds a weak reference to the cache, and exits when this channel is
+closed or the weak reference can't be upgraded. It is implemented as a thread.
+
+The consumer decides whether a file changed by its stamp, as Git's index does: a
+snapshot of the file's length and modification time (plus its change time and
+inode on Unix), taken before its contents were read. A stamp taken within a few
+seconds of the file's last write never matches, since a later write in the same
+timestamp tick could leave this metadata unchanged (Git's "racy clean" problem).
+
+* If the file exists and doesn't match its cached stamp or if the file's status
+ is `Outdated`:
+
+ * If the file is the TOC and doesn't match its cached stamp, send
+ `TocChanged` to all connections once the processing below is done, even if
+ it ignores the file: each connection's sidebar renders the TOC from the
+ text it last loaded, which it must therefore re-load (see `Connections`
+ below).
+
+ * If the file is the current file for any of the cache's connections, ignore
+ it. This is handled by the standard IDE to Client translation mechanism.
+
+ * If not translatable, the file is still added to the cache, to prevent
+ reprocessing if the directory is re-walked.
+
+ * If translatable, it translates the file to the CodeChatForWeb format in
+ order to update the cache for that file. If the `SaveOnScan` option is
+ enabled, it then translates the file back to source code and writes the
+ result if there are any changes, updating the file's stamp in the cache
+ after the write. This costs additional CPU time and produces watcher events
+ and may rewrite files open in the IDE, motivating the choice of a
+ user-selected option.
+* If the path doesn't exist, search the cache for it; if found, remove it.
+ Otherwise, assume the path was a directory. Search the cache for all files in
+ this directory, removing them.
+
+* A `Rescan(file)` asks the consumer to bring a file the cache already has up to
+ date: one a commit outdated, or a connection's current file once no connection
+ shows it, since the consumer ignores current files. Unlike a path from the
+ walker, a `Rescan` hasn't passed the walker's ignore rules. It's therefore
+ skipped if the cache has since forgotten the file (a walk may have found it
+ newly ignored, for example), and it doesn't keep a walk in progress from
+ forgetting the file.
+
+* If this is a `WalkStarted`, put all files in the cache which this directory
+ contains in a set. Remove files produced by the walk from this set. After
+ `WalkFinished`, remove any files remaining in the set from the cache, since
+ these no longer exist.
+
+Neither kind of removal applies to a file a connection shows -- its current
+file, or the TOC -- since the connection's IDE may still hold the file's text.
+Instead, each connection whose current file it is re-translates it, and, if it's
+the TOC, every connection re-loads it.
+
+The cache doesn't record images or multimedia files referenced by a source file
+as dependencies, requiring the user to manually refresh the page when these
+files change.
+
+### Connections
+
+Every connection in a project also shows the project's TOC in its Client's
+sidebar, so whenever a commit marks the TOC outdated, the cache sends
+`TocOutdated` to every connection in the project. The connection re-renders the
+TOC from the text its sidebar last loaded -- from the IDE if the IDE has the TOC
+open, otherwise from disk -- then, if the rendering changed, sends the new HTML
+to the Client in an `UpdateToc` message. The Client replaces the contents of its
+sidebar in place, which preserves the sidebar's scroll position along with the
+listener its page attached to intercept clicks on the TOC's links; reloading the
+sidebar would discard both.
+
+Committing the TOC itself doesn't outdate the TOC, since a commit never outdates
+its own file. This matters because each sidebar load translates, and therefore
+commits, the TOC: were that commit to outdate the TOC, each refresh would
+trigger another. Instead, edits to the TOC reach the sidebar in other ways:
+
+* When a connection's current file is the TOC, the connection refreshes its
+ sidebar after each edit to the TOC, whether made in the IDE or the Client.
+* When the TOC changes on disk, the consumer sends `TocChanged` to every
+ connection in the project. Since the TOC's text then differs from the text the
+ sidebar last loaded, this prompts the connection to fetch the TOC's new text
+ from its IDE (with a `LoadFile`), falling back to disk, before re-rendering
+ it.
+
+### Shutdown
+
+Overall shutdown sequence:
+
+1. The LRU map drops a `Cache` and the upgraded Weak reference by the consumer
+ thread is dropped.
+2. The `Cache` drops the `_watcher` and the `consumer_tx` channel.
+3. When dropped, the `_watcher` closes its channel to the walker, stops the
+ watcher, drops its `consumer_tx` channel, and exits.
+4. The consumer thread exits when either its consumer rx channel closes or the
+ weak pointer fails to upgrade.
+
+TODO:
+
+* `SaveOnScan` and search functionality are not implemented yet. This is future
+ work.
diff --git a/server/src/processing/cache.rs b/server/src/processing/cache.rs
index d2847df2..f4171bf7 100644
--- a/server/src/processing/cache.rs
+++ b/server/src/processing/cache.rs
@@ -1,8 +1,3 @@
-// TODO: Remove these once outdated-file scheduling is implemented. It will use
-// `root`, plus the hooks a file watcher or project scan needs -- `remove_file`
-// and `mark_unknown` -- which nothing calls yet.
-#![allow(unused_variables)]
-#![allow(unused)]
// Copyright (C) 2026 Bryan A. Jones.
//
// This file is part of the CodeChat Editor. The CodeChat Editor is free
@@ -18,49 +13,97 @@
// You should have received a copy of the GNU General Public License along with
// the CodeChat Editor. If not, see
// [http://www.gnu.org/licenses](http://www.gnu.org/licenses).
-//! `cache.rs` -- Keep a cache used to store all targets in a project
-//! =================================================================
+//! `cache.rs` -- Cache all targets and fragments in a project
+//! ==========================================================
//!
//! See the [cache spec](cache-spec.md).
// Imports
// -------
//
// ### Standard library
+#[cfg(unix)]
+use std::os::unix::fs::MetadataExt;
use std::{
- collections::{HashMap, HashSet},
- fs::Metadata,
+ collections::{BTreeMap, HashMap, HashSet},
+ fs::{self, File, Metadata},
+ io::{self, Read},
mem,
+ num::NonZeroUsize,
+ ops::Bound,
path::{Path, PathBuf},
- sync::{Arc, Mutex},
+ str,
+ sync::{
+ Arc, Mutex, PoisonError, Weak,
+ mpsc::{self, Receiver, SendError, Sender},
+ },
+ thread,
+ time::{Duration, SystemTime},
};
// ### Third-party
+use ignore::{IncrementalIgnore, WalkBuilder};
+use log::warn;
+use lru::LruCache;
+use notify_debouncer_full::{
+ DebounceEventResult, Debouncer, RecommendedCache, new_debouncer_opt,
+ notify::{
+ self, Event, EventKind, RecommendedWatcher, RecursiveMode,
+ event::{ModifyKind, RenameMode},
+ },
+};
use rand::random;
+use tokio::sync::mpsc::UnboundedSender;
// ### Local
-//
-// None.
+use super::{SourceToCodeChatForWebError, source_to_codechat_for_web};
// Data structures
// ---------------
/// The set of all project caches, keyed by each project's root directory (the
/// directory containing its `toc.md`). The outer mutex guards the addition of
/// newly-discovered projects; each inner mutex guards a single project's cache,
-/// so that work on one project doesn't block work on another.
-pub type CacheMap = Arc>>>>;
+/// so that work on one project doesn't block work on another. When both are
+/// needed, lock the outer mutex first.
+pub type CacheMap = Arc>;
-/// This defines the cache used to store all targets in a project.
+/// The contents of a `CacheMap`. Insert into and resize it only through
+/// `get_or_insert_cache` and `set_cache_map_capacity`, which evict the least
+/// recently used cache when full -- but never a cache in which a connection is
+/// registered.
+pub type ProjectCaches = LruCache>>;
+
+/// The number of project caches the `CacheMap` holds per connection; see
+/// `cache_map_capacity`.
+pub const LRU_ENTRIES_PER_CONNECTION: usize = 10;
+
+/// This defines the cache used to store all targets and fragments in a project.
pub struct Cache {
- /// Provide rapid access to a file by its absolute path; it must be within
- /// the project's root directory. This owns all per-file data.
- pub(super) files: HashMap,
+ /// Provide rapid access to a file by its normalized, absolute path; it must
+ /// be within the project's root directory. This owns all per-file data.
+ /// Paths order component by component, so the files beneath a directory
+ /// immediately follow it in this map: finding them, as a deleted or
+ /// re-walked directory requires, is a range query rather than a search of
+ /// every file.
+ pub(super) files: BTreeMap,
/// Provide rapid access to a `Target`/gather element or `Fragment` by its
/// id.
pub(super) ids: HashMap,
- /// All files for which `FileEntry::status == FileStatus::Unknown`.
- pub(super) pending_files: Vec,
- /// The root directory of this project.
+ /// The root directory of this project, or a specific file if this is a
+ /// single-file "project".
pub(super) root: PathBuf,
+ /// Sends work to this project's consumer thread: the walker sends what it
+ /// finds through a clone of this sender, `mark_outdated` sends outdated
+ /// files, and dropping a `ConnectionRegistration` sends the file it showed.
+ /// `None` if there's no consumer -- a single-file "project", or a test.
+ consumer_tx: Option>,
+ /// This project's watcher, which the watcher/walker thread stores here once
+ /// it registers the watcher; `None` until then, if there's no consumer, or
+ /// if the watcher couldn't start. Dropping it stops the watch, which begins
+ /// the shutdown sequence described in the spec.
+ _watcher: Option>,
+ /// The connections whose current file lies in this project, keyed by
+ /// connection id; each entry is owned by a `ConnectionRegistration`.
+ connections: HashMap,
// TODO: search engine data storage. Search fields: target ID, contents,
// file name. Perhaps [Tantivy](https://docs.rs/tantivy/latest/tantivy/)?
}
@@ -70,13 +113,12 @@ pub struct Cache {
/// `None`.
#[derive(Default)]
pub(super) struct FileEntry {
- /// Metadata used to determine if this data represents the actual state of
- /// the file; if the file is newer, then this file's status must be changed
- /// to `Unknown`. `None` if the file doesn't exist or the metadata can't be
- /// determined.
- pub(super) metadata: Option,
- /// The status of this file. Note that this overlaps with
- /// `Cache::pending_files` and should be kept in sync with it.
+ /// The file's stamp, taken when its content was read; the consumer
+ /// reprocesses the file when it no longer matches. `None` if the file
+ /// doesn't exist, its metadata can't be determined, or the consumer's
+ /// last translation of it failed.
+ pub(super) stamp: Option,
+ /// The status of this file.
pub(super) status: FileStatus,
/// All targets on this page, keyed by id.
pub(super) targets: HashMap,
@@ -91,6 +133,65 @@ pub(super) struct FileEntry {
pub(super) gathers: Vec,
}
+/// How long after a file's modification time a stamp of it must be taken for
+/// that time to be trusted; see `FileStamp::is_racy`. This spans FAT's 2-second
+/// timestamp resolution plus some clock skew on network shares.
+const RACY_WINDOW: Duration = Duration::from_secs(3);
+
+/// A snapshot of a file's metadata, taken immediately before its contents are
+/// read, which tells the consumer whether the cache still represents the file
+/// without reading it again. Git's index stores the same snapshot of each
+/// tracked file for the same purpose; see
+/// [Use of index and Racy Git problem](https://github.com/git/git/blob/master/Documentation/technical/racy-git.adoc).
+pub(super) struct FileStamp {
+ /// The metadata compared against a later snapshot.
+ stat: StatData,
+ /// True if the file was modified within `RACY_WINDOW` of this snapshot. A
+ /// file's timestamp advances only every few milliseconds -- every 2
+ /// seconds, on FAT -- so a write which follows this snapshot within the
+ /// same tick, without changing the file's length, changes its contents
+ /// while leaving its metadata unchanged. A racy stamp therefore never
+ /// matches, forcing the file to be read again; the stamp taken then is
+ /// trustworthy once the file's last write is old enough. Git calls such
+ /// entries "racily clean".
+ is_racy: bool,
+}
+
+/// The parts of a file's metadata which change when its contents do. Unix
+/// supplies the fields Git compares. On Windows, the change time and the file
+/// index which would stand in for Unix's change time and inode are available
+/// only in nightly Rust, leaving just the length and modification time. The
+/// creation time can't take their place: NTFS caches a file's creation time for
+/// a short period after the file is deleted or renamed away, then gives it to a
+/// new file which takes that name -- precisely what an editor does when it
+/// saves by writing a temporary file, then renaming it over the original.
+/// Windows calls this file system tunneling; see the remarks in
+/// [`FileSystemInfo.CreationTime`](https://learn.microsoft.com/en-us/dotnet/api/system.io.filesysteminfo.creationtime#remarks)
+/// and Raymond Chen's
+/// [The apocryphal history of file system tunnelling](https://devblogs.microsoft.com/oldnewthing/20050715-14/?p=34923).
+#[derive(PartialEq, Eq)]
+struct StatData {
+ len: u64,
+ /// The modification time; `None` if the platform doesn't report one. Two
+ /// `None`s compare equal, which would report every same-length edit as no
+ /// change, so `FileStamp::new` marks a stamp without a modification time as
+ /// racy, which keeps it from ever matching. Compare these times only for
+ /// equality, never order:
+ /// [`SystemTime`](https://doc.rust-lang.org/std/time/struct.SystemTime.html)
+ /// isn't monotonic, so a file saved after another may carry an earlier
+ /// time.
+ modified: Option,
+ /// The change time as (seconds, nanoseconds). Unlike the modification time,
+ /// which `touch -d`, `cp -p`, `rsync -t`, and archive extraction set to
+ /// arbitrary values, no program can set it.
+ #[cfg(unix)]
+ ctime: (i64, i64),
+ /// The device and inode numbers, which change when an editor saves by
+ /// renaming a new file over the old one.
+ #[cfg(unix)]
+ dev_ino: (u64, u64),
+}
+
/// Everything the cache knows about one id: where (if anywhere) it's defined,
/// and which files must be rebuilt when that changes.
#[derive(Default)]
@@ -100,7 +201,8 @@ pub(super) struct IdEntry {
pub(super) state: IdState,
/// Every file whose rendered output depends on this id. Note that this does
/// *not* capture the reverse dependency of a `Fragment` on the gather
- /// elements which list it; see `Design` and `commit_file`.
+ /// elements which list it; see and
+ /// `commit_file`.
pub(super) dependents: HashSet,
}
@@ -125,8 +227,7 @@ pub(super) enum IdState {
/// The status of a file from the cache's perspective.
#[derive(Debug, Default, PartialEq, Eq)]
pub(super) enum FileStatus {
- /// The file's content is unknown -- either the file hasn't been processed,
- /// or it's been modified since it was last processed.
+ /// The file hasn't been processed: an entry's state until its first commit.
#[default]
Unknown,
/// The file needs to be re-processed to update cross-references or gather
@@ -136,11 +237,11 @@ pub(super) enum FileStatus {
UpToDate,
}
-/// Contains all information about a target. A target is any HTML element with
-/// an id; the id (which should be globally unique within the project; duplicate
-/// ids are flagged as errors) is the key of `FileEntry::targets`, and the
-/// containing file is the entry holding this value, so neither is duplicated
-/// here.
+/// Contains all information about a target: any HTML element with an id other
+/// than a ``, which is a `Fragment` instead. The id (which should be
+/// globally unique within the project; duplicate ids are flagged as errors) is
+/// the key of `FileEntry::targets`, and the containing file is the entry
+/// holding this value, so neither is duplicated here.
pub(super) struct Target {
/// The inner HTML of this element.
pub(super) inner_html: String,
@@ -153,7 +254,7 @@ pub(super) struct Target {
/// to find the corresponding Target (the anchor is the id), then use this
/// field to identify the line on which that id resides in order to scroll
/// to the appropriate location in the document.
- pub(super) line: usize,
+ pub(super) _line: usize,
}
/// This defines a cross reference to a `Target`.
@@ -172,7 +273,7 @@ pub(super) struct Fragment {
/// stores the result via `Cache::update_fragment_content`.
pub(super) content: String,
/// The line number of this `Fragment` in its file; see `Target::line`.
- pub(super) line: usize,
+ pub(super) _line: usize,
}
// ### Facts
@@ -202,7 +303,7 @@ pub(super) struct TargetFact {
/// The index of the doc block which contains this `Target` in the vec of
/// `CodeDocBlock`s for this file. This is pass-local -- the cache doesn't
/// store it -- but the caller needs it to place hydrated content.
- pub(super) doc_block_index: usize,
+ pub(super) _doc_block_index: usize,
/// The ids listed by this element's `data-gather` attribute; empty if this
/// isn't a gather element.
pub(super) gather_ids: Vec,
@@ -217,18 +318,21 @@ pub(super) struct FragmentFact {
pub(super) line: usize,
/// The index of the first doc block of this `Fragment` in the vec of
/// `CodeDocBlock`s for this file.
- pub(super) doc_block_start_index: usize,
+ pub(super) _doc_block_start_index: usize,
/// The index of the last code/doc block of this `Fragment` in the vec of
/// `CodeDocBlock`s for this file.
- pub(super) code_doc_block_end_index: usize,
+ pub(super) _code_doc_block_end_index: usize,
}
// ### Commit results
/// The result of committing one file's facts to the cache.
pub(super) struct CommitOutcome {
/// Files (other than the committed file) whose rendered output is
- /// invalidated by this commit; their status has already been set to
- /// `Outdated`. The caller should schedule them for reprocessing.
+ /// invalidated by this commit. Their status has already been set to
+ /// `Outdated`, and each has been passed on to be re-rendered: to the
+ /// connections showing it, or else to the consumer; see
+ /// `Cache::mark_outdated`.
+ #[cfg(test)]
pub(super) outdated: HashSet,
}
@@ -252,6 +356,9 @@ pub(super) enum IdResolution<'a> {
/// The id is defined more than once: a map from each defining file to the
/// number of definitions it contains. Carrying the counts lets the caller
/// report a duplicate within a single file.
+ ///
+ /// TODO: this is currently unused.
+ #[allow(dead_code)]
Multiple(&'a HashMap),
}
@@ -267,25 +374,230 @@ pub(super) struct GatherBacklink<'a> {
pub(super) target: &'a Target,
}
+// ### Watcher, walker, and consumer
+//
+// The watcher reports changed paths to the walker, which forwards those its
+// ignore files don't exclude -- and the files found by walking each directory
+// reported -- to the consumer, which applies them to the cache. See `Cache
+// population and maintenance` in the spec.
+/// A path reported by the watcher. A rename is reported as the deletion of its
+/// old path followed by the creation of its new one.
+#[derive(Debug, PartialEq, Eq)]
+pub(super) enum WatcherPathState {
+ /// A file which was created or modified, or a directory which was created.
+ CreatedOrModified(PathBuf),
+ /// A file or directory which was deleted.
+ Deleted(PathBuf),
+}
+
+/// Messages sent to the consumer.
+#[derive(Debug, PartialEq, Eq)]
+pub(super) enum ConsumerPathState {
+ /// A path from the walker: a file a walk found, or a path the watcher
+ /// reported.
+ WatcherPathState(WatcherPathState),
+ /// The walker is about to walk this directory. Every file the walk finds
+ /// arrives before the matching `WalkFinished`.
+ WalkStarted(PathBuf),
+ /// The walk announced by the preceding `WalkStarted` is complete.
+ WalkFinished,
+ /// A request to bring a file the cache already has up to date: a commit
+ /// outdated its rendered output (see `Cache::mark_outdated`), or a
+ /// connection stopped showing it (see `ConnectionRegistration::drop`).
+ /// Unlike a path from the walker, this is no evidence that the file isn't
+ /// ignored, so it can't keep a walk in progress from forgetting the file,
+ /// and is moot once the cache has forgotten it.
+ Rescan(PathBuf),
+}
+
+// ### Connections
+//
+// A connection's translation task owns its current file; the cache keeps only a
+// copy, refreshed by the task each time that file changes. See `Connections` in
+// the cache spec.
+/// One connection whose current file lies in this project.
+struct ConnectionView {
+ /// The file shown in the connection's Client, canonicalized to match the
+ /// paths the cache stores.
+ current_file: PathBuf,
+ /// Where to notify the connection. This is unbounded, since the cache lock
+ /// is held while sending.
+ notify_tx: UnboundedSender,
+}
+
+/// Notifications the cache sends to a registered connection.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum CacheNotification {
+ /// A change elsewhere in the project outdated the rendered output of the
+ /// connection's current file, which the connection should re-translate.
+ CurrentFileOutdated,
+ /// A change elsewhere in the project outdated the rendered output of the
+ /// project's TOC, which every connection in the project shows in its
+ /// sidebar; the connection should re-render it.
+ TocOutdated,
+ /// The contents of the TOC changed on disk, or the TOC was deleted; the
+ /// connection should re-load then re-render it. (Since the IDE is editing
+ /// the current file, changes made to it on disk are superseded by the IDE
+ /// content, so that a `CurrentFileChanged` message is meaningless.)
+ TocChanged,
+}
+
+/// Lists a connection in `Cache::connections` for as long as this value lives;
+/// dropping it removes the listing.
+pub struct ConnectionRegistration {
+ /// The cache the connection is listed in. This is weak, so that only the
+ /// `CacheMap` keeps a cache alive; the map in turn never evicts a cache
+ /// while a connection is listed in it.
+ cache: Weak>,
+ /// The connection's key in `Cache::connections`.
+ connection_id: String,
+}
+
// Code
// ----
+/// The capacity of the `CacheMap` when the Server has the given number of
+/// connections; `translation_task` applies this as connections come and go. Per
+/// the spec, this balances the memory the caches consume against the time
+/// needed to load the cache of a project which isn't cached.
+#[must_use]
+pub fn cache_map_capacity(
+ // The number of connections,
+ // \\(n\\).
+ connections: usize,
+ // \\(n c + 1\\), where
+ // \\(c\\) is
+ // `LRU_ENTRIES_PER_CONNECTION`.
+) -> NonZeroUsize {
+ NonZeroUsize::MIN.saturating_add(connections.saturating_mul(LRU_ENTRIES_PER_CONNECTION))
+}
+
+/// Return the cache of the project rooted at `root`, first creating it if the
+/// `CacheMap` has none.
+pub fn get_or_insert_cache(
+ // The contents of the locked `CacheMap`.
+ caches: &mut ProjectCaches,
+ // The project's root directory, as a canonicalized absolute path.
+ root: &Path,
+ // Creates the project's cache, if the map has none.
+ create: impl FnOnce() -> Arc>,
+ // The project's cache.
+) -> Arc> {
+ if let Some(cache) = caches.get(root) {
+ return cache.clone();
+ }
+ // Inserting into a full map evicts its least recently used cache, even one
+ // with connections, so make room first.
+ evict_unconnected(caches, caches.cap().get() - 1);
+ if caches.len() == caches.cap().get() {
+ caches.resize(caches.cap().saturating_add(1));
+ }
+ let cache = create();
+ caches.put(root.to_path_buf(), cache.clone());
+ cache
+}
+
+/// Set the capacity of the `CacheMap`; `translation_task` applies
+/// `cache_map_capacity` as connections come and go.
+pub fn set_cache_map_capacity(
+ // The contents of the locked `CacheMap`.
+ caches: &mut ProjectCaches,
+ // The new capacity.
+ capacity: NonZeroUsize,
+) {
+ evict_unconnected(caches, capacity.get());
+ // Shrinking the map evicts its least recently used caches, even those with
+ // connections, so keep room for any which remain.
+ caches.resize(capacity.max(NonZeroUsize::new(caches.len()).unwrap_or(capacity)));
+}
+
+/// Evict the least recently used caches in which no connection is registered,
+/// until the `CacheMap` holds no more than `capacity` caches. Per the spec, a
+/// cache with connections is never evicted: a `ConnectionRegistration` holds
+/// only a weak reference to its cache, so evicting the cache would drop it
+/// while its connections still depend on it. Since a connection registers with
+/// at most one cache, and the map's capacity exceeds the number of connections
+/// (see `cache_map_capacity`), this normally leaves the map within `capacity`.
+fn evict_unconnected(
+ // The contents of the locked `CacheMap`.
+ caches: &mut ProjectCaches,
+ // The number of caches to keep, if possible.
+ capacity: usize,
+) {
+ let excess = caches.len().saturating_sub(capacity);
+ let evicted: Vec = caches
+ .iter()
+ .rev()
+ .filter(|(_, cache)| {
+ cache
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner)
+ .connections
+ .is_empty()
+ })
+ .map(|(root, _)| root.clone())
+ .take(excess)
+ .collect();
+ for root in evicted {
+ caches.pop(&root);
+ }
+}
+
impl Cache {
+ /// Create an empty cache with no consumer to keep it current: the cache of
+ /// a single-file "project", which consists only of that file, or of a test.
+ /// A project's cache comes from `Cache::new_project`.
#[must_use]
pub fn new(
// The root directory of this project, as a canonicalized absolute path;
- // all cached files must live within it.
+ // all cached files must live within it. For a single-file "project",
+ // this is that file.
root: PathBuf,
+ // The empty cache.
) -> Self {
Cache {
- files: HashMap::new(),
+ files: BTreeMap::new(),
ids: HashMap::new(),
- pending_files: vec![],
root,
+ consumer_tx: None,
+ _watcher: None,
+ connections: HashMap::new(),
}
}
- /// Apply the facts collected from one file's DOM walk to the cache in a
- /// single transaction. This satisfies several requirements:
+ /// Create the cache of a project, then start the threads which keep it
+ /// current: a watcher thread, which registers a watcher to report changes
+ /// to the project's files, then becomes the walker, which filters those
+ /// reports through the project's ignore files; and a consumer, which
+ /// applies the walker's results to the cache. The walker's first task is a
+ /// walk of the entire project, so that the consumer brings every file in it
+ /// into the cache. See `Cache population and maintenance` in the spec.
+ #[must_use]
+ pub fn new_project(
+ // The project's root directory, as a canonicalized absolute path; all
+ // cached files must live within it.
+ root: PathBuf,
+ // The cache, shared with the threads which keep it current.
+ ) -> Arc> {
+ let (consumer_tx, consumer_rx) = mpsc::channel();
+ let walker_consumer_tx = consumer_tx.clone();
+ let cache = Arc::new(Mutex::new(Cache {
+ consumer_tx: Some(consumer_tx),
+ ..Cache::new(root.clone())
+ }));
+ // Give these threads only weak references, so that they don't keep the
+ // cache alive; see `Shutdown` in the spec.
+ let weak_cache = Arc::downgrade(&cache);
+ thread::spawn(move || consume(&weak_cache, &consumer_rx));
+ let weak_cache = Arc::downgrade(&cache);
+ thread::spawn(move || watch_then_walk(&weak_cache, root, walker_consumer_tx));
+ cache
+ }
+
+ /// commit_file
+ ///
+ /// Apply the facts collected
+ /// from one file's DOM walk to the cache in a single transaction. This
+ /// satisfies several requirements:
///
/// * Determine if any files containing cross-references need to be rebuilt
/// due to changes in the `Target`s in this file: any target which was
@@ -324,16 +636,16 @@ impl Cache {
/// * Otherwise only the symmetric difference is affected: the fragments
/// which gained or lost this backlink.
///
- /// See the `Design` section for why this doesn't cycle back.
+ /// See for why this doesn't cycle back.
#[allow(clippy::too_many_lines)]
pub(super) fn commit_file(
&mut self,
// The file whose facts these are.
path: &Path,
- // The file's metadata, captured when its content was read; `None` if
- // unavailable. Read this *before* locking the cache, so no I/O happens
- // while the lock is held.
- metadata: Option,
+ // The file's stamp, taken when its content was read; `None` if its
+ // metadata is unavailable. Take this *before* locking the cache, so no
+ // I/O happens while the lock is held.
+ stamp: Option,
// The facts collected from the file's DOM.
facts: FileFacts,
// The outcome: outdated files. Reprocessing these outdated files may
@@ -394,7 +706,7 @@ impl Cache {
Target {
inner_html: fact.inner_html,
gather_ids: fact.gather_ids,
- line: fact.line,
+ _line: fact.line,
},
);
}
@@ -415,7 +727,7 @@ impl Cache {
fact.id,
Fragment {
content,
- line: fact.line,
+ _line: fact.line,
},
);
}
@@ -441,6 +753,9 @@ impl Cache {
.chain(old_fragments.keys())
.chain(new_counts.keys())
.collect();
+ // The ids this commit left without any definition, which are forgotten
+ // below unless something references them.
+ let mut undefined_ids: Vec = Vec::new();
for id in affected_ids {
let id_entry = self.ids.entry(id.clone()).or_default();
let mut definitions = id_entry.state.definitions();
@@ -453,6 +768,9 @@ impl Cache {
id_entry.state = new_state;
outdated.extend(id_entry.dependents.iter().cloned());
}
+ if id_entry.state == IdState::Missing {
+ undefined_ids.push(id.clone());
+ }
}
// ### Outdate the dependents of changed definitions
@@ -511,12 +829,12 @@ impl Cache {
// file is written and re-read (see `Auto-assignment of ids`), at which
// point they arrive here like any other id.
let entry = self.files.get_mut(path).expect("entry was created above");
- entry.metadata = metadata;
+ entry.stamp = stamp;
entry.targets = new_targets;
entry.fragments = new_fragments;
entry.gathers = new_gathers;
entry.xrefs = facts.xrefs.into_iter().map(|id| Xref { id }).collect();
- self.set_status(path, FileStatus::UpToDate);
+ entry.status = FileStatus::UpToDate;
// ### Link the new version's outgoing references
//
@@ -530,16 +848,25 @@ impl Cache {
.insert(path.to_path_buf());
}
+ // ### Forget unused ids
+ //
+ // Only now, with this file's references linked, is it known which of
+ // the ids it stopped defining nothing references.
+ for id in &undefined_ids {
+ self.remove_if_unused(id);
+ }
+
// ### Mark outdated files
//
// A file never outdates itself: its rendered output was just produced
// from the state committed here.
outdated.remove(path);
- for outdated_path in &outdated {
- self.set_status(outdated_path, FileStatus::Outdated);
- }
+ self.mark_outdated(&outdated);
- CommitOutcome { outdated }
+ CommitOutcome {
+ #[cfg(test)]
+ outdated,
+ }
}
/// Remove everything the cache knows about a file -- when the file is
@@ -555,47 +882,9 @@ impl Cache {
) -> CommitOutcome {
let outcome = self.commit_file(path, None, FileFacts::default());
self.files.remove(path);
- self.pending_files.retain(|pending| pending != path);
outcome
}
- /// Record that a file's cached content no longer reflects the file
- /// itself -- it's been written or modified since it was last processed --
- /// which queues it in `pending_files` for reprocessing. This is how an id
- /// auto-assigned during hydration reaches the cache: the assignment is only
- /// in the file until the file is saved, re-read, and committed.
- pub(super) fn mark_unknown(
- &mut self,
- // The file whose content is now unknown.
- path: &Path,
- ) {
- self.files.entry(path.to_path_buf()).or_default();
- self.set_status(path, FileStatus::Unknown);
- }
-
- /// Set a file's status, keeping `pending_files` -- which holds exactly the
- /// files whose status is `Unknown` -- in sync with it.
- fn set_status(
- &mut self,
- // The file whose status changed; nothing happens if the cache has no
- // entry for it.
- path: &Path,
- // Its new status.
- status: FileStatus,
- ) {
- let Some(entry) = self.files.get_mut(path) else {
- return;
- };
- let unknown = status == FileStatus::Unknown;
- entry.status = status;
- let queued = self.pending_files.iter().any(|pending| pending == path);
- if unknown && !queued {
- self.pending_files.push(path.to_path_buf());
- } else if !unknown && queued {
- self.pending_files.retain(|pending| pending != path);
- }
- }
-
/// Generate an id for an element which requests one (`id="*"`). The result
/// is deliberately *not* recorded in the cache: it enters the cache only
/// once the file holding it is written and re-read, so that a write which
@@ -644,8 +933,8 @@ impl Cache {
id: &str,
// The fragment's code/doc block content, rendered as HTML.
content: String,
- // The files (other than `path`) marked outdated by this change; their
- // status has already been set to `Outdated`.
+ // The files (other than `path`) marked outdated by this change, as
+ // described by `CommitOutcome::outdated`.
) -> HashSet {
// Apply the new content first, so that no borrow of `self.files`
// outlives the check; report whether it differed from the old content.
@@ -670,12 +959,141 @@ impl Cache {
// Gather elements in the fragment's own file are updated by the caller
// in the same processing pass.
outdated.remove(path);
- for outdated_path in &outdated {
- self.set_status(outdated_path, FileStatus::Outdated);
- }
+ self.mark_outdated(&outdated);
outdated
}
+ /// mark_outdated
+ ///
+ /// Mark files outdated, then see that each is re-rendered. A connection
+ /// re-renders a file it shows -- its current file, or the TOC in its
+ /// sidebar, which every connection in the project shows -- since the text
+ /// it shows may be newer than the file on disk; the consumer reprocesses
+ /// the rest from disk.
+ fn mark_outdated(
+ &mut self,
+ // The files to mark.
+ outdated: &HashSet,
+ ) {
+ for path in outdated {
+ if let Some(entry) = self.files.get_mut(path) {
+ entry.status = FileStatus::Outdated;
+ }
+ }
+ self.notify_current_file_outdated(outdated);
+ if outdated.contains(&self.toc_path()) {
+ self.notify_toc(CacheNotification::TocOutdated);
+ }
+ if let Some(consumer_tx) = &self.consumer_tx {
+ for path in outdated.iter().filter(|path| !self.is_shown(path)) {
+ // A send fails only once the consumer has exited, when this
+ // cache is being dropped and so needs no more updates.
+ let _ = consumer_tx.send(ConsumerPathState::Rescan(path.clone()));
+ }
+ }
+ }
+
+ /// Notify each connection whose current file is one of these files, so that
+ /// the connection re-translates it. Several connections may share a current
+ /// file, if it's being edited in each simultaneously.
+ fn notify_current_file_outdated(
+ &self,
+ // The files whose rendered output is outdated.
+ outdated: &HashSet,
+ ) {
+ for connection in self.connections.values() {
+ if outdated.contains(&connection.current_file) {
+ // A send fails only once the connection's translation task has
+ // exited, leaving nothing to notify.
+ let _ = connection
+ .notify_tx
+ .send(CacheNotification::CurrentFileOutdated);
+ }
+ }
+ }
+
+ /// Notify every connection about the TOC, which every connection in the
+ /// project shows in its sidebar.
+ fn notify_toc(
+ &self,
+ // `TocOutdated` if only the TOC's rendering is outdated; `TocChanged`
+ // if its text changed.
+ notification: CacheNotification,
+ ) {
+ for connection in self.connections.values() {
+ // A send fails only once the connection's translation task has
+ // exited, leaving nothing to notify.
+ let _ = connection.notify_tx.send(notification);
+ }
+ }
+
+ /// The project's TOC: the project's root is the directory containing it;
+ /// see `find_path_to_toc`.
+ fn toc_path(
+ &self,
+ // The TOC's path.
+ ) -> PathBuf {
+ self.root.join("toc.md")
+ }
+
+ /// Report whether a connection shows this file, which makes the connection
+ /// responsible for re-rendering it: it's a connection's current file, or
+ /// it's the TOC, which every connection's sidebar shows.
+ fn is_shown(
+ &self,
+ // The file to check.
+ path: &Path,
+ // True if a connection shows the file.
+ ) -> bool {
+ self.is_current_file(path) || (!self.connections.is_empty() && path == self.toc_path())
+ }
+
+ /// Report whether this file is some connection's current file.
+ fn is_current_file(
+ &self,
+ // The file to check.
+ path: &Path,
+ // True if it's some connection's current file.
+ ) -> bool {
+ self.connections
+ .values()
+ .any(|connection| connection.current_file == path)
+ }
+
+ /// Iterate over the cached files beneath a directory, which immediately
+ /// follow the directory itself in `files`.
+ fn files_beneath<'a>(
+ &'a self,
+ // The directory.
+ dir: &'a Path,
+ // The cached files beneath it, at any depth.
+ ) -> impl Iterator- {
+ self.files
+ .range::((Bound::Excluded(dir), Bound::Unbounded))
+ .map(|(path, _)| path)
+ .take_while(move |path| path.starts_with(dir))
+ }
+
+ /// Forget a file which no longer exists, or which the walker now ignores --
+ /// unless a connection shows it. The connection's IDE may still hold the
+ /// file's text, so instead have the connection re-translate that text,
+ /// which keeps the file in the cache; for the TOC, every connection
+ /// re-loads it.
+ fn forget_file(
+ &mut self,
+ // The file to forget.
+ path: &Path,
+ ) {
+ if self.is_shown(path) {
+ self.notify_current_file_outdated(&HashSet::from([path.to_path_buf()]));
+ if path == self.toc_path() {
+ self.notify_toc(CacheNotification::TocChanged);
+ }
+ } else if self.files.contains_key(path) {
+ self.remove_file(path);
+ }
+ }
+
/// Look up an id, returning the target or fragment it names along with the
/// defining file, `Missing` if no file defines it, or `Multiple` if more
/// than one definition exists.
@@ -754,12 +1172,22 @@ impl Cache {
) {
if let Some(id_entry) = self.ids.get_mut(id) {
id_entry.dependents.remove(referrer);
- // An id which nothing defines and nothing references carries no
- // information; drop it rather than accumulating an entry for every
- // id ever mistyped.
- if id_entry.state == IdState::Missing && id_entry.dependents.is_empty() {
- self.ids.remove(id);
- }
+ }
+ self.remove_if_unused(id);
+ }
+
+ /// Forget an id which nothing defines and nothing references. Its entry
+ /// carries no information, so keeping it would accumulate an entry for
+ /// every id ever mistyped, deleted, or renamed.
+ fn remove_if_unused(
+ &mut self,
+ // The id to check.
+ id: &str,
+ ) {
+ if self.ids.get(id).is_some_and(|id_entry| {
+ id_entry.state == IdState::Missing && id_entry.dependents.is_empty()
+ }) {
+ self.ids.remove(id);
}
}
}
@@ -789,6 +1217,101 @@ impl IdState {
}
}
+impl FileStamp {
+ pub(super) fn new(
+ // The file's metadata, which this converts to a `FileStamp`.
+ metadata: &Metadata,
+ ) -> Self {
+ let modified = metadata.modified().ok();
+ // A modification time in the future -- from clock skew between this
+ // machine and a network share, for example -- is racy as well.
+ let is_racy = modified.is_none_or(|modified| {
+ !SystemTime::now()
+ .duration_since(modified)
+ .is_ok_and(|age| age >= RACY_WINDOW)
+ });
+ FileStamp {
+ stat: StatData {
+ len: metadata.len(),
+ modified,
+ #[cfg(unix)]
+ ctime: (metadata.ctime(), metadata.ctime_nsec()),
+ #[cfg(unix)]
+ dev_ino: (metadata.dev(), metadata.ino()),
+ },
+ is_racy,
+ }
+ }
+
+ /// Report whether the file's contents are unchanged since this stamp was
+ /// taken.
+ fn is_unmodified(
+ &self,
+ // A stamp taken from the file now.
+ current: &FileStamp,
+ // True if the contents are unchanged; false if they may have changed.
+ ) -> bool {
+ !self.is_racy && self.stat == current.stat
+ }
+}
+
+impl ConnectionRegistration {
+ /// List a connection in a project's cache. Drop any earlier registration of
+ /// this connection first: dropping it afterward would remove this listing,
+ /// since both share one key.
+ #[must_use]
+ pub fn new(
+ // The cache of the project containing `current_file`.
+ cache: &Arc>,
+ // The connection's id, unique among all connections.
+ connection_id: String,
+ // The file shown in the connection's Client, as a canonicalized
+ // absolute path.
+ current_file: PathBuf,
+ // Where the cache sends this connection's notifications.
+ notify_tx: UnboundedSender,
+ ) -> Self {
+ cache.lock().unwrap().connections.insert(
+ connection_id.clone(),
+ ConnectionView {
+ current_file,
+ notify_tx,
+ },
+ );
+ ConnectionRegistration {
+ cache: Arc::downgrade(cache),
+ connection_id,
+ }
+ }
+}
+
+impl Drop for ConnectionRegistration {
+ fn drop(&mut self) {
+ // The `CacheMap` keeps a cache with connections alive, so this fails
+ // only once the map itself is gone, taking the listing with it.
+ let Some(cache) = self.cache.upgrade() else {
+ return;
+ };
+ // Tolerate a poisoned lock: this may run while unwinding from another
+ // panic, where a second panic aborts the process.
+ let mut cache = cache.lock().unwrap_or_else(PoisonError::into_inner);
+ let Some(connection) = cache.connections.remove(&self.connection_id) else {
+ return;
+ };
+ // The consumer leaves a connection's current file to that connection,
+ // so it may have skipped a change to this file on disk, or its
+ // deletion. Once no connection shows the file, have the consumer check
+ // the file against the disk. If the file on disk is unchanged, what the
+ // connection committed from its IDE's text stays in the cache, since
+ // the IDE may still hold that text.
+ if !cache.is_current_file(&connection.current_file)
+ && let Some(consumer_tx) = &cache.consumer_tx
+ {
+ let _ = consumer_tx.send(ConsumerPathState::Rescan(connection.current_file));
+ }
+ }
+}
+
/// Diff a file's old and new gather elements, reporting the ids whose rendered
/// backlinks changed as a result -- the reverse edge described in
/// `commit_file`, whose documentation gives the rule applied here. The file
@@ -842,6 +1365,607 @@ fn gathered_ids<'a>(
.flat_map(|target| target.gather_ids.iter())
}
+// ### Watcher
+/// The watcher thread: register a watcher for the project and give it to the
+/// cache, then become the walker, starting with a walk of the entire project.
+/// Registering the watcher can take a while -- on some platforms, it visits
+/// every file in the project -- so this runs on its own thread, spawned by
+/// `Cache::new_project`, rather than delaying the translation which created the
+/// cache.
+fn watch_then_walk(
+ // The cache which owns the watcher.
+ cache: &Weak>,
+ // The project's root directory.
+ root: PathBuf,
+ // Where the walker sends what it finds.
+ consumer_tx: Sender,
+) {
+ let (walker_tx, walker_rx) = mpsc::channel();
+ // Start watching before walking, so that a file created during the walk is
+ // reported by one or the other. A watcher failure isn't fatal: the cache
+ // still works, but won't learn of files which change after the walk. On
+ // Linux, this happens once the per-user inotify limits are reached, since
+ // each watcher uses its own inotify instance.
+ let watcher = watch_project(&root, walker_tx.clone())
+ .inspect_err(|err| warn!("Unable to watch {}: {err}", root.display()))
+ .ok();
+ // The cache owns the watcher, so that dropping the cache stops the watch. A
+ // cache which is already gone leaves nothing to walk for; returning drops
+ // the watcher, stopping the watch.
+ let Some(cache) = cache.upgrade() else {
+ return;
+ };
+ // The field's leading underscore marks it as held only for what dropping it
+ // does; this is the one place which sets it.
+ #[allow(clippy::used_underscore_binding)]
+ {
+ cache.lock().unwrap()._watcher = watcher;
+ }
+ drop(cache);
+ // Give the walker its first task, then drop this sender. Only the watcher
+ // holds another, so if it failed, the walker exits after this walk. The
+ // send can't fail, since `walker_rx` is alive.
+ let _ = walker_tx.send(WatcherPathState::CreatedOrModified(root.clone()));
+ drop(walker_tx);
+ Walker::new(root, consumer_tx).run(&walker_rx);
+}
+
+/// Watch `root` recursively, sending the walker each path created, modified, or
+/// deleted within it. The event handler runs on the debouncer's thread.
+fn watch_project(
+ // The project's root directory.
+ root: &Path,
+ // Where to send each path reported; the walker owns the receiving end.
+ walker_tx: Sender,
+ // The running watcher; dropping it stops the watch.
+) -> notify::Result> {
+ let handler_root = root.to_path_buf();
+ // The watcher only needs to learn of files added to, removed from, or
+ // modified in the project. notify 8 has no `Config::with_event_kinds`, so
+ // `watcher_path_states` must discard the events which don't matter.
+ //
+ // TODO: pass the kinds which matter to `Config::with_event_kinds` once
+ // notify-debouncer-full depends on notify 9.
+ let mut debouncer = new_debouncer_opt::<_, RecommendedWatcher, RecommendedCache>(
+ Duration::from_secs(2),
+ None,
+ move |result: DebounceEventResult| match result {
+ Ok(debounced_events) => {
+ for path_state in debounced_events
+ .iter()
+ .flat_map(|debounced_event| watcher_path_states(debounced_event, &handler_root))
+ {
+ // A send fails only once the walker has exited, when the
+ // cache is being dropped -- along with this watcher.
+ if walker_tx.send(path_state).is_err() {
+ return;
+ }
+ }
+ }
+ // A path which can't be watched isn't usable by the IDE either, so
+ // report this only to the log.
+ Err(errors) => {
+ for err in errors {
+ warn!("Unable to watch {}: {err}", handler_root.display());
+ }
+ }
+ },
+ RecommendedCache::new(),
+ // Per the spec, symlinks aren't followed, since a file reached through
+ // one would be cached twice. The walker skips a symlink the watcher
+ // reports; this keeps the watcher from reporting the files beneath a
+ // symlinked directory, which on some platforms it would otherwise
+ // watch.
+ notify::Config::default().with_follow_symlinks(false),
+ )?;
+ debouncer.watch(root, RecursiveMode::Recursive)?;
+ Ok(debouncer)
+}
+
+/// Translate one watcher event into the paths it reports to the walker: per the
+/// spec, the files created, modified, or deleted, and the directories created
+/// or deleted.
+fn watcher_path_states(
+ // The event to translate.
+ event: &Event,
+ // The project's root directory, which is re-walked if the watcher lost
+ // events.
+ root: &Path,
+ // The paths to report, in order.
+) -> Vec {
+ use WatcherPathState::{CreatedOrModified, Deleted};
+
+ if event.need_rescan() {
+ return vec![CreatedOrModified(root.to_path_buf())];
+ }
+ let mut paths = event.paths.iter().cloned();
+ match event.kind {
+ // The debouncer stitches the two halves of a rename into one event,
+ // listing the path renamed from, then the path renamed to, when it can
+ // match them; otherwise, it passes each half through on its own, as a
+ // deletion or a creation.
+ EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => paths
+ .next()
+ .map(Deleted)
+ .into_iter()
+ .chain(paths.map(CreatedOrModified))
+ .collect(),
+ EventKind::Create(_) | EventKind::Modify(ModifyKind::Name(RenameMode::To)) => {
+ paths.map(CreatedOrModified).collect()
+ }
+ EventKind::Remove(_) | EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
+ paths.map(Deleted).collect()
+ }
+ // The platform couldn't tell which half of a rename this is, so ask the
+ // filesystem.
+ EventKind::Modify(ModifyKind::Name(_)) => paths
+ .map(|path| {
+ if path.exists() {
+ CreatedOrModified(path)
+ } else {
+ Deleted(path)
+ }
+ })
+ .collect(),
+ // A directory is modified whenever its contents change, which are
+ // reported in their own right, so skip directories here.
+ EventKind::Modify(_) | EventKind::Any => paths
+ .filter(|path| !path.is_dir())
+ .map(CreatedOrModified)
+ .collect(),
+ EventKind::Access(_) | EventKind::Other => vec![],
+ }
+}
+
+// ### Walker
+/// The walker, which filters the paths the watcher reports through the
+/// project's ignore files and walks each directory reported, sending the
+/// results to the consumer. See `Walker` in the spec.
+struct Walker {
+ /// The project's root directory.
+ root: PathBuf,
+ /// Decides whether a path the watcher reports is ignored, without walking
+ /// the project. It keeps the ignore files it reads, so it must be rebuilt
+ /// when one of them changes.
+ ignore: IncrementalIgnore,
+ /// Where to send the files found.
+ consumer_tx: Sender,
+}
+
+impl Walker {
+ /// Create the walker for a project.
+ fn new(
+ // The project's root directory.
+ root: PathBuf,
+ // Where to send the files found.
+ consumer_tx: Sender,
+ // The walker, which reads the project's ignore files only as it needs
+ // them.
+ ) -> Self {
+ Walker {
+ ignore: build_ignore(&root),
+ root,
+ consumer_tx,
+ }
+ }
+
+ /// Process each path the watcher sends, until the walker's channel
+ /// closes -- when the cache is dropped, which drops the watcher, or after
+ /// the first walk if the watcher couldn't start -- or the consumer exits,
+ /// closing its own channel. This runs on the watcher thread; see
+ /// `watch_then_walk`.
+ fn run(
+ mut self,
+ // The paths the watcher sends.
+ walker_rx: &Receiver,
+ ) {
+ while let Ok(path_state) = walker_rx.recv() {
+ if self.process(path_state).is_err() {
+ return;
+ }
+ }
+ }
+
+ /// Process one path the watcher sent, sending the results to the consumer.
+ fn process(
+ &mut self,
+ // The path to process.
+ path_state: WatcherPathState,
+ // An error once the consumer has exited.
+ ) -> Result<(), SendError> {
+ let (WatcherPathState::CreatedOrModified(path) | WatcherPathState::Deleted(path)) =
+ &path_state;
+ // An ignore file is hidden, which makes it ignored itself, so look for
+ // one first.
+ if let Some(dir) = self.rewalk_dir(path) {
+ return self.walk(&dir);
+ }
+ if let WatcherPathState::CreatedOrModified(path) = &path_state
+ && let Ok(metadata) = fs::symlink_metadata(path)
+ {
+ // Per the spec, symlinks aren't walked, since a file reached
+ // through one would be cached twice.
+ if metadata.is_dir() {
+ return self.walk(path);
+ }
+ if metadata.is_symlink() {
+ return Ok(());
+ }
+ }
+ // Nothing says whether a deleted path was a file or a directory.
+ // Checking it as a file errs toward sending it to the consumer, which
+ // costs only a search of the cache for files which aren't there.
+ if self.is_ignored(path, false) {
+ Ok(())
+ } else {
+ self.consumer_tx
+ .send(ConsumerPathState::WatcherPathState(path_state))
+ }
+ }
+
+ /// If `path` is one of the ignore files which the walk obeys, return the
+ /// directory to re-walk, which reloads the ignore files: the directory an
+ /// `.ignore` or `.gitignore` governs, or the entire project for
+ /// `.git/info/exclude`. A file with an ignore file's name prompts this even
+ /// where Git wouldn't read it -- outside a repository, for example -- which
+ /// costs only an unneeded walk.
+ fn rewalk_dir(
+ &self,
+ // The path to check.
+ path: &Path,
+ // The directory to re-walk, or `None` if this isn't an ignore file.
+ ) -> Option {
+ let dir = path.parent()?;
+ match path.file_name()?.to_str()? {
+ ".ignore" | ".gitignore" => Some(dir.to_path_buf()),
+ "exclude" if dir.ends_with(Path::new(".git").join("info")) => Some(self.root.clone()),
+ _ => None,
+ }
+ }
+
+ /// Report whether the walk skips this path: it's hidden or excluded by an
+ /// ignore file, or it lies within a directory which is.
+ fn is_ignored(
+ &mut self,
+ // The path to check.
+ path: &Path,
+ // True if the path is a directory, which some ignore rules match
+ // specifically.
+ is_dir: bool,
+ // True if the walk skips the path.
+ ) -> bool {
+ // The matcher takes paths relative to the project's root; anything
+ // outside the project is ignored.
+ path.strip_prefix(&self.root)
+ .ok()
+ .is_none_or(|relative| self.ignore.matched(relative, is_dir).is_ignore())
+ }
+
+ /// Walk a directory, sending the consumer each file found, preceded by a
+ /// `WalkStarted` and followed by a `WalkFinished`.
+ fn walk(
+ &mut self,
+ // The directory to walk.
+ dir: &Path,
+ // An error once the consumer has exited.
+ ) -> Result<(), SendError> {
+ // A walk never filters its own starting point, so skip an ignored
+ // directory here. This includes a directory whose ignore file changed:
+ // the walk never enters that directory, so the change can't matter,
+ // which spares a reload for each such file a package manager creates,
+ // say.
+ if self.is_ignored(dir, true) {
+ return Ok(());
+ }
+ // The walk reads the ignore files afresh, so rebuild the matcher to
+ // agree with it. The ignore files may have changed without the watcher
+ // reporting it: the debouncer drops the events within a deleted
+ // directory, for example, including the deletion of an ignore file
+ // there.
+ self.ignore = build_ignore(&self.root);
+ self.consumer_tx
+ .send(ConsumerPathState::WalkStarted(dir.to_path_buf()))?;
+ // The walk reports an error (an unreadable directory, say) as an entry
+ // of its own, then continues. Skip these, since a file the walk can't
+ // read isn't usable by the IDE either. Symlinks aren't followed, so a
+ // symlink loop can't make the walk run forever.
+ for entry in WalkBuilder::new(dir).build().flatten() {
+ if entry
+ .file_type()
+ .is_some_and(|file_type| file_type.is_file())
+ {
+ self.consumer_tx.send(ConsumerPathState::WatcherPathState(
+ WatcherPathState::CreatedOrModified(entry.into_path()),
+ ))?;
+ }
+ }
+ self.consumer_tx.send(ConsumerPathState::WalkFinished)
+ }
+}
+
+/// Build the matcher which decides whether a path the watcher reports is
+/// ignored. It applies the same filters as every walk: the
+/// [ignore](https://docs.rs/ignore) crate's defaults, which are ripgrep's.
+/// These skip hidden files and directories (such as `.git`), anything excluded
+/// by an `.ignore` file, and -- only inside a Git repository -- anything
+/// excluded by `.gitignore` or Git's other exclude rules.
+fn build_ignore(
+ // The project's root directory, to which the matcher's paths are relative.
+ root: &Path,
+ // The matcher, which reads the ignore files only as it needs them.
+) -> IncrementalIgnore {
+ WalkBuilder::new(root)
+ .build_matchers()
+ .pop()
+ .expect("A builder given one path builds one matcher.")
+}
+
+// ### Consumer
+/// Apply each message sent to the consumer, until the channel closes or the
+/// cache is dropped. This runs on its own thread, spawned by
+/// `Cache::new_project`.
+fn consume(
+ // The cache to update. This is upgraded only while applying a message, so
+ // that the consumer doesn't keep a dropped cache alive while it waits for
+ // the next one.
+ cache: &Weak>,
+ // The messages to apply.
+ consumer_rx: &Receiver,
+) {
+ let mut consumer = Consumer::default();
+ while let Ok(message) = consumer_rx.recv() {
+ let Some(cache) = cache.upgrade() else {
+ return;
+ };
+ consumer.apply(&cache, message);
+ }
+}
+
+/// The consumer's state between messages.
+#[derive(Default)]
+struct Consumer {
+ /// While a walk is in progress, the cached files beneath the directory
+ /// being walked which the walk hasn't found; `None` otherwise.
+ unwalked: Option>,
+}
+
+impl Consumer {
+ /// Apply one message to the cache.
+ fn apply(
+ &mut self,
+ // The cache to update.
+ cache: &Arc>,
+ // The message to apply.
+ message: ConsumerPathState,
+ ) {
+ match message {
+ // The walker walks one directory at a time, so this replaces only
+ // the state of a walk which never finished; lacking its
+ // `WalkFinished`, the files it didn't find can't be presumed gone.
+ ConsumerPathState::WalkStarted(dir) => {
+ self.unwalked = Some(cache.lock().unwrap().files_beneath(&dir).cloned().collect());
+ }
+ // A cached file which the walk didn't find no longer exists, or is
+ // now ignored. Release the lock between files, so that forgetting
+ // many doesn't hold up translations.
+ ConsumerPathState::WalkFinished => {
+ if let Some(unwalked) = self.unwalked.take() {
+ for path in unwalked {
+ cache.lock().unwrap().forget_file(&path);
+ }
+ }
+ }
+ ConsumerPathState::WatcherPathState(WatcherPathState::CreatedOrModified(path)) => {
+ if let Some(unwalked) = &mut self.unwalked {
+ unwalked.remove(&path);
+ }
+ scan_file(cache, &path, true);
+ }
+ // Per the spec, only a path which no longer exists is forgotten.
+ // One which exists again by the time this is applied -- recreated
+ // by a version control checkout, say, or by an editor which saves
+ // by renaming a new file over the old one -- is scanned instead.
+ ConsumerPathState::WatcherPathState(WatcherPathState::Deleted(path)) => {
+ match fs::symlink_metadata(&path) {
+ Ok(metadata) if !metadata.is_symlink() => scan_file(cache, &path, true),
+ _ => forget_path(cache, &path),
+ }
+ }
+ ConsumerPathState::Rescan(path) => scan_file(cache, &path, false),
+ }
+ }
+}
+
+/// Forget a path which no longer exists. Nothing says whether it was a file or
+/// a directory, so if the cache has no file at this path, assume it was a
+/// directory and forget every cached file beneath it. The lock is released
+/// between files, so that forgetting a large directory doesn't hold up
+/// translations.
+fn forget_path(
+ // The cache to update.
+ cache: &Mutex,
+ // The path which no longer exists.
+ path: &Path,
+) {
+ let paths: Vec = {
+ let cache = cache.lock().unwrap();
+ if cache.files.contains_key(path) {
+ vec![path.to_path_buf()]
+ } else {
+ cache.files_beneath(path).cloned().collect()
+ }
+ };
+ for path in paths {
+ cache.lock().unwrap().forget_file(&path);
+ }
+}
+
+/// The outcome of the consumer's translation of a file.
+enum ScanOutcome {
+ /// The translation committed the file's facts to the cache.
+ Translated,
+ /// The file isn't text, or no lexer recognizes it, so nothing was
+ /// committed.
+ Untranslatable,
+ /// The translation failed, perhaps after committing the file's facts.
+ Failed,
+}
+
+/// Bring the cache's record of a file up to date with the file on disk by
+/// translating the file to the CodeChatForWeb format, which commits its facts
+/// to the cache. A connection's current file is skipped: the connection keeps
+/// it up to date from its IDE's text, which supersedes the file on disk.
+fn scan_file(
+ // The cache to update.
+ cache: &Arc>,
+ // The file to scan.
+ path: &Path,
+ // True if the walker sent this file, which therefore passed its ignore
+ // rules; only such a file is scanned when the cache doesn't have it. A
+ // rescan of a file the cache has since forgotten -- one a walk found
+ // ignored, say -- is moot.
+ is_from_walker: bool,
+) {
+ // Perform I/O before locking the cache, so that the lock isn't held during
+ // it.
+ let stamp = match fs::metadata(path) {
+ Ok(metadata) if metadata.is_file() => FileStamp::new(&metadata),
+ // The file no longer exists -- or a directory has replaced it, which
+ // the walker reports separately.
+ _ => {
+ cache.lock().unwrap().forget_file(path);
+ return;
+ }
+ };
+ // True if the TOC's text differs from the text the cache last read, which a
+ // connection's sidebar may still show. When the cache has no record of the
+ // TOC, no sidebar can show it: loading it in a sidebar records it.
+ let is_toc_changed;
+ {
+ let cache = cache.lock().unwrap();
+ let entry = cache.files.get(path);
+ let is_modified = entry
+ .and_then(|entry| entry.stamp.as_ref())
+ .is_none_or(|cached| !cached.is_unmodified(&stamp));
+ is_toc_changed = is_modified && entry.is_some() && path == cache.toc_path();
+ if cache.is_current_file(path) {
+ // The connections' sidebars show the TOC from their own IDE's text
+ // or from disk, so each must still re-load it.
+ if is_toc_changed {
+ cache.notify_toc(CacheNotification::TocChanged);
+ }
+ return;
+ }
+ // A cached file needs scanning if it changed, or if a commit outdated
+ // it, which its status records.
+ let is_skipped = match entry {
+ None => !is_from_walker,
+ Some(entry) => !is_modified && entry.status == FileStatus::UpToDate,
+ };
+ if is_skipped {
+ return;
+ }
+ }
+ // A file which can't be read -- another program may have it locked, for
+ // example -- leaves the cache as it is; the watcher reports the file again
+ // once it changes.
+ let Ok(contents) = read_text(path) else {
+ return;
+ };
+ // The lexer requires Unix-style line endings.
+ //
+ // TODO: once the Server has user-selected options, implement the spec's
+ // `SaveOnScan` option here: translate the result back to source, then write
+ // any changes to the file and update its cached stamp.
+ let outcome = match contents.map(|contents| {
+ source_to_codechat_for_web(
+ &contents.replace("\r\n", "\n"),
+ path,
+ 0.0,
+ false,
+ Some(cache.clone()),
+ )
+ }) {
+ Some(Ok(_)) => ScanOutcome::Translated,
+ None
+ | Some(Err(
+ SourceToCodeChatForWebError::NoLexer | SourceToCodeChatForWebError::UnknownLexer(_),
+ )) => ScanOutcome::Untranslatable,
+ Some(Err(err)) => {
+ warn!("Unable to scan {}: {err}", path.display());
+ ScanOutcome::Failed
+ }
+ };
+ let mut cache = cache.lock().unwrap();
+ if cache.is_current_file(path) {
+ // A connection made this file its current file during the translation,
+ // which may then have committed the file's content on disk over the
+ // content the connection shows, which may be newer. Have the connection
+ // re-translate its content to restore it.
+ cache.notify_current_file_outdated(&HashSet::from([path.to_path_buf()]));
+ } else {
+ match outcome {
+ // The translation's commit recorded a stamp taken as of the commit,
+ // which follows the reading of its text. A write in between would
+ // leave the cache describing the old text with the new stamp, so
+ // the consumer would skip the watcher's report of that write.
+ // Record the stamp taken before the text was read instead, which no
+ // such write leaves the file matching.
+ ScanOutcome::Translated => {
+ if let Some(entry) = cache.files.get_mut(path) {
+ entry.stamp = Some(stamp);
+ }
+ }
+ // A file which can't be translated defines nothing. Committing it
+ // as such withdraws whatever it defined before it stopped being
+ // translatable, and records its stamp, so that later walks skip it
+ // until it changes rather than reading it again.
+ ScanOutcome::Untranslatable => {
+ cache.commit_file(path, Some(stamp), FileFacts::default());
+ }
+ // The translation may have failed after committing the file's
+ // facts, along with a stamp which matches the file; discard that
+ // stamp, so that the next walk tries again rather than skipping a
+ // file whose processing never finished.
+ ScanOutcome::Failed => {
+ if let Some(entry) = cache.files.get_mut(path) {
+ entry.stamp = None;
+ }
+ }
+ }
+ }
+ if is_toc_changed {
+ cache.notify_toc(CacheNotification::TocChanged);
+ }
+}
+
+/// Read a file as text. The walker sends the consumer every file in the
+/// project, including any large media files, so a file is read in full only
+/// once its first block proves to be text.
+fn read_text(
+ // The file to read.
+ path: &Path,
+ // Its contents; `None` if it isn't UTF-8, which most likely makes it a
+ // binary file.
+) -> io::Result> {
+ // The size of the block checked before reading the rest of the file.
+ const FIRST_BLOCK_LEN: u64 = 8192;
+ let mut file = File::open(path)?;
+ let mut bytes = Vec::new();
+ file.by_ref()
+ .take(FIRST_BLOCK_LEN)
+ .read_to_end(&mut bytes)?;
+ // A character may straddle the end of the block, leaving an incomplete
+ // sequence there, for which `error_len` returns `None`.
+ if let Err(err) = str::from_utf8(&bytes)
+ && err.error_len().is_some()
+ {
+ return Ok(None);
+ }
+ file.read_to_end(&mut bytes)?;
+ Ok(String::from_utf8(bytes).ok())
+}
+
impl Default for Cache {
fn default() -> Self {
Cache::new(PathBuf::new())
@@ -854,10 +1978,31 @@ impl Default for Cache {
mod tests {
use std::{
collections::{HashMap, HashSet},
+ fs,
+ num::NonZeroUsize,
path::{Path, PathBuf},
+ sync::{
+ Arc, Mutex,
+ mpsc::{self as std_mpsc, Receiver},
+ },
+ thread::sleep,
+ time::{Duration, Instant, SystemTime},
};
- use super::{Cache, FileFacts, FileStatus, FragmentFact, IdResolution, IdState, TargetFact};
+ use notify_debouncer_full::notify::{
+ Event, EventKind,
+ event::{AccessKind, CreateKind, DataChange, Flag, ModifyKind, RemoveKind, RenameMode},
+ };
+ use test_utils::prep_test_dir;
+ use tokio::sync::mpsc::{self, error::TryRecvError};
+
+ use super::{
+ Cache, CacheNotification, ConnectionRegistration, Consumer, ConsumerPathState, FileFacts,
+ FileStamp, FileStatus, FragmentFact, IdResolution, IdState, LRU_ENTRIES_PER_CONNECTION,
+ ProjectCaches, RACY_WINDOW, TargetFact, Walker, WatcherPathState, cache_map_capacity,
+ get_or_insert_cache, set_cache_map_capacity, watcher_path_states,
+ };
+ use crate::webserver::canonicalize;
// ### Test helpers
//
@@ -867,7 +2012,7 @@ mod tests {
id: id.to_string(),
inner_html: inner_html.to_string(),
line: 0,
- doc_block_index: 0,
+ _doc_block_index: 0,
gather_ids: vec![],
}
}
@@ -885,8 +2030,8 @@ mod tests {
FragmentFact {
id: id.to_string(),
line: 0,
- doc_block_start_index: 0,
- code_doc_block_end_index: 1,
+ _doc_block_start_index: 0,
+ _code_doc_block_end_index: 1,
}
}
@@ -937,6 +2082,51 @@ mod tests {
paths.iter().map(|path| (*path).clone()).collect()
}
+ // Collect every message sent so far on a channel.
+ fn drain(rx: &Receiver) -> Vec {
+ rx.try_iter().collect()
+ }
+
+ // Build the message the walker sends the consumer for a file.
+ fn found(path: &Path) -> ConsumerPathState {
+ ConsumerPathState::WatcherPathState(WatcherPathState::CreatedOrModified(path.to_path_buf()))
+ }
+
+ // Register a connection whose current file is `current_file`, returning its
+ // registration and the receiver of its notifications.
+ fn register(
+ cache: &Arc>,
+ connection_id: &str,
+ current_file: &Path,
+ ) -> (
+ ConnectionRegistration,
+ mpsc::UnboundedReceiver,
+ ) {
+ let (notify_tx, notify_rx) = mpsc::unbounded_channel();
+ let registration = ConnectionRegistration::new(
+ cache,
+ connection_id.to_string(),
+ current_file.to_path_buf(),
+ notify_tx,
+ );
+ (registration, notify_rx)
+ }
+
+ fn set_modified(path: &Path, modified: SystemTime) {
+ fs::File::options()
+ .write(true)
+ .open(path)
+ .unwrap()
+ .set_modified(modified)
+ .unwrap();
+ }
+
+ // Move a file's modification time back past `RACY_WINDOW`, as for a file
+ // which wasn't just written, so that a stamp taken of it can match.
+ fn backdate(path: &Path) {
+ set_modified(path, SystemTime::now() - 2 * RACY_WINDOW);
+ }
+
// ### Cross-reference tests
//
// Verify that a cross-reference to an existing target records the
@@ -1049,6 +2239,42 @@ mod tests {
assert!(!cache.ids.contains_key("nobody"));
}
+ // Verify that an id a file stops defining is forgotten once nothing
+ // references it -- but not while anything does, including the file which
+ // stopped defining it.
+ #[test]
+ fn test_undefined_id_forgotten() {
+ let mut cache = Cache::default();
+ let a = PathBuf::from("a.py");
+
+ // A target which nothing references is forgotten once deleted...
+ cache.commit_file(&a, None, facts_target("foo", "Foo!"));
+ cache.commit_file(&a, None, FileFacts::default());
+ assert!(!cache.ids.contains_key("foo"));
+
+ // ...as is a fragment in a file which is removed.
+ cache.commit_file(&a, None, facts_fragment("frag"));
+ cache.remove_file(&a);
+ assert!(cache.ids.is_empty());
+
+ // A file which references its own target keeps the id after deleting
+ // the target, until the reference goes too.
+ cache.commit_file(
+ &a,
+ None,
+ FileFacts {
+ targets: vec![target_fact("foo", "Foo!")],
+ xrefs: vec!["foo".to_string()],
+ ..Default::default()
+ },
+ );
+ cache.commit_file(&a, None, facts_xref("foo"));
+ assert!(matches!(cache.resolve_id("foo"), IdResolution::Missing));
+ assert_eq!(*dependents(&cache, "foo"), paths(&[&a]));
+ cache.commit_file(&a, None, FileFacts::default());
+ assert!(!cache.ids.contains_key("foo"));
+ }
+
// ### File lifecycle tests
//
// Verify that removing a file withdraws both its definitions and its
@@ -1075,28 +2301,6 @@ mod tests {
assert!(cache.files.is_empty());
}
- // Verify that `pending_files` holds exactly those files whose status is
- // `Unknown`.
- #[test]
- fn test_pending_files() {
- let mut cache = Cache::default();
- let a = PathBuf::from("a.md");
- cache.commit_file(&a, None, facts_target("foo", "Foo!"));
- assert!(cache.pending_files.is_empty());
-
- // A file written or modified since it was processed is queued for
- // reprocessing, and queued only once.
- cache.mark_unknown(&a);
- cache.mark_unknown(&a);
- assert_eq!(*status(&cache, &a), FileStatus::Unknown);
- assert_eq!(cache.pending_files, vec![a.clone()]);
-
- // Reprocessing it dequeues it.
- cache.commit_file(&a, None, facts_target("foo", "Foo!"));
- assert_eq!(*status(&cache, &a), FileStatus::UpToDate);
- assert!(cache.pending_files.is_empty());
- }
-
// Verify that an auto-assigned id is 10 characters drawn from
// `[A-Za-z0-9_-]` and collides neither with a cached id nor with another id
// assigned in the same pass.
@@ -1120,6 +2324,26 @@ mod tests {
}
}
+ // Verify that the files beneath a directory are found, at any depth, and
+ // that no file merely sharing a prefix of the directory's name is.
+ #[test]
+ fn test_files_beneath() {
+ let mut cache = Cache::default();
+ let beneath = [PathBuf::from("a/b/c.md"), PathBuf::from("a/b/d/e.md")];
+ let not_beneath = [
+ PathBuf::from("a/b"),
+ PathBuf::from("a/b.md"),
+ PathBuf::from("a/bc/f.md"),
+ PathBuf::from("g.md"),
+ ];
+ for path in beneath.iter().chain(not_beneath.iter()) {
+ cache.commit_file(path, None, FileFacts::default());
+ }
+ let found: Vec<&PathBuf> = cache.files_beneath(Path::new("a/b")).collect();
+ assert_eq!(found, beneath.iter().collect::>());
+ assert_eq!(cache.files_beneath(Path::new("h")).count(), 0);
+ }
+
// ### Duplicate id tests
//
// Verify that duplicate ids are recorded as such, in one file or across
@@ -1449,4 +2673,824 @@ mod tests {
let outdated = cache.update_fragment_content(&a, "frag", "content".to_string());
assert!(outdated.is_empty());
}
+
+ // ### Connection tests
+ //
+ // Verify that a registered connection is notified only when its current
+ // file is outdated, and not after its registration is dropped.
+ #[test]
+ fn test_connection_notification() {
+ let cache = Arc::new(Mutex::new(Cache::default()));
+ let a = PathBuf::from("a.md");
+ let b = PathBuf::from("b.md");
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&b, None, facts_xref("foo"));
+ let (registration, mut notify_rx) = register(&cache, "conn", &b);
+
+ // Defining the id `b` references outdates `b`.
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&a, None, facts_target("foo", "Foo!"));
+ assert_eq!(
+ notify_rx.try_recv(),
+ Ok(CacheNotification::CurrentFileOutdated)
+ );
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+
+ // Recommitting `b` itself outdates nothing.
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&b, None, facts_xref("foo"));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+
+ // Dropping the registration drops the cache's only sender.
+ drop(registration);
+ assert!(cache.lock().unwrap().connections.is_empty());
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&a, None, facts_target("foo", "Bar!"));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Disconnected));
+
+ // A registration doesn't keep its cache alive, and outlives it
+ // harmlessly.
+ let (registration, _notify_rx) = register(&cache, "conn", &b);
+ let weak_cache = Arc::downgrade(&cache);
+ drop(cache);
+ assert!(weak_cache.upgrade().is_none());
+ drop(registration);
+ }
+
+ // Verify that outdating the TOC notifies every registered connection, since
+ // each shows the TOC in its sidebar, and that recommitting the TOC itself
+ // notifies none of them.
+ #[test]
+ fn test_toc_notification() {
+ let cache = Arc::new(Mutex::new(Cache::default()));
+ // The default cache's root is empty, so this is its TOC.
+ let toc = PathBuf::from("toc.md");
+ let a = PathBuf::from("a.md");
+ let b = PathBuf::from("b.md");
+ // The TOC cross-references a heading in `a`.
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&a, None, facts_target("foo", "Foo"));
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&toc, None, facts_xref("foo"));
+ let (_definer, mut definer_rx) = register(&cache, "definer", &a);
+ let (_bystander, mut bystander_rx) = register(&cache, "bystander", &b);
+ let (_toc_viewer, mut toc_viewer_rx) = register(&cache, "toc_viewer", &toc);
+
+ // Renaming the heading outdates the TOC. A connection whose current
+ // file is the TOC must re-translate it as well.
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&a, None, facts_target("foo", "Bar"));
+ for notify_rx in [&mut definer_rx, &mut bystander_rx] {
+ assert_eq!(notify_rx.try_recv(), Ok(CacheNotification::TocOutdated));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+ }
+ assert_eq!(
+ toc_viewer_rx.try_recv(),
+ Ok(CacheNotification::CurrentFileOutdated)
+ );
+ assert_eq!(toc_viewer_rx.try_recv(), Ok(CacheNotification::TocOutdated));
+ assert_eq!(toc_viewer_rx.try_recv(), Err(TryRecvError::Empty));
+
+ // Every sidebar load translates, and therefore recommits, the TOC; this
+ // must not outdate the TOC again.
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&toc, None, facts_xref("foo"));
+ for notify_rx in [&mut definer_rx, &mut bystander_rx, &mut toc_viewer_rx] {
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+ }
+ }
+
+ // Verify that an outdated file goes to the consumer unless a connection
+ // shows it, as its current file or as the TOC; a connection re-renders
+ // those itself.
+ #[test]
+ fn test_outdated_files_go_to_consumer() {
+ let cache = Arc::new(Mutex::new(Cache::default()));
+ let (consumer_tx, consumer_rx) = std_mpsc::channel();
+ cache.lock().unwrap().consumer_tx = Some(consumer_tx);
+ let toc = PathBuf::from("toc.md");
+ let a = PathBuf::from("a.md");
+ let b = PathBuf::from("b.md");
+ let c = PathBuf::from("c.md");
+ for referrer in [&toc, &b, &c] {
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(referrer, None, facts_xref("foo"));
+ }
+
+ // With no connections, the consumer reprocesses every outdated file,
+ // the TOC included.
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&a, None, facts_target("foo", "Foo"));
+ let mut sent = drain(&consumer_rx);
+ sent.sort_by_key(|message| format!("{message:?}"));
+ assert_eq!(
+ sent,
+ vec![
+ ConsumerPathState::Rescan(b.clone()),
+ ConsumerPathState::Rescan(c.clone()),
+ ConsumerPathState::Rescan(toc.clone()),
+ ]
+ );
+
+ // A connection showing `c` re-renders `c` and the TOC; the consumer
+ // reprocesses only `b`.
+ let (_registration, mut notify_rx) = register(&cache, "conn", &c);
+ cache
+ .lock()
+ .unwrap()
+ .commit_file(&a, None, facts_target("foo", "Bar"));
+ assert_eq!(drain(&consumer_rx), vec![ConsumerPathState::Rescan(b)]);
+ let mut notifications = vec![notify_rx.try_recv().unwrap(), notify_rx.try_recv().unwrap()];
+ notifications.sort_by_key(|notification| format!("{notification:?}"));
+ assert_eq!(
+ notifications,
+ vec![
+ CacheNotification::CurrentFileOutdated,
+ CacheNotification::TocOutdated
+ ]
+ );
+ }
+
+ // Verify that the `CacheMap` holds `LRU_ENTRIES_PER_CONNECTION` caches per
+ // connection, plus one.
+ #[test]
+ fn test_cache_map_capacity() {
+ assert_eq!(cache_map_capacity(0).get(), 1);
+ assert_eq!(
+ cache_map_capacity(3).get(),
+ 3 * LRU_ENTRIES_PER_CONNECTION + 1
+ );
+ }
+
+ // Verify that the `CacheMap` evicts its least recently used caches, except
+ // for those in which a connection is registered.
+ #[test]
+ fn test_cache_map_eviction() {
+ let a = PathBuf::from("a");
+ let b = PathBuf::from("b");
+ let c = PathBuf::from("c");
+ let d = PathBuf::from("d");
+ let mut caches = ProjectCaches::new(NonZeroUsize::new(3).unwrap());
+ let insert = |caches: &mut ProjectCaches, root: &PathBuf| {
+ get_or_insert_cache(caches, root, || {
+ Arc::new(Mutex::new(Cache::new(root.clone())))
+ })
+ };
+ let roots = |caches: &ProjectCaches| {
+ let mut roots: Vec = caches.iter().map(|(root, _)| root.clone()).collect();
+ roots.sort();
+ roots
+ };
+ let cache_a = insert(&mut caches, &a);
+ insert(&mut caches, &b);
+ insert(&mut caches, &c);
+ // Getting an existing cache returns it, rather than a new one.
+ assert!(Arc::ptr_eq(&insert(&mut caches, &a), &cache_a));
+
+ // A connection in the least recently used cache keeps it from being
+ // evicted, so the next least recently used is evicted instead.
+ let (_registration, _notify_rx) = register(&cache_a, "conn", &a.join("x.md"));
+ insert(&mut caches, &b);
+ insert(&mut caches, &c);
+ insert(&mut caches, &d);
+ assert_eq!(roots(&caches), vec![a.clone(), c.clone(), d.clone()]);
+
+ // Shrinking the map likewise keeps the cache with a connection.
+ set_cache_map_capacity(&mut caches, NonZeroUsize::MIN);
+ assert_eq!(roots(&caches), vec![a.clone()]);
+ assert_eq!(caches.cap(), NonZeroUsize::MIN);
+
+ // When only caches with connections remain, the map grows rather than
+ // evict one.
+ let cache_d = insert(&mut caches, &d);
+ let (_registration, _notify_rx) = register(&cache_d, "conn2", &d.join("x.md"));
+ set_cache_map_capacity(&mut caches, NonZeroUsize::MIN);
+ assert_eq!(roots(&caches), vec![a.clone(), d.clone()]);
+ insert(&mut caches, &b);
+ assert_eq!(roots(&caches), vec![a, b, d]);
+ }
+
+ // ### Watcher tests
+ //
+ // Verify that watcher events become the paths the walker receives: renames
+ // become a deletion then a creation, a modified directory is skipped, and
+ // lost events request a walk of the entire project.
+ #[test]
+ fn test_watcher_path_states() {
+ use WatcherPathState::{CreatedOrModified, Deleted};
+
+ let (temp_dir, root) = prep_test_dir!();
+ let file = root.join("file.md");
+ fs::write(&file, "").unwrap();
+ let missing = root.join("missing.md");
+ let translate = |kind: EventKind, event_paths: &[&PathBuf]| {
+ let event = event_paths.iter().fold(Event::new(kind), |event, path| {
+ event.add_path((*path).clone())
+ });
+ watcher_path_states(&event, &root)
+ };
+
+ assert_eq!(
+ translate(EventKind::Create(CreateKind::File), &[&file]),
+ vec![CreatedOrModified(file.clone())]
+ );
+ assert_eq!(
+ translate(EventKind::Remove(RemoveKind::Any), &[&missing]),
+ vec![Deleted(missing.clone())]
+ );
+ assert_eq!(
+ translate(
+ EventKind::Modify(ModifyKind::Name(RenameMode::Both)),
+ &[&missing, &file]
+ ),
+ vec![Deleted(missing.clone()), CreatedOrModified(file.clone())]
+ );
+ assert_eq!(
+ translate(
+ EventKind::Modify(ModifyKind::Name(RenameMode::From)),
+ &[&missing]
+ ),
+ vec![Deleted(missing.clone())]
+ );
+ assert_eq!(
+ translate(
+ EventKind::Modify(ModifyKind::Name(RenameMode::To)),
+ &[&file]
+ ),
+ vec![CreatedOrModified(file.clone())]
+ );
+ // An unclassified rename is resolved by checking for the path.
+ assert_eq!(
+ translate(
+ EventKind::Modify(ModifyKind::Name(RenameMode::Any)),
+ &[&file, &missing]
+ ),
+ vec![CreatedOrModified(file.clone()), Deleted(missing.clone())]
+ );
+ assert_eq!(
+ translate(
+ EventKind::Modify(ModifyKind::Data(DataChange::Any)),
+ &[&file, &root]
+ ),
+ vec![CreatedOrModified(file.clone())]
+ );
+ assert_eq!(
+ translate(EventKind::Access(AccessKind::Any), &[&file]),
+ vec![]
+ );
+ let rescan = Event::new(EventKind::Other).set_flag(Flag::Rescan);
+ assert_eq!(
+ watcher_path_states(&rescan, &root),
+ vec![CreatedOrModified(root.clone())]
+ );
+
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
+
+ // ### Walker tests
+ //
+ // Verify that the walker forwards only those paths its ignore files don't
+ // exclude, walks each directory reported, and re-walks after an ignore file
+ // changes. The fixture holds hidden files, and files excluded by an
+ // `.ignore` file.
+ #[test]
+ fn test_walk_project() {
+ let (temp_dir, test_dir) = prep_test_dir!();
+ let root = canonicalize(&test_dir).unwrap();
+ let a = root.join("a.md");
+ let ignored = root.join("ignored.md");
+ let b = root.join("sub").join("b.py");
+ let sub_ignored = root.join("sub").join("ignored.md");
+ // Confirm that copying the fixture preserved the hidden files, so their
+ // absence from the walk below is the walker's doing.
+ assert!(root.join(".ignore").is_file());
+ assert!(root.join(".hidden").join("c.md").is_file());
+
+ let (consumer_tx, consumer_rx) = std_mpsc::channel();
+ let mut walker = Walker::new(root.clone(), consumer_tx);
+ // Process one path from the watcher, returning everything the walker
+ // sends the consumer as a result. The order in which a walk finds files
+ // varies by platform, so sort the files found.
+ let mut process = |path_state: WatcherPathState| {
+ walker.process(path_state).unwrap();
+ let mut sent = drain(&consumer_rx);
+ if let [
+ ConsumerPathState::WalkStarted(_),
+ files @ ..,
+ ConsumerPathState::WalkFinished,
+ ] = sent.as_mut_slice()
+ {
+ files.sort_by_key(|message| format!("{message:?}"));
+ }
+ sent
+ };
+ // The messages a walk of `dir` which finds `files` sends.
+ let walk = |dir: &PathBuf, files: &[&PathBuf]| {
+ let mut sent = vec![ConsumerPathState::WalkStarted(dir.clone())];
+ let mut files: Vec<&PathBuf> = files.to_vec();
+ files.sort();
+ sent.extend(files.into_iter().map(|file| found(file)));
+ sent.push(ConsumerPathState::WalkFinished);
+ sent
+ };
+
+ // The initial walk skips hidden and ignored files.
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(root.clone())),
+ walk(&root, &[&a, &b])
+ );
+
+ // A modified file is forwarded, unless it's ignored or hidden.
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(a.clone())),
+ vec![found(&a)]
+ );
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(ignored.clone())),
+ vec![]
+ );
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(
+ root.join(".hidden").join("c.md")
+ )),
+ vec![]
+ );
+
+ // Likewise for a deleted path.
+ let gone = root.join("gone.md");
+ assert_eq!(
+ process(WatcherPathState::Deleted(gone.clone())),
+ vec![ConsumerPathState::WatcherPathState(
+ WatcherPathState::Deleted(gone)
+ )]
+ );
+ assert_eq!(process(WatcherPathState::Deleted(ignored.clone())), vec![]);
+
+ // A created directory is walked, unless it's hidden.
+ let new_dir = root.join("new_dir");
+ let d = new_dir.join("d.md");
+ let new_dir_ignored = new_dir.join("ignored.md");
+ fs::create_dir(&new_dir).unwrap();
+ fs::write(&d, "# d.md").unwrap();
+ fs::write(&new_dir_ignored, "# Ignored.").unwrap();
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(new_dir.clone())),
+ walk(&new_dir, &[&d])
+ );
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(root.join(".hidden"))),
+ vec![]
+ );
+
+ // An ignore file inside an ignored directory can't change what the walk
+ // finds, so it prompts no walk.
+ let hidden_ignore = root.join(".hidden").join(".ignore");
+ fs::write(&hidden_ignore, "c.md\n").unwrap();
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(hidden_ignore)),
+ vec![]
+ );
+
+ // Changing an `.ignore` file reloads the ignore files, then re-walks
+ // the directory it governs.
+ fs::write(root.join(".ignore"), "a.md\n").unwrap();
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(root.join(".ignore"))),
+ walk(&root, &[&ignored, &b, &sub_ignored, &d, &new_dir_ignored])
+ );
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(a.clone())),
+ vec![]
+ );
+
+ // Git's ignore files apply only within a repository.
+ fs::create_dir_all(root.join(".git").join("info")).unwrap();
+ fs::write(new_dir.join(".gitignore"), "d.md\n").unwrap();
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(
+ new_dir.join(".gitignore")
+ )),
+ walk(&new_dir, &[&new_dir_ignored])
+ );
+ // A change to `.git/info/exclude` re-walks the entire project.
+ let exclude = root.join(".git").join("info").join("exclude");
+ fs::write(&exclude, "b.py\n").unwrap();
+ assert_eq!(
+ process(WatcherPathState::CreatedOrModified(exclude)),
+ walk(&root, &[&ignored, &sub_ignored, &new_dir_ignored])
+ );
+
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
+
+ // ### File stamp tests
+ //
+ // Verify that a stamp taken soon after its file was written never matches,
+ // while a stamp of an older file matches until the file changes.
+ #[test]
+ fn test_file_stamp() {
+ let (temp_dir, test_dir) = prep_test_dir!();
+ let path = test_dir.join("a.md");
+ let stamp = |path: &Path| FileStamp::new(&fs::metadata(path).unwrap());
+
+ // Simulate a second write in the same timestamp tick as the first,
+ // which leaves the length and modification time unchanged.
+ fs::write(&path, "one").unwrap();
+ let racy = stamp(&path);
+ let modified = fs::metadata(&path).unwrap().modified().unwrap();
+ fs::write(&path, "two").unwrap();
+ set_modified(&path, modified);
+ let current = stamp(&path);
+ // Only the racy rule tells these apart on Windows; on Unix, the write
+ // also changes the change time.
+ #[cfg(not(unix))]
+ assert!(racy.stat == current.stat);
+ assert!(!racy.is_unmodified(¤t));
+
+ backdate(&path);
+ let trusted = stamp(&path);
+ assert!(trusted.is_unmodified(&stamp(&path)));
+ fs::write(&path, "three").unwrap();
+ assert!(!trusted.is_unmodified(&stamp(&path)));
+
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
+
+ // ### Consumer tests
+ //
+ // Verify that the consumer scans the files the walker sends into the cache,
+ // rescanning one only when it's changed or outdated, and leaves a
+ // connection's current file to that connection.
+ #[test]
+ fn test_consumer_scans() {
+ let (temp_dir, test_dir) = prep_test_dir!();
+ let root = canonicalize(&test_dir).unwrap();
+ let a = root.join("a.md");
+ let b = root.join("b.md");
+ fs::write(&a, "
Foo
\n").unwrap();
+ fs::write(&b, "See .\n").unwrap();
+ backdate(&a);
+ backdate(&b);
+ let cache = Arc::new(Mutex::new(Cache::new(root.clone())));
+ let (consumer_tx, consumer_rx) = std_mpsc::channel();
+ cache.lock().unwrap().consumer_tx = Some(consumer_tx);
+ let mut consumer = Consumer::default();
+ let inner_html = |cache: &Arc>| {
+ cache.lock().unwrap().files[&a].targets["foo"]
+ .inner_html
+ .clone()
+ };
+
+ consumer.apply(&cache, found(&a));
+ consumer.apply(&cache, found(&b));
+ assert_eq!(inner_html(&cache), "Foo");
+ assert_eq!(*dependents(&cache.lock().unwrap(), "foo"), paths(&[&b]));
+ assert_eq!(*status(&cache.lock().unwrap(), &b), FileStatus::UpToDate);
+
+ // An unmodified file isn't rescanned. To observe this, alter its cached
+ // target, which a rescan would restore.
+ cache
+ .lock()
+ .unwrap()
+ .files
+ .get_mut(&a)
+ .unwrap()
+ .targets
+ .get_mut("foo")
+ .unwrap()
+ .inner_html = "Stale".to_string();
+ consumer.apply(&cache, found(&a));
+ assert_eq!(inner_html(&cache), "Stale");
+
+ // A modified file is rescanned, which outdates the file referencing it;
+ // the consumer is sent that file.
+ fs::write(&a, "Foo bar
\n").unwrap();
+ consumer.apply(&cache, found(&a));
+ assert_eq!(inner_html(&cache), "Foo bar");
+ assert_eq!(*status(&cache.lock().unwrap(), &b), FileStatus::Outdated);
+ assert_eq!(
+ drain(&consumer_rx),
+ vec![ConsumerPathState::Rescan(b.clone())]
+ );
+ // Scanning the outdated file brings it up to date, so a second request,
+ // as when several commits outdate one file, isn't scanned. To observe
+ // this, discard its cached cross-references, which a rescan restores.
+ consumer.apply(&cache, ConsumerPathState::Rescan(b.clone()));
+ assert_eq!(*status(&cache.lock().unwrap(), &b), FileStatus::UpToDate);
+ cache
+ .lock()
+ .unwrap()
+ .files
+ .get_mut(&b)
+ .unwrap()
+ .xrefs
+ .clear();
+ consumer.apply(&cache, ConsumerPathState::Rescan(b.clone()));
+ assert!(cache.lock().unwrap().files[&b].xrefs.is_empty());
+ // A request for a file the cache has since forgotten is moot.
+ let forgotten = root.join("forgotten.md");
+ fs::write(&forgotten, "# Forgotten\n").unwrap();
+ consumer.apply(&cache, ConsumerPathState::Rescan(forgotten.clone()));
+ assert!(!cache.lock().unwrap().files.contains_key(&forgotten));
+
+ // A connection's current file isn't scanned...
+ let (registration, mut notify_rx) = register(&cache, "conn", &a);
+ fs::write(&a, "Foo baz
\n").unwrap();
+ consumer.apply(&cache, found(&a));
+ assert_eq!(inner_html(&cache), "Foo bar");
+ // ...nor forgotten when it's deleted: the connection's IDE may still
+ // hold its text, so the connection is told to re-translate it.
+ fs::remove_file(&a).unwrap();
+ consumer.apply(
+ &cache,
+ ConsumerPathState::WatcherPathState(WatcherPathState::Deleted(a.clone())),
+ );
+ assert!(cache.lock().unwrap().files.contains_key(&a));
+ assert_eq!(
+ notify_rx.try_recv(),
+ Ok(CacheNotification::CurrentFileOutdated)
+ );
+ // Once no connection shows the file, the consumer is asked to check it;
+ // this finds it gone, so it's forgotten, which outdates the file
+ // referencing it.
+ drop(registration);
+ assert_eq!(
+ drain(&consumer_rx),
+ vec![ConsumerPathState::Rescan(a.clone())]
+ );
+ consumer.apply(&cache, ConsumerPathState::Rescan(a.clone()));
+ assert!(!cache.lock().unwrap().files.contains_key(&a));
+ assert_eq!(
+ drain(&consumer_rx),
+ vec![ConsumerPathState::Rescan(b.clone())]
+ );
+
+ // A file which isn't text, or which no lexer recognizes, defines
+ // nothing; it's cached as such, so that it isn't read again until it
+ // changes. One with a lexer directive is translated, whatever its
+ // extension.
+ let binary = root.join("binary.md");
+ let unknown = root.join("unknown.xyz");
+ let directive = root.join("directive.xyz");
+ fs::write(&binary, [0xff, 0xfe, 0x00]).unwrap();
+ fs::write(&unknown, "Unknown
\n").unwrap();
+ fs::write(
+ &directive,
+ // Split this so the lexer doesn't get set for this file.
+ concat!(
+ "# CodeChat Editor ",
+ "lexer: python\n# Directive
\n"
+ ),
+ )
+ .unwrap();
+ for path in [&binary, &unknown, &directive] {
+ consumer.apply(&cache, found(path));
+ }
+ {
+ let cache_guard = cache.lock().unwrap();
+ for path in [&binary, &unknown] {
+ assert!(cache_guard.files[path].targets.is_empty());
+ assert_eq!(*status(&cache_guard, path), FileStatus::UpToDate);
+ }
+ assert!(matches!(
+ cache_guard.resolve_id("unknown"),
+ IdResolution::Missing
+ ));
+ assert!(matches!(
+ cache_guard.resolve_id("directive"),
+ IdResolution::Target { .. }
+ ));
+ }
+ // A file which stops being translatable withdraws its definitions.
+ fs::write(&directive, "# Directive
\n").unwrap();
+ consumer.apply(&cache, found(&directive));
+ assert!(matches!(
+ cache.lock().unwrap().resolve_id("directive"),
+ IdResolution::Missing
+ ));
+
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
+
+ // Verify that the consumer tells every connection to re-load the TOC when
+ // its text changes on disk -- even when the TOC is a connection's current
+ // file, which the consumer leaves to that connection -- and when it's
+ // deleted.
+ #[test]
+ fn test_consumer_toc_changed() {
+ let (temp_dir, test_dir) = prep_test_dir!();
+ let root = canonicalize(&test_dir).unwrap();
+ let toc = root.join("toc.md");
+ let a = root.join("a.md");
+ fs::write(&toc, "# Contents\n").unwrap();
+ let cache = Arc::new(Mutex::new(Cache::new(root.clone())));
+ let mut consumer = Consumer::default();
+ let (_registration, mut notify_rx) = register(&cache, "conn", &a);
+ let is_target = |cache: &Arc>, id: &str| {
+ matches!(
+ cache.lock().unwrap().resolve_id(id),
+ IdResolution::Target { .. }
+ )
+ };
+
+ // No sidebar can show a TOC the cache has no record of, since loading
+ // the TOC in a sidebar records it, so its first scan is not a change.
+ consumer.apply(&cache, found(&toc));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+
+ // A changed TOC is scanned, then reported; an unchanged one isn't.
+ fs::write(&toc, "Contents
\n").unwrap();
+ backdate(&toc);
+ consumer.apply(&cache, found(&toc));
+ assert!(is_target(&cache, "contents"));
+ assert_eq!(notify_rx.try_recv(), Ok(CacheNotification::TocChanged));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+ consumer.apply(&cache, found(&toc));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+
+ // When the TOC is a connection's current file, the change isn't
+ // scanned, but is still reported to every connection.
+ let (_toc_registration, mut toc_notify_rx) = register(&cache, "toc_conn", &toc);
+ fs::write(&toc, "Contents
\n").unwrap();
+ consumer.apply(&cache, found(&toc));
+ assert!(is_target(&cache, "contents"));
+ for notify_rx in [&mut notify_rx, &mut toc_notify_rx] {
+ assert_eq!(notify_rx.try_recv(), Ok(CacheNotification::TocChanged));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+ }
+
+ // Deleting the TOC is reported as a change, too. The connection whose
+ // current file it is keeps the TOC in the cache by re-translating it.
+ fs::remove_file(&toc).unwrap();
+ consumer.apply(
+ &cache,
+ ConsumerPathState::WatcherPathState(WatcherPathState::Deleted(toc.clone())),
+ );
+ assert!(cache.lock().unwrap().files.contains_key(&toc));
+ assert_eq!(notify_rx.try_recv(), Ok(CacheNotification::TocChanged));
+ assert_eq!(notify_rx.try_recv(), Err(TryRecvError::Empty));
+ assert_eq!(
+ toc_notify_rx.try_recv(),
+ Ok(CacheNotification::CurrentFileOutdated)
+ );
+ assert_eq!(toc_notify_rx.try_recv(), Ok(CacheNotification::TocChanged));
+ assert_eq!(toc_notify_rx.try_recv(), Err(TryRecvError::Empty));
+
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
+
+ // Verify that the consumer forgets deleted files and directories, and the
+ // cached files a walk doesn't find.
+ #[test]
+ fn test_consumer_forgets() {
+ let (temp_dir, test_dir) = prep_test_dir!();
+ let root = canonicalize(&test_dir).unwrap();
+ let sub = root.join("sub");
+ fs::create_dir(&sub).unwrap();
+ let a = root.join("a.md");
+ let b = sub.join("b.md");
+ let c = sub.join("c.md");
+ let d = sub.join("d.md");
+ for path in [&a, &b, &c, &d] {
+ fs::write(path, "# Text\n").unwrap();
+ }
+ let cache = Arc::new(Mutex::new(Cache::new(root.clone())));
+ let mut consumer = Consumer::default();
+ for path in [&a, &b, &c, &d] {
+ consumer.apply(&cache, found(path));
+ }
+ let cached = |cache: &Arc>| {
+ cache
+ .lock()
+ .unwrap()
+ .files
+ .keys()
+ .cloned()
+ .collect::>()
+ };
+ assert_eq!(
+ cached(&cache),
+ vec![a.clone(), b.clone(), c.clone(), d.clone()]
+ );
+
+ // A walk which doesn't find a cached file means it no longer exists.
+ // Files outside the directory walked are unaffected.
+ fs::remove_file(&c).unwrap();
+ consumer.apply(&cache, ConsumerPathState::WalkStarted(sub.clone()));
+ consumer.apply(&cache, found(&b));
+ consumer.apply(&cache, found(&d));
+ consumer.apply(&cache, ConsumerPathState::WalkFinished);
+ assert_eq!(cached(&cache), vec![a.clone(), b.clone(), d.clone()]);
+
+ // A file a connection stopped showing during a walk wasn't found by
+ // that walk, so the walk still forgets it -- as it should if, say, the
+ // walk skipped the file as ignored.
+ consumer.apply(&cache, ConsumerPathState::WalkStarted(sub.clone()));
+ consumer.apply(&cache, found(&b));
+ consumer.apply(&cache, ConsumerPathState::Rescan(d.clone()));
+ consumer.apply(&cache, ConsumerPathState::WalkFinished);
+ assert_eq!(cached(&cache), vec![a.clone(), b.clone()]);
+
+ // A deletion reported for a path which exists again, as after a version
+ // control checkout, rescans the file rather than forgetting it.
+ fs::write(&a, "Recreated
\n").unwrap();
+ consumer.apply(
+ &cache,
+ ConsumerPathState::WatcherPathState(WatcherPathState::Deleted(a.clone())),
+ );
+ assert_eq!(cached(&cache), vec![a.clone(), b.clone()]);
+ assert!(matches!(
+ cache.lock().unwrap().resolve_id("recreated"),
+ IdResolution::Target { .. }
+ ));
+
+ // A deleted directory takes its files with it.
+ fs::remove_dir_all(&sub).unwrap();
+ consumer.apply(
+ &cache,
+ ConsumerPathState::WatcherPathState(WatcherPathState::Deleted(sub)),
+ );
+ assert_eq!(cached(&cache), vec![a.clone()]);
+
+ // As does a deleted file.
+ fs::remove_file(&a).unwrap();
+ consumer.apply(
+ &cache,
+ ConsumerPathState::WatcherPathState(WatcherPathState::Deleted(a.clone())),
+ );
+ assert_eq!(cached(&cache), Vec::::new());
+
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
+
+ // Verify the whole pipeline: a project's cache learns of its files through
+ // the initial walk, then of files created and deleted afterward through the
+ // watcher.
+ #[test]
+ fn test_new_project() {
+ // Wait for `condition` to hold, since the threads which maintain the
+ // cache run on their own schedule. The watcher debounces events for 2
+ // seconds, which this timeout comfortably exceeds.
+ fn wait_for(condition: impl Fn() -> bool) {
+ let start = Instant::now();
+ while !condition() {
+ assert!(
+ start.elapsed() < Duration::from_secs(30),
+ "timed out waiting for the cache"
+ );
+ sleep(Duration::from_millis(50));
+ }
+ }
+
+ let (temp_dir, test_dir) = prep_test_dir!();
+ let root = canonicalize(&test_dir).unwrap();
+ let a = root.join("a.md");
+ let b = root.join("b.md");
+ fs::write(&a, "Foo
\n").unwrap();
+ let cache = Cache::new_project(root.clone());
+ let is_cached = |path: &Path| cache.lock().unwrap().files.contains_key(path);
+
+ wait_for(|| is_cached(&a));
+ fs::write(&b, "See .\n").unwrap();
+ wait_for(|| is_cached(&b));
+ assert_eq!(*dependents(&cache.lock().unwrap(), "foo"), paths(&[&b]));
+ fs::remove_file(&a).unwrap();
+ wait_for(|| !is_cached(&a));
+ assert!(matches!(
+ cache.lock().unwrap().resolve_id("foo"),
+ IdResolution::Missing
+ ));
+
+ // Dropping the cache stops its watcher, so that the temporary directory
+ // can be removed.
+ drop(cache);
+ // Report any errors produced when removing the temporary directory.
+ temp_dir.close().unwrap();
+ }
}
diff --git a/server/src/processing/tests.rs b/server/src/processing/tests.rs
index f5097e0d..552094c0 100644
--- a/server/src/processing/tests.rs
+++ b/server/src/processing/tests.rs
@@ -50,10 +50,10 @@ use crate::{
CodeDocBlockVecToSourceError, CodeMirrorDiffable, CodeMirrorDocBlockDelete,
CodeMirrorDocBlockTransaction, CodeMirrorDocBlockUpdate, CodechatForWebToSourceError,
HtmlToMarkdownWrapped, SourceToCodeChatForWebError, UNICODE_CURSOR_MARKER, byte_index_of,
- cache::Cache, code_doc_block_vec_to_source, code_mirror_to_code_doc_blocks,
- codechat_for_web_to_source, dehydrating_walk_node, diff_code_mirror_doc_blocks, diff_str,
- doc_block_html_to_markdown, html_to_dom, hydrate_html, is_css_identifier, markdown_to_html,
- source_to_codechat_for_web,
+ cache::{Cache, FileStatus},
+ code_doc_block_vec_to_source, code_mirror_to_code_doc_blocks, codechat_for_web_to_source,
+ dehydrating_walk_node, diff_code_mirror_doc_blocks, diff_str, doc_block_html_to_markdown,
+ html_to_dom, hydrate_html, is_css_identifier, markdown_to_html, source_to_codechat_for_web,
},
};
use test_utils::{cast, prep_test_dir, test_utils::stringit};
@@ -1808,6 +1808,67 @@ fn test_hydrate_xref_missing() {
);
}
+// Verify that a cross-reference lacking a `ref` attribute hydrates to an error
+// message.
+#[test]
+fn test_hydrate_xref_without_ref() {
+ assert_eq!(
+ source_to_codechat_for_web(
+ "// See .",
+ Path::new("foo.js"),
+ 0.0,
+ false,
+ None
+ ),
+ Ok(build_codechat_for_web(
+ "javascript",
+ "\n",
+ vec![build_codemirror_doc_block(
+ 0,
+ 1,
+ "",
+ "//",
+ "See a cross-reference requires a \"ref\" attribute."
+ )]
+ ))
+ );
+}
+
+// Verify that a cross-reference with an `id` attribute hydrates to an error
+// message, even when the id requests auto-assignment.
+#[test]
+fn test_hydrate_xref_with_id() {
+ assert_eq!(
+ source_to_codechat_for_web(
+ "//
Title
\nlet x = 1;\n// See and .",
+ Path::new("foo.js"),
+ 0.0,
+ false,
+ None
+ ),
+ Ok(build_codechat_for_web(
+ "javascript",
+ "\nlet x = 1;\n\n",
+ vec![
+ build_codemirror_doc_block(0, 1, "", "//", "Title
"),
+ build_codemirror_doc_block(
+ 12,
+ 13,
+ "",
+ "//",
+ concat!(
+ "See ",
+ "a cross-reference may not have an \"id\" attribute",
+ " and ",
+ "a cross-reference may not have an \"id\" attribute",
+ "."
+ )
+ )
+ ]
+ ))
+ );
+}
+
// Verify that a gather element and the fragment it lists hydrate in a single
// pass over their common file: the gather element receives the fragment's
// contents (the fragment's doc block plus the following code block), and the
@@ -1958,6 +2019,76 @@ fn test_hydrate_xref_cross_file() {
);
}
+// Verify that a cross-reference within a gathered fragment links from the
+// gathering file, not from the fragment's file: here the target shares the
+// fragment's file, where the link is just `#t`, while the gathering file lies
+// in another directory. A cross-reference which hydrated to an error keeps it,
+// even one whose error -- an `id` -- the gathered content no longer shows.
+#[test]
+fn test_hydrate_gathered_xref() {
+ let cache = Arc::new(Mutex::new(Cache::default()));
+ source_to_codechat_for_web(
+ "//
Title
\nlet a = 1;\n// See and .\nlet b = 2;",
+ Path::new("b.js"),
+ 0.0,
+ false,
+ Some(cache.clone()),
+ )
+ .unwrap();
+ let translation = source_to_codechat_for_web(
+ "// Gathered
",
+ Path::new("sub/a.js"),
+ 0.0,
+ false,
+ Some(cache),
+ )
+ .unwrap();
+ let CodeMirrorDiffable::Plain(code_mirror) = translation.source else {
+ panic!("No diff!");
+ };
+ assert_eq!(
+ code_mirror.doc_blocks[0].contents,
+ concat!(
+ "Gathered
",
+ "",
+ "
From ../b.js:",
+ "
",
+ "
3
",
+ "
",
+ "
See Title",
+ " and a cross-reference may not have an \"id\" attribute.",
+ "
",
+ "
4let b = 2;
",
+ "
Gathered
";
+ let fragments = "// First.\nlet a = 1;\n// Second.\nlet b = 2;";
+ translate(gathering, "a.js");
+ translate(fragments, "b.js");
+ // Storing the fragment's content outdated the gathering file; bring it up
+ // to date, then re-process the fragments' file.
+ translate(gathering, "a.js");
+ translate(fragments, "b.js");
+
+ let cache = cache.lock().unwrap();
+ assert_eq!(cache.files[Path::new("a.js")].status, FileStatus::UpToDate);
+ let content = &cache.files[Path::new("b.js")].fragments["frag"].content;
+ assert!(content.contains("First.") && !content.contains("Second."));
+}
+
// Verify that a fragment in a Markdown document is an error.
#[test]
fn test_hydrate_fragment_in_markdown() {
@@ -1977,6 +2108,62 @@ fn test_hydrate_fragment_in_markdown() {
);
}
+// Verify that a fragment without an id -- whether its `id` attribute is missing
+// or empty -- hydrates to an error message requesting the id. Dehydration
+// removes the message, so saving the hydrated result writes back a bare
+// ``. Since a fragment is cached under its id, the cache
+// records nothing for one which lacks an id.
+#[test]
+fn test_hydrate_fragment_without_id() {
+ // Both cases produce the same output. Hydration sees the empty `id`, which
+ // it must treat as missing rather than as an invalid id; the minifier then
+ // discards the empty attribute as redundant, so the Client never receives
+ // it and saving can't restore it.
+ for fragment in ["", ""] {
+ let cache = Arc::new(Mutex::new(Cache::default()));
+ let translation = source_to_codechat_for_web(
+ &format!("# {fragment}Doc.\nx = 1\n"),
+ Path::new("foo.py"),
+ 0.0,
+ false,
+ Some(cache.clone()),
+ )
+ .unwrap();
+ assert_eq!(
+ translation,
+ build_codechat_for_web(
+ "python",
+ "\nx = 1\n",
+ vec![build_codemirror_doc_block(
+ 0,
+ 1,
+ "",
+ "#",
+ "a fragment requires an \"id\" attributeDoc."
+ )]
+ ),
+ "hydrating {fragment}"
+ );
+ assert_eq!(
+ codechat_for_web_to_source(&translation).unwrap(),
+ "# Doc.\nx = 1\n",
+ "saving {fragment}"
+ );
+ // The commit recorded the file, but neither a fragment in it nor any
+ // id.
+ let cache_guard = cache.lock().unwrap();
+ assert_eq!(cache_guard.files.len(), 1, "caching {fragment}");
+ assert!(
+ cache_guard
+ .files
+ .values()
+ .all(|file_entry| file_entry.fragments.is_empty()),
+ "caching {fragment}"
+ );
+ assert!(cache_guard.ids.is_empty(), "caching {fragment}");
+ }
+}
+
// Verify auto-assignment of ids: `id="*"` is replaced by a generated, valid CSS
// identifier which is *not* recorded in the cache, and which the cache picks up
// only once the file carrying it is written and processed again.
diff --git a/server/src/translation.rs b/server/src/translation.rs
index 823f854d..7de7faec 100644
--- a/server/src/translation.rs
+++ b/server/src/translation.rs
@@ -160,6 +160,11 @@
//! Server ->> Client: Update(String: new Client contents)
//! else Main file is not editable
//! Server ->> Client: Update(String: new text contents)
+//! note over Client: Reload the document showing the main file
+//! Client ->> Server: HTTP request(URL of main file)
+//! Server ->> IDE: LoadFile(String: path to main file)
+//! IDE ->> Server: Response(LoadFile(String: file contents of main file))
+//! Server ->> Client: HTTP response(contents of main file)
//! end
//! Client ->> IDE: Response(String: OK)
//! ```
@@ -167,6 +172,14 @@
//! * If the current file's contents in the Client are edited, the Client sends
//! the IDE an `Update` with the revised contents.
//!
+//! * If a change elsewhere in the project outdates the rendering of the current
+//! file or of the project's TOC, the project's cache notifies the Server,
+//! which re-renders it and sends any change to the Client: an `Update` for
+//! the current file, or an `UpdateToc` for the TOC in the Client's sidebar.
+//! The Client replies to each with a `Result`. Editing the TOC itself also
+//! sends an `UpdateToc`. See `Connections` in the
+//! [cache spec](processing/cache-spec.md).
+//!
//! * When the PC goes to sleep then wakes up, the IDE client and the Editor
//! client both reconnect to the websocket URL containing their assigned ID.
//!
@@ -210,6 +223,7 @@ use std::{
collections::HashMap,
ffi::OsStr,
fmt::Debug,
+ iter,
path::{Path, PathBuf},
rc::Rc,
sync::LazyLock,
@@ -223,7 +237,7 @@ use regex::Regex;
use tokio::{
fs::File,
select,
- sync::mpsc::{self, Receiver, Sender},
+ sync::mpsc::{self, Receiver, Sender, UnboundedReceiver, UnboundedSender},
};
// ### Local
@@ -233,17 +247,23 @@ use crate::{
processing::{
CodeChatForWeb, CodeMirror, CodeMirrorDiff, CodeMirrorDiffable, CodeMirrorDocBlock,
CodeMirrorDocBlockVec, SourceFileMetadata, TranslationResultsString, UNICODE_CURSOR_MARKER,
- byte_index_of, cache::CacheMap, codechat_for_web_to_source, diff_code_mirror_doc_blocks,
- diff_str, doc_block_html_to_markdown, minify, remove_tinymce_data,
+ byte_index_of,
+ cache::{
+ CacheMap, CacheNotification, ConnectionRegistration, cache_map_capacity,
+ set_cache_map_capacity,
+ },
+ codechat_for_web_to_source, diff_code_mirror_doc_blocks, diff_str,
+ doc_block_html_to_markdown, find_path_to_toc, minify, project_cache, remove_tinymce_data,
source_to_codechat_for_web_string, transform_html,
},
queue_send, queue_send_func,
webserver::{
CursorPosition, EditorMessage, EditorMessageContents, INITIAL_MESSAGE_ID,
MESSAGE_ID_INCREMENT, ProcessingTaskHttpRequest, ProcessingTaskHttpRequestFlags,
- ResultErrTypes, ResultOkTypes, SimpleHttpResponse, SimpleHttpResponseError,
- UpdateMessageContents, WebAppState, WebsocketQueues, file_to_response, log_capture_event,
- path_to_url, send_response, try_canonicalize, try_read_as_text, url_to_path,
+ ResultErrTypes, ResultOkTypes, ServedContents, SimpleHttpResponse, SimpleHttpResponseError,
+ UpdateMessageContents, WebAppState, WebsocketQueues, canonicalize, file_to_response,
+ log_capture_event, path_to_url, send_response, try_canonicalize, try_read_as_text,
+ url_to_path,
},
};
@@ -389,11 +409,34 @@ pub fn create_translation_queues(
})
}
+/// The project TOC which the Client's sidebar shows, retained so that the
+/// sidebar can be updated when the TOC's rendering is outdated.
+struct SidebarToc {
+ /// The TOC's path, as requested by the sidebar.
+ path: PathBuf,
+ /// The TOC's text, from the IDE if it has the TOC open, otherwise from
+ /// disk.
+ source: String,
+ /// The HTML the sidebar shows, rendered from `source`.
+ html: String,
+}
+
+/// The purpose of a `LoadFile` request sent to the IDE, which determines how
+/// its response is handled.
+enum LoadFileRequest {
+ /// Serve the file to the Client, which requested it over HTTP.
+ Http(ProcessingTaskHttpRequest),
+ /// Re-load the TOC at this path, which the sidebar shows, after the cache
+ /// reported that the TOC changed; see `reload_sidebar_toc`.
+ SidebarToc(PathBuf),
+}
+
/// This holds the state used by the main loop of the translation task; this
/// allows factoring out lengthy contents in the loop into subfunctions.
struct TranslationTask {
// These parameters are passed to us.
app_state: WebAppState,
+ connection_id: String,
connection_id_raw: String,
prefix: &'static [&'static str],
allow_source_diffs: bool,
@@ -405,10 +448,20 @@ struct TranslationTask {
cache: CacheMap,
// These parameters are internal state.
- /// The file currently loaded in the Client.
+ /// The file currently loaded in the Client; change it only through
+ /// `set_current_file`.
current_file: PathBuf,
+ /// Lists this connection with the cache of the project containing
+ /// `current_file`, if any.
+ cache_registration: Option,
+ /// The channel on which that cache notifies this connection.
+ cache_notify_tx: UnboundedSender,
+ cache_notify_rx: UnboundedReceiver,
+ /// The TOC which the sidebar of the current file's page shows; `None`
+ /// until the sidebar loads it, or if the page has no sidebar.
+ sidebar_toc: Option,
/// A map of `LoadFile` requests sent to the IDE, awaiting its response.
- load_file_requests: HashMap,
+ load_file_requests: HashMap,
/// The id for messages created by the server. Leave space for a server
/// message during the init phase.
id: f64,
@@ -449,6 +502,22 @@ struct TranslationTask {
capture_context: CaptureContext,
}
+/// Resize the map of project caches to suit the number of connections, each of
+/// which has a translation task listed in `processing_task_queue_tx`; see
+/// `cache_map_capacity`.
+fn resize_cache_map(
+ // The state holding the map and the list of translation tasks.
+ app_state: &WebAppState,
+) {
+ // Hold the list's lock while resizing, so that connections which start or
+ // stop at the same time can't apply their counts out of order.
+ let processing_task_queue_tx = app_state.processing_task_queue_tx.lock().unwrap();
+ set_cache_map_capacity(
+ &mut app_state.cache.lock().unwrap(),
+ cache_map_capacity(processing_task_queue_tx.len()),
+ );
+}
+
/// This is the processing task for the Visual Studio Code IDE. It handles all
/// the core logic to moving data between the IDE and the client.
#[allow(clippy::too_many_arguments)]
@@ -476,10 +545,13 @@ pub async fn translation_task(
.lock()
.unwrap()
.insert(connection_id.clone(), from_http_tx);
+ resize_cache_map(&app_state);
+ let (cache_notify_tx, cache_notify_rx) = mpsc::unbounded_channel();
let mut continue_loop = true;
let mut tt = TranslationTask {
app_state: app_state.clone(),
+ connection_id: connection_id.clone(),
connection_id_raw,
prefix,
allow_source_diffs,
@@ -490,6 +562,10 @@ pub async fn translation_task(
from_http_rx,
cache: app_state.cache.clone(),
current_file: PathBuf::new(),
+ cache_registration: None,
+ cache_notify_tx,
+ cache_notify_rx,
+ sidebar_toc: None,
load_file_requests: HashMap::new(),
id: INITIAL_MESSAGE_ID + MESSAGE_ID_INCREMENT,
source_code: String::new(),
@@ -516,7 +592,8 @@ pub async fn translation_task(
EditorMessageContents::Opened(_) |
EditorMessageContents::OpenUrl(_) |
EditorMessageContents::LoadFile(..) |
- EditorMessageContents::ClientHtml(_) => {
+ EditorMessageContents::ClientHtml(_) |
+ EditorMessageContents::UpdateToc(_) => {
let err = ResultErrTypes::IdeIllegalMessage;
error!("{err:?}");
send_response(&tt.to_ide_tx, ide_message.id, Err(err)).await;
@@ -557,14 +634,7 @@ pub async fn translation_task(
path_to_url(&tt.prefix_str, Some(&tt.connection_id_raw), &clean_file_path), Some(true)
)
}));
- // Store the canonicalized path, not the
- // path the IDE sent: the comparisons
- // against `current_file` below assume every
- // path has one spelling.
- tt.current_file = clean_file_path;
- // Since this is a new file, mark it as
- // unsent in full.
- tt.sent_full = false;
+ tt.set_current_file(clean_file_path);
}
Err(err) => {
error!("{err:?}");
@@ -594,7 +664,7 @@ pub async fn translation_task(
}));
// Store the ID and request, which are needed to send a
// response when the `LoadFile` result is received.
- tt.load_file_requests.insert(tt.id.to_bits(), http_request);
+ tt.load_file_requests.insert(tt.id.to_bits(), LoadFileRequest::Http(http_request));
tt.id += MESSAGE_ID_INCREMENT;
}
@@ -606,7 +676,8 @@ pub async fn translation_task(
EditorMessageContents::Opened(_) |
EditorMessageContents::LoadFile(..) |
EditorMessageContents::RequestClose |
- EditorMessageContents::ClientHtml(_) => {
+ EditorMessageContents::ClientHtml(_) |
+ EditorMessageContents::UpdateToc(_) => {
let err = ResultErrTypes::ClientIllegalMessage;
error!("{err:?}");
send_response(&tt.to_client_tx, client_message.id, Err(err)).await;
@@ -680,10 +751,7 @@ pub async fn translation_task(
id: client_message.id,
message: EditorMessageContents::CurrentFile(file_path_string.to_string(), Some(is_text))
}));
- tt.current_file = file_path;
- // Since this is a new file, the full text
- // hasn't been sent yet.
- tt.sent_full = false;
+ tt.set_current_file(file_path);
Ok(())
}
}
@@ -697,6 +765,10 @@ pub async fn translation_task(
}
},
+ // Handle notifications from the cache of the current file's
+ // project.
+ Some(cache_notification) = tt.cache_notify_rx.recv() => continue_loop = tt.cache_notified(cache_notification).await,
+
else => break
}
}
@@ -712,6 +784,7 @@ pub async fn translation_task(
{
error!("Unable to remove connection ID {connection_id} from processing task queue.");
}
+ resize_cache_map(&app_state);
if app_state
.client_queues
.lock()
@@ -889,10 +962,11 @@ impl TranslationTask {
queue_send_func!(self.to_client_tx.send(ide_message));
return true;
}
- // Ensure there's an HTTP request for this `LoadFile` result.
- let Some(http_request) = self.load_file_requests.remove(&ide_message.id.to_bits()) else {
+ // Ensure there's a request for this `LoadFile` result.
+ let Some(load_file_request) = self.load_file_requests.remove(&ide_message.id.to_bits())
+ else {
error!(
- "Error: no HTTP request found for LoadFile result ID {}.",
+ "Error: no request found for LoadFile result ID {}.",
ide_message.id
);
return true;
@@ -915,12 +989,23 @@ impl TranslationTask {
ResultOkTypes::LoadFile(file_contents) => file_contents,
},
};
+ let http_request = match load_file_request {
+ LoadFileRequest::Http(http_request) => http_request,
+ LoadFileRequest::SidebarToc(toc_path) => {
+ return self
+ .reload_sidebar_toc(
+ toc_path,
+ file_contents_option.map(|(file_contents, _version)| file_contents),
+ )
+ .await;
+ }
+ };
// Process the file contents. Since VSCode doesn't have a PDF viewer,
// determine if this is a PDF file. (TODO: look at the magic number also
// -- "%PDF").
let use_pdf_js = http_request.file_path.extension() == Some(OsStr::new("pdf"));
- let ((simple_http_response, option_update), file_contents) = match file_contents_option {
+ let ((simple_http_response, served_contents), file_contents) = match file_contents_option {
Some((file_contents, new_version)) => {
// Only pay attention to the version if this is an editable
// Client file.
@@ -929,9 +1014,6 @@ impl TranslationTask {
{
self.version = new_version;
}
- // The IDE just sent the full contents; we're sending full
- // contents to the Client.
- self.sent_full = true;
(
file_to_response(
&http_request,
@@ -978,25 +1060,54 @@ impl TranslationTask {
}
}
};
- if let Some(update) = option_update {
- let Some(ref tmp) = update.contents else {
- panic!("Contents must always be provided.");
- };
- let CodeMirrorDiffable::Plain(ref plain) = tmp.source else {
- panic!("Diff not supported.");
- };
- self.source_code = file_contents;
- self.eol = find_eol_type(&self.source_code);
- // We must clone here, since the original is placed in the TX queue.
- self.code_mirror_doc.clone_from(&plain.doc);
- self.code_mirror_doc_blocks = Some(plain.doc_blocks.clone());
-
- debug!("Sending Update from LoadFile to Client, id = {}.", self.id);
- queue_send_func!(self.to_client_tx.send(EditorMessage {
- id: self.id,
- message: EditorMessageContents::Update(update)
- }));
- self.id += MESSAGE_ID_INCREMENT;
+ match served_contents {
+ Some(ServedContents::Client(update)) => {
+ let Some(ref tmp) = update.contents else {
+ panic!("Contents must always be provided.");
+ };
+ let CodeMirrorDiffable::Plain(ref plain) = tmp.source else {
+ panic!("Diff not supported.");
+ };
+ self.source_code = file_contents;
+ self.eol = find_eol_type(&self.source_code);
+ // We must clone here, since the original is placed in the TX
+ // queue.
+ self.code_mirror_doc.clone_from(&plain.doc);
+ self.code_mirror_doc_blocks = Some(plain.doc_blocks.clone());
+
+ debug!("Sending Update from LoadFile to Client, id = {}.", self.id);
+ queue_send_func!(self.to_client_tx.send(EditorMessage {
+ id: self.id,
+ message: EditorMessageContents::Update(update)
+ }));
+ self.id += MESSAGE_ID_INCREMENT;
+ // The Client now has the current file's full contents. Set
+ // this only here: results of other `LoadFile` requests, such as
+ // the TOC's, may arrive before the current file's and must not
+ // enable diffs against the previous file's contents. If these
+ // contents came from disk because the IDE lacks the file, diffs
+ // to the IDE are still safe: VSCode rejects any update to a
+ // document it doesn't have open, and the standalone file
+ // watcher, which never has the file, receives no diffs (see
+ // `allow_source_diffs`).
+ self.sent_full = true;
+ }
+ // Record the TOC which the sidebar now shows, unless it's the TOC
+ // of another project -- requested by the sidebar of a page which
+ // the Client has since left.
+ Some(ServedContents::Toc(html))
+ if http_request
+ .file_path
+ .parent()
+ .is_some_and(|project_root| self.current_file.starts_with(project_root)) =>
+ {
+ self.sidebar_toc = Some(SidebarToc {
+ path: http_request.file_path.clone(),
+ source: file_contents,
+ html,
+ });
+ }
+ Some(ServedContents::Toc(_)) | None => {}
}
debug!("Sending HTTP response.");
if let Err(err) = http_request.response_queue.send(simple_http_response) {
@@ -1109,6 +1220,9 @@ impl TranslationTask {
// Update to the version of the file just
// sent.
self.version = contents.version;
+ if !self.refresh_sidebar_if_toc_edited().await {
+ return false;
+ }
Ok(ResultOkTypes::Void)
}
// TODO
@@ -1209,6 +1323,83 @@ impl TranslationTask {
}
}
+ /// Determine if a re-translation of the current file differs from the
+ /// Client's contents, `self.code_mirror_doc` / `self.code_mirror_doc_blocks`.
+ /// Note that strict HTML comparison fails, since TinyMCE modifies
+ /// whitespace and some characters; for example, `👨👦` becomes `👨👦`
+ /// after processing by TinyMCE. Therefore, HTML processed by TinyMCE needs
+ /// to be normalized before it can be compared.
+ ///
+ /// 1. Document-only: HTML is stored in `.doc`; ignore `.doc_blocks`.
+ /// 2. Normal mode: HTML is stored in `.doc_blocks`; perform a binary
+ /// compare of `.doc`, then an HTML comparison of `.doc_blocks`.
+ ///
+ /// It looks like comparing HTML is risky, since TinyMCE (or something)
+ /// store emojis differently. Perhaps we need to compare Markdown instead of
+ /// HTML? Or HTML after processing? The second seems easier.
+ fn re_translation_differs(
+ &self,
+ // The current file's metadata.
+ metadata: &SourceFileMetadata,
+ // The re-translation.
+ code_mirror_translated: &CodeMirror,
+ ) -> bool {
+ if metadata.mode == MARKDOWN_MODE {
+ !compare_html(&code_mirror_translated.doc, &self.code_mirror_doc)
+ } else {
+ code_mirror_translated.doc != self.code_mirror_doc
+ || !doc_blocks_compare(
+ &code_mirror_translated.doc_blocks,
+ self.code_mirror_doc_blocks.as_ref().unwrap(),
+ )
+ }
+ }
+
+ /// Send the Client a re-translation of the current file as a diff against
+ /// its contents, then record the result as the Client's contents. The
+ /// Client ignores a re-translation while its own edits are unsent (see
+ /// `setContent` in `CodeChatEditor.mts`) or in flight (see the `Update`
+ /// handler in `CodeChatEditorFramework.mts`); `client_update` re-translates
+ /// those edits once they arrive.
+ async fn send_re_translation(
+ &mut self,
+ // The current file's path, spelled as in earlier updates to the Client,
+ // which ignores an update for any other path.
+ file_path: String,
+ // The current file's metadata.
+ metadata: SourceFileMetadata,
+ // The version of the Client's contents which the diff applies to.
+ before_version: f64,
+ // The version of the Client's contents after applying the diff.
+ version: f64,
+ // The re-translation.
+ code_mirror_translated: CodeMirror,
+ // False if the Client's queue is closed.
+ ) -> bool {
+ let client_contents =
+ self.diff_code_mirror(metadata, before_version, version, &code_mirror_translated);
+ debug!(
+ "Sending re-translation update id = {} back to the Client.",
+ self.id
+ );
+ queue_send_func!(self.to_client_tx.send(EditorMessage {
+ id: self.id,
+ message: EditorMessageContents::Update(UpdateMessageContents {
+ file_path,
+ // Don't change the current position, since the Client editing
+ // position should be left undisturbed.
+ cursor_position: None,
+ scroll_position: None,
+ is_re_translation: true,
+ contents: Some(client_contents),
+ })
+ }));
+ self.id += MESSAGE_ID_INCREMENT;
+ self.code_mirror_doc = code_mirror_translated.doc;
+ self.code_mirror_doc_blocks = Some(code_mirror_translated.doc_blocks);
+ true
+ }
+
#[allow(clippy::too_many_lines)]
async fn client_update(&mut self, client_message: EditorMessage) -> bool {
let EditorMessageContents::Update(update_message_contents) = client_message.message else {
@@ -1251,7 +1442,12 @@ impl TranslationTask {
let mut cfw_version = cfw.version;
// Translate back to the Client to see if there are
- // any changes after this conversion.
+ // any changes after this conversion, such as line
+ // wrapping in doc blocks which changes line
+ // numbering, creation of a new doc block from
+ // previous code block text, insertion of math, or
+ // updates from future document intelligence such as
+ // renamed headings.
if let Ok(ccfws) = source_to_codechat_for_web_string(
&new_source_code,
&clean_file_path,
@@ -1262,88 +1458,29 @@ impl TranslationTask {
&& let CodeMirrorDiffable::Plain(code_mirror_translated) =
ccfw.source
&& self.sent_full
+ && self
+ .re_translation_differs(&cfw.metadata, &code_mirror_translated)
{
- // Determine if the re-translation includes
- // changes (such as line wrapping in doc blocks
- // which changes line numbering, creation of a
- // new doc block from previous code block text,
- // insertion of math, or updates from future
- // document intelligence such as renamed
- // headings, etc.). Note that strict HTML
- // comparison fails, since TinyMCE modifies
- // whitespace and some characters; for example,
- // `👨👦` becomes `👨👦` after processing by
- // TinyMCE. Therefore, HTML processed by TinyMCE
- // needs to be normalized before it can be
- // compared.
- //
- // 1. Document-only: HTML is stored in `.doc`;
- // ignore `.doc_blocks`.
- // 2. Normal mode: HTML is stored in
- // `.doc_blocks`; perform a binary compare of
- // `.doc`, then an HTML comparison of
- // `.doc_blocks`.
- //
- // It looks like comparing HTML is risky, since
- // TinyMCE (or something) store emojis
- // differently. Perhaps we need to compare
- // Markdown instead of HTML? Or HTML after
- // processing? The second seems easier.
- let is_markdown_mode = cfw.metadata.mode == MARKDOWN_MODE;
- if (is_markdown_mode
- && !compare_html(
- &code_mirror_translated.doc,
- &self.code_mirror_doc,
- ))
- || (!is_markdown_mode
- && (code_mirror_translated.doc != self.code_mirror_doc
- || !doc_blocks_compare(
- &code_mirror_translated.doc_blocks,
- self.code_mirror_doc_blocks.as_ref().unwrap(),
- )))
- {
- // Use a whole number to avoid encoding
- // differences with fractional values.
- // Precision loss from the u64 -> f64 cast
- // is fine, since we just need a unique-ish
- // version number.
- cfw_version = {
- #[allow(clippy::cast_precision_loss)]
- let v = random::() as f64;
- v
- };
- // The Client needs an update.
- let client_contents = self.diff_code_mirror(
+ // Use a whole number to avoid encoding
+ // differences with fractional values. Precision
+ // loss from the u64 -> f64 cast is fine, since
+ // we just need a unique-ish version number.
+ cfw_version = {
+ #[allow(clippy::cast_precision_loss)]
+ let v = random::() as f64;
+ v
+ };
+ if !self
+ .send_re_translation(
+ update_message_contents.file_path,
cfw.metadata.clone(),
cfw.version,
cfw_version,
- &code_mirror_translated,
- );
- debug!(
- "Sending re-translation update id = {} back to the Client.",
- self.id
- );
- queue_send_func!(self.to_client_tx.send(EditorMessage {
- id: self.id,
- message: EditorMessageContents::Update(
- UpdateMessageContents {
- file_path: update_message_contents.file_path,
- // Don't change the current position, since
- // the Client editing position should be
- // left undisturbed.
- cursor_position: None,
- scroll_position: None,
- is_re_translation: true,
- contents: Some(client_contents),
- }
- )
- }));
- self.id += MESSAGE_ID_INCREMENT;
- // Update with what was just sent to the
- // client.
- self.code_mirror_doc = code_mirror_translated.doc;
- self.code_mirror_doc_blocks =
- Some(code_mirror_translated.doc_blocks);
+ code_mirror_translated,
+ )
+ .await
+ {
+ return false;
}
}
// Correct EOL endings for use with the IDE.
@@ -1375,6 +1512,9 @@ impl TranslationTask {
};
self.version = cfw_version;
self.source_code = new_source_code_eol;
+ if !self.refresh_sidebar_if_toc_edited().await {
+ return false;
+ }
new_cfw
}
Err(message) => {
@@ -1504,6 +1644,245 @@ impl TranslationTask {
true
}
+
+ /// Change the file loaded in the Client, moving this connection's cache
+ /// registration to the project containing the new file.
+ fn set_current_file(
+ &mut self,
+ // The new current file.
+ current_file: PathBuf,
+ ) {
+ // Store the canonicalized path: the comparisons against `current_file`
+ // assume every path has one spelling, which is also the spelling the
+ // cache uses. A path from the IDE is already canonical, but one from
+ // the Client may not be. As in hydration, fall back to the path as
+ // given if it doesn't exist on disk.
+ self.current_file = canonicalize(¤t_file).unwrap_or(current_file);
+ // Since this is a new file, the full text hasn't been sent yet.
+ self.sent_full = false;
+ // Nor has its page's sidebar, if any, loaded the TOC.
+ self.sidebar_toc = None;
+ // Drop the old registration first; see `ConnectionRegistration::new`.
+ self.cache_registration = None;
+ self.cache_registration = find_path_to_toc(&self.current_file).map(|path_to_toc| {
+ ConnectionRegistration::new(
+ &project_cache(&self.cache, &self.current_file, &path_to_toc),
+ self.connection_id.clone(),
+ self.current_file.clone(),
+ self.cache_notify_tx.clone(),
+ )
+ });
+ }
+
+ /// Respond to notifications from the cache of the current file's project.
+ /// One translation elsewhere may send several notifications, such as one
+ /// per changed fragment, so gather all those queued, then re-render each
+ /// kind of outdated content once.
+ async fn cache_notified(
+ &mut self,
+ // The notification which woke this task.
+ notification: CacheNotification,
+ // False if the Client's queue is closed.
+ ) -> bool {
+ let mut is_current_file_outdated = false;
+ let mut is_toc_outdated = false;
+ let mut is_toc_changed = false;
+ let queued = iter::from_fn(|| self.cache_notify_rx.try_recv().ok());
+ for notification in iter::once(notification).chain(queued) {
+ match notification {
+ CacheNotification::CurrentFileOutdated => is_current_file_outdated = true,
+ CacheNotification::TocOutdated => is_toc_outdated = true,
+ CacheNotification::TocChanged => is_toc_changed = true,
+ }
+ }
+ if is_current_file_outdated && !self.re_translate_current_file().await {
+ return false;
+ }
+ // Re-loading the TOC re-renders it, which also renders anew a TOC whose
+ // rendering is outdated.
+ if is_toc_changed {
+ return self.request_sidebar_toc().await;
+ }
+ if is_toc_outdated && !self.refresh_sidebar_toc().await {
+ return false;
+ }
+ true
+ }
+
+ /// Re-translate the current file after its project's cache reports that
+ /// its rendered output is outdated -- because another connection changed a
+ /// target this file cross-references, for example -- then send any change
+ /// to the Client. The source is unchanged, so the IDE isn't updated.
+ async fn re_translate_current_file(
+ &mut self,
+ // False if the Client's queue is closed.
+ ) -> bool {
+ // Until the Client has the full text, there's nothing to diff against;
+ // the full text, once sent, is translated with the current cache.
+ if !self.sent_full {
+ return true;
+ }
+ // Translate the IDE's contents, which may be newer than the file on
+ // disk; see `ide_update` for the EOL handling.
+ match source_to_codechat_for_web_string(
+ &self.source_code.replace("\r\n", "\n"),
+ &self.current_file,
+ self.version,
+ false,
+ &self.cache,
+ ) {
+ Ok((TranslationResultsString::CodeChat(ccfw), _)) => {
+ let CodeMirrorDiffable::Plain(code_mirror_translated) = ccfw.source else {
+ panic!("Unexpected diff value.");
+ };
+ if self.re_translation_differs(&ccfw.metadata, &code_mirror_translated) {
+ // Keep the version: the source is unchanged, and the IDE,
+ // which receives nothing, must still agree with the Client
+ // on the version.
+ return self
+ .send_re_translation(
+ self.current_file.to_str().expect("Since the path started as a string, assume it losslessly translates back to a string.").to_string(),
+ ccfw.metadata,
+ self.version,
+ self.version,
+ code_mirror_translated,
+ )
+ .await;
+ }
+ }
+ // Other file types have no content which depends on the cache.
+ Ok(_) => {}
+ Err(err) => error!(
+ "Unable to re-translate {}: {err}",
+ self.current_file.display()
+ ),
+ }
+ true
+ }
+
+ /// Re-render the TOC which the Client's sidebar shows, then send the Client
+ /// the result if it changed. The Client replaces the sidebar's contents in
+ /// place, which preserves its scroll position.
+ async fn refresh_sidebar_toc(
+ &mut self,
+ // False if the Client's queue is closed.
+ ) -> bool {
+ // Until the sidebar loads the TOC, there's nothing to refresh; loading
+ // it renders the TOC from the current cache.
+ let Some(sidebar_toc) = &mut self.sidebar_toc else {
+ return true;
+ };
+ let html = match source_to_codechat_for_web_string(
+ &sidebar_toc.source.replace("\r\n", "\n"),
+ &sidebar_toc.path,
+ // A TOC's HTML doesn't include a version.
+ 0.0,
+ true,
+ &self.cache,
+ ) {
+ Ok((TranslationResultsString::Toc(html), _)) => html,
+ result => {
+ error!(
+ "Unable to render the TOC {}: {result:?}",
+ sidebar_toc.path.display()
+ );
+ return true;
+ }
+ };
+ if html == sidebar_toc.html {
+ return true;
+ }
+ sidebar_toc.html.clone_from(&html);
+ debug!("Sending TOC update id = {} to the Client.", self.id);
+ queue_send_func!(self.to_client_tx.send(EditorMessage {
+ id: self.id,
+ message: EditorMessageContents::UpdateToc(html),
+ }));
+ self.id += MESSAGE_ID_INCREMENT;
+ true
+ }
+
+ /// Ask the IDE for the text of the TOC which the Client's sidebar shows,
+ /// after the project's cache reports that this text changed on disk. The
+ /// IDE's text, if it has the TOC open, supersedes the text on disk, as when
+ /// the sidebar first loaded the TOC. `reload_sidebar_toc` handles the IDE's
+ /// response.
+ async fn request_sidebar_toc(
+ &mut self,
+ // False if the IDE's queue is closed.
+ ) -> bool {
+ // Until the sidebar loads the TOC, there's nothing to re-load; loading
+ // it reads the TOC's current text.
+ let Some(sidebar_toc) = &self.sidebar_toc else {
+ return true;
+ };
+ let toc_path = sidebar_toc.path.clone();
+ debug!(
+ "Sending LoadFile id = {} to reload the sidebar's TOC.",
+ self.id
+ );
+ queue_send_func!(self.to_ide_tx.send(EditorMessage {
+ id: self.id,
+ // As when the sidebar loaded it, the TOC isn't loaded as the file to
+ // edit, so it isn't assigned a version.
+ message: EditorMessageContents::LoadFile(toc_path.clone(), false),
+ }));
+ self.load_file_requests
+ .insert(self.id.to_bits(), LoadFileRequest::SidebarToc(toc_path));
+ self.id += MESSAGE_ID_INCREMENT;
+ true
+ }
+
+ /// Re-render the TOC which the Client's sidebar shows from its re-loaded
+ /// text, then send the Client the result if it changed.
+ async fn reload_sidebar_toc(
+ &mut self,
+ // The TOC re-loaded.
+ toc_path: PathBuf,
+ // The TOC's text from the IDE; `None` if the IDE doesn't have the TOC
+ // open, in which case its text is read from disk.
+ ide_text: Option,
+ // False if the Client's queue is closed.
+ ) -> bool {
+ let text = match ide_text {
+ Some(text) => Some(text),
+ None => match File::open(&toc_path).await {
+ Ok(mut file) => try_read_as_text(&mut file).await,
+ Err(_) => None,
+ },
+ };
+ // While the IDE responded, the Client may have left the page whose
+ // sidebar showed this TOC.
+ let Some(sidebar_toc) = self
+ .sidebar_toc
+ .as_mut()
+ .filter(|sidebar_toc| sidebar_toc.path == toc_path)
+ else {
+ return true;
+ };
+ // A TOC which can't be read -- since it was deleted, say -- keeps its
+ // last text, but may still need rendering anew.
+ if let Some(text) = text {
+ sidebar_toc.source = text;
+ }
+ self.refresh_sidebar_toc().await
+ }
+
+ /// After an edit to the current file, refresh the Client's sidebar if that
+ /// file is the TOC it shows. The edit doesn't outdate the TOC in the cache:
+ /// a file's translation never outdates the file itself.
+ async fn refresh_sidebar_if_toc_edited(
+ &mut self,
+ // False if the Client's queue is closed.
+ ) -> bool {
+ match &mut self.sidebar_toc {
+ Some(sidebar_toc) if sidebar_toc.path == self.current_file => {
+ sidebar_toc.source.clone_from(&self.source_code);
+ self.refresh_sidebar_toc().await
+ }
+ _ => true,
+ }
+ }
}
// Given a string using LF-only (Unix style) line endings, convert it to CRLF
diff --git a/server/src/webserver.rs b/server/src/webserver.rs
index 459fb96f..37f87728 100644
--- a/server/src/webserver.rs
+++ b/server/src/webserver.rs
@@ -63,6 +63,7 @@ use log4rs::{
self,
config::{Deserializers, load_config_file},
};
+use lru::LruCache;
use mime::Mime;
use mime_guess;
use path_slash::PathBufExt;
@@ -86,7 +87,8 @@ use url::Url;
// ### Local
//use crate::capture::EventCapture;
use crate::processing::{
- CodeChatForWeb, SourceToCodeChatForWebError, TranslationResultsString, cache::CacheMap,
+ CodeChatForWeb, SourceToCodeChatForWebError, TranslationResultsString,
+ cache::{CacheMap, cache_map_capacity},
find_path_to_toc, source_to_codechat_for_web_string,
};
@@ -153,6 +155,17 @@ pub enum SimpleHttpResponse {
Bin(PathBuf),
}
+/// What a page served by `file_to_response` shows, which the processing task
+/// tracks in order to update that page later.
+#[derive(Debug)]
+pub enum ServedContents {
+ /// The page is a Client, which this `Update` populates with the file's
+ /// contents.
+ Client(UpdateMessageContents),
+ /// The page is the TOC shown in a sidebar, which consists of this HTML.
+ Toc(String),
+}
+
// List all the possible errors when responding to an HTTP request. See
// [The definitive guide to error handling in Rust](https://www.howtocodeit.com/articles/the-definitive-guide-to-rust-error-handling).
#[derive(Debug, thiserror::Error)]
@@ -237,6 +250,10 @@ pub enum EditorMessageContents {
/// HTML for the CodeChat Editor Client to display in its built-in browser.
/// Valid destinations: IDE.
ClientHtml(String),
+ /// Replace the contents of the TOC in the Client's sidebar with the
+ /// provided HTML, the TOC re-rendered after a change which outdated it.
+ /// Valid destinations: Client.
+ UpdateToc(String),
/// Sent when the IDE or Client websocket was closed, indicating that the
/// unclosed websocket should be closed as well. Therefore, this message
/// will never be received by the IDE or Client. Valid destinations: Server.
@@ -932,9 +949,9 @@ pub async fn file_to_response(
// The response to send back to the HTTP endpoint.
SimpleHttpResponse,
// If the response is a Client, also return the appropriate `Update` data to
- // populate the Client with the parsed `file_contents`. In all other cases,
- // return None.
- Option,
+ // populate the Client with the parsed `file_contents`; if it's a TOC, also
+ // return the TOC's HTML. In all other cases, return None.
+ Option,
) {
// Use a lossy conversion, since this is UI display, not filesystem access.
let file_path = &http_request.file_path;
@@ -1102,7 +1119,7 @@ pub async fn file_to_response(