From fa408af7f48665e78b769089cd929a495b8be10c Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Wed, 26 Aug 2026 22:47:22 +0200 Subject: [PATCH 01/13] Watcher: Start recording metrics for git index change handling --- Cargo.lock | 4 +- README.md | 4 +- crates/bin/docs_rs_watcher/Cargo.toml | 1 + crates/bin/docs_rs_watcher/src/db/delete.rs | 3 + .../bin/docs_rs_watcher/src/index_watcher.rs | 106 ++++++++- crates/bin/docs_rs_watcher/src/lib.rs | 11 +- crates/bin/docs_rs_watcher/src/metrics.rs | 110 +++++++++ crates/lib/docs_rs_crates_io/Cargo.toml | 7 +- crates/lib/docs_rs_crates_io/src/events.rs | 214 ++++++++++++------ 9 files changed, 367 insertions(+), 93 deletions(-) create mode 100644 crates/bin/docs_rs_watcher/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 48b171579a..7596b158aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1259,10 +1259,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-link", ] @@ -2144,6 +2142,7 @@ dependencies = [ "chrono", "serde", "serde_json", + "test-case", ] [[package]] @@ -2494,6 +2493,7 @@ dependencies = [ "docs_rs_build_queue", "docs_rs_config", "docs_rs_context", + "docs_rs_crates_io", "docs_rs_database", "docs_rs_env_vars", "docs_rs_fastly", diff --git a/README.md b/README.md index 1adf5b1eb9..d8a9d02f44 100644 --- a/README.md +++ b/README.md @@ -166,8 +166,8 @@ $ just lint ``` Linting GitHub Actions workflows requires -[`actionlint`](https://github.com/rhysd/actionlint/blob/main/docs/install.md). If -it is not installed, that check is skipped with a warning. +[`actionlint`](https://github.com/rhysd/actionlint/blob/main/docs/install.md). +If it is not installed, that check is skipped with a warning. Run all formatters with: diff --git a/crates/bin/docs_rs_watcher/Cargo.toml b/crates/bin/docs_rs_watcher/Cargo.toml index 30f38a6e8b..f4ab5b8e68 100644 --- a/crates/bin/docs_rs_watcher/Cargo.toml +++ b/crates/bin/docs_rs_watcher/Cargo.toml @@ -16,6 +16,7 @@ crates-index-diff = { version = "31.0.0", default-features = false, features = [ docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" } docs_rs_config = { path = "../../lib/docs_rs_config" } docs_rs_context = { path = "../../lib/docs_rs_context" } +docs_rs_crates_io = { path = "../../lib/docs_rs_crates_io" } docs_rs_database = { path = "../../lib/docs_rs_database" } docs_rs_env_vars = { path = "../../lib/docs_rs_env_vars" } docs_rs_fastly = { path = "../../lib/docs_rs_fastly" } diff --git a/crates/bin/docs_rs_watcher/src/db/delete.rs b/crates/bin/docs_rs_watcher/src/db/delete.rs index 7a5a3fb448..39d8b82a0c 100644 --- a/crates/bin/docs_rs_watcher/src/db/delete.rs +++ b/crates/bin/docs_rs_watcher/src/db/delete.rs @@ -5,12 +5,14 @@ use docs_rs_storage::{AsyncStorage, rustdoc_archive_path, source_archive_path}; use docs_rs_types::{CrateId, KrateName, Version}; use sqlx::Connection; use tokio::fs; +use tracing::instrument; /// List of directories in docs.rs's underlying storage (either the database or S3) containing a /// subdirectory named after the crate. Those subdirectories will be deleted. static LIBRARY_STORAGE_PATHS_TO_DELETE: &[&str] = &["rustdoc", "rustdoc-json", "sources"]; static OTHER_STORAGE_PATHS_TO_DELETE: &[&str] = &["sources"]; +#[instrument(skip_all, fields(name=%name))] pub async fn delete_crate( conn: &mut sqlx::PgConnection, storage: &AsyncStorage, @@ -56,6 +58,7 @@ pub async fn delete_crate( Ok(()) } +#[instrument(skip_all, fields(name=%name, version=%version))] pub async fn delete_version( conn: &mut sqlx::PgConnection, storage: &AsyncStorage, diff --git a/crates/bin/docs_rs_watcher/src/index_watcher.rs b/crates/bin/docs_rs_watcher/src/index_watcher.rs index 3d03d3ce94..eba5b926bf 100644 --- a/crates/bin/docs_rs_watcher/src/index_watcher.rs +++ b/crates/bin/docs_rs_watcher/src/index_watcher.rs @@ -2,18 +2,57 @@ use crate::{ Config, db::{delete_crate, delete_version}, index::Index, + metrics::{EventSource, WatcherMetrics}, }; use anyhow::{Context as _, Result}; use crates_index_diff::Change; use docs_rs_build_queue::PRIORITY_MANUAL_FROM_CRATES_IO; use docs_rs_context::Context; +use docs_rs_crates_io::events::ChangeKind; use docs_rs_database::{ crate_details::update_latest_version_id, service_config::{ConfigName, get_config, set_config}, }; use docs_rs_fastly::{Cdn, CdnBehaviour as _}; use docs_rs_types::{CrateId, KrateName, Version}; -use tracing::{debug, error, info, warn}; +use std::time::Instant; +use tracing::{debug, error, info, instrument, warn}; + +trait ChangeExt { + fn name(&self) -> &str; + fn version(&self) -> Option<&str>; + fn kind(&self) -> ChangeKind; + fn first_crate_version(&self) -> &crates_index_diff::CrateVersion; +} + +impl ChangeExt for Change { + fn first_crate_version(&self) -> &crates_index_diff::CrateVersion { + self.versions().first().expect("always exists") + } + + fn name(&self) -> &str { + self.first_crate_version().name.as_str() + } + + fn version(&self) -> Option<&str> { + if let Change::CrateDeleted { .. } = self { + None + } else { + Some(self.first_crate_version().version.as_str()) + } + } + + fn kind(&self) -> ChangeKind { + match *self { + Change::Added(_) => ChangeKind::Added, + Change::Yanked(_) => ChangeKind::Yanked, + Change::CrateDeleted { .. } => ChangeKind::CrateDeleted, + Change::VersionDeleted(_) => ChangeKind::VersionDeleted, + Change::Unyanked(_) => ChangeKind::Unyanked, + Change::AddedAndYanked(_) => ChangeKind::AddedAndYanked, + } + } +} #[derive(Debug)] pub(crate) struct CrateVersion { @@ -94,6 +133,7 @@ pub(crate) async fn get_new_crates( context: &Context, index: &Index, config: &Config, + metrics: &WatcherMetrics, ) -> Result { let mut conn = context.pool()?.get_async().await?; @@ -115,7 +155,10 @@ pub(crate) async fn get_new_crates( debug!(last_seen_reference=%last_seen_reference, new_reference=%new_reference, "queueing changes"); - let crates_added = process_changes(context, &changes, config).await; + metrics.record_events_received(EventSource::Git, changes.len()); + // NOTE: `Box::pin` to type-erase this future, otherwise we'll run into `recursion_limit` + // errors. + let crates_added = Box::pin(process_changes(context, &changes, config, metrics)).await; if let Err(err) = context.build_queue()?.reevaluate_priorities().await { error!(?err, "error reevaluating queued release priorities"); @@ -129,32 +172,66 @@ pub(crate) async fn get_new_crates( Ok(crates_added) } -async fn process_changes(context: &Context, changes: &Vec, config: &Config) -> usize { +async fn process_changes( + context: &Context, + changes: &Vec, + config: &Config, + metrics: &WatcherMetrics, +) -> usize { let mut crates_added = 0; for change in changes { - match process_change(context, change, config).await { + let start = Instant::now(); + let crate_name = change.name(); + let crate_version = change.version(); + let change_type = change.kind(); + + // Start temporarily loging all changes, as preparation for the SQS event migration. + debug!( + target: "docs_rs_watcher::index_event", + source = %EventSource::Git, + %change_type, + crate_name, + crate_version, + "crates.io index event" + ); + + let success = match process_change(context, change, config).await { Ok(added) => { + metrics.record_change_applied(EventSource::Git, change_type); if added { crates_added += 1; } + true } Err(err) => { error!(?change, ?err, "failed to process change"); + false } - } + }; + metrics.record_event_processing_time( + EventSource::Git, + Some(change_type), + success, + start.elapsed(), + ); } crates_added } /// Process a crate change, returning whether the change was a crate addition or not. +#[instrument(skip_all, fields(name, version))] async fn process_change(context: &Context, change: &Change, config: &Config) -> Result { - let crate_version: CrateVersion = change - .versions() - .first() - .expect("always exists") - .clone() - .try_into()?; + // 1: use the `CrateVersion` from `crates-index-diff`. + let crate_version = change.first_crate_version(); + + // record name & version on the tracing span for performance instrumentation. + tracing::Span::current() + .record("name", crate_version.name.as_str()) + .record("version", crate_version.version.as_str()); + + // 2: now, convert to our own internal `CrateVersion.` + let crate_version: CrateVersion = crate_version.clone().try_into()?; match change { Change::Added(_release) => process_version_added(context, &crate_version).await?, @@ -177,6 +254,7 @@ async fn process_change(context: &Context, change: &Change, config: &Config) -> } /// Processes crate changes, whether they got yanked or unyanked. +#[instrument(skip_all)] async fn process_version_yank_status(context: &Context, release: &CrateVersion) -> Result<()> { // FIXME: delay yanks of crates that have not yet finished building // https://github.com/rust-lang/docs.rs/issues/1934 @@ -185,6 +263,7 @@ async fn process_version_yank_status(context: &Context, release: &CrateVersion) Ok(()) } +#[instrument(skip_all)] async fn process_version_added(context: &Context, release: &CrateVersion) -> Result<()> { let build_queue = context.build_queue()?; @@ -217,6 +296,7 @@ async fn process_version_added(context: &Context, release: &CrateVersion) -> Res Ok(()) } +#[instrument(skip_all)] async fn process_version_deleted( context: &Context, config: &Config, @@ -251,6 +331,7 @@ async fn process_version_deleted( Ok(()) } +#[instrument(skip_all)] async fn process_crate_deleted( context: &Context, config: &Config, @@ -270,6 +351,7 @@ async fn process_crate_deleted( context.build_queue()?.remove_crate_from_queue(krate).await } +#[instrument(skip_all, fields(name=%name, version=%version, yanked=%yanked))] pub(crate) async fn set_yanked( context: &Context, name: &KrateName, @@ -518,6 +600,7 @@ mod tests { version: V2, ..Default::default() }; + let metrics = WatcherMetrics::new(&env.context().meter_provider); let added = process_changes( &env, &vec![ @@ -531,6 +614,7 @@ mod tests { Change::VersionDeleted(non_existing_version.into()), ], env.config(), + &metrics, ) .await; diff --git a/crates/bin/docs_rs_watcher/src/lib.rs b/crates/bin/docs_rs_watcher/src/lib.rs index 833a6c6885..862324ebb2 100644 --- a/crates/bin/docs_rs_watcher/src/lib.rs +++ b/crates/bin/docs_rs_watcher/src/lib.rs @@ -3,6 +3,7 @@ pub mod consistency; mod db; mod index; pub mod index_watcher; +mod metrics; mod rebuilds; mod service_metrics; #[cfg(test)] @@ -13,7 +14,11 @@ pub use db::{delete_crate, delete_version}; pub use index::Index; pub use rebuilds::queue_rebuilds; -use crate::{index_watcher::get_new_crates, service_metrics::OtelServiceMetrics}; +use crate::{ + index_watcher::get_new_crates, + metrics::{EventSource, WatcherMetrics}, + service_metrics::OtelServiceMetrics, +}; use anyhow::Result; use docs_rs_context::Context; use docs_rs_utils::start_async_cron; @@ -28,6 +33,7 @@ pub async fn watch_registry(config: &Config, context: &Context) -> Result<()> { let mut last_gc = Instant::now(); let queue = context.build_queue()?; + let metrics = WatcherMetrics::new(context.meter_provider()); loop { if queue.is_locked().await? { @@ -36,9 +42,10 @@ pub async fn watch_registry(config: &Config, context: &Context) -> Result<()> { debug!("Checking new crates"); let index = Index::from_config(config).await?; - match get_new_crates(context, &index, config).await { + match get_new_crates(context, &index, config, &metrics).await { Ok(n) => debug!("{} crates added to queue", n), Err(e) => { + metrics.record_poll_error(EventSource::Git); error!(?e, "Failed to get new crates"); } } diff --git a/crates/bin/docs_rs_watcher/src/metrics.rs b/crates/bin/docs_rs_watcher/src/metrics.rs new file mode 100644 index 0000000000..dbfd279f57 --- /dev/null +++ b/crates/bin/docs_rs_watcher/src/metrics.rs @@ -0,0 +1,110 @@ +use docs_rs_crates_io::events::ChangeKind; +use docs_rs_opentelemetry::AnyMeterProvider; +use opentelemetry::{ + KeyValue, + metrics::{Counter, Histogram}, +}; +use std::{fmt, time::Duration}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum EventSource { + Git, + // NOTE: Sqs will be added later +} + +impl EventSource { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Git => "git", + } + } +} + +impl fmt::Display for EventSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug)] +pub(crate) struct WatcherMetrics { + /// received event count, by source + events_received_total: Counter, + /// poll errors, by source + poll_errors_total: Counter, + /// changes applied, by source and change-kind + changes_applied_total: Counter, + /// event processing time, by source and change-kind + event_processing_time: Histogram, +} + +impl WatcherMetrics { + pub(crate) fn new(meter_provider: &AnyMeterProvider) -> Self { + let meter = meter_provider.meter("watcher"); + const PREFIX: &str = "docsrs.watcher"; + Self { + events_received_total: meter + .u64_counter(format!("{PREFIX}.events_received_total")) + .with_unit("1") + .build(), + poll_errors_total: meter + .u64_counter(format!("{PREFIX}.poll_errors_total")) + .with_unit("1") + .build(), + changes_applied_total: meter + .u64_counter(format!("{PREFIX}.changes_applied_total")) + .with_unit("1") + .build(), + event_processing_time: meter + .f64_histogram(format!("{PREFIX}.event_processing_time")) + // Boundaries for the histogram, should be min/max for the processing time + .with_boundaries(vec![ + // that's what we expect in processing time, between <1s, and 5 minutes. + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, + 45.0, 55.0, 60.0, 65.0, 90.0, 120.0, 180.0, 300.0, + // these are just for outliers, so we see them + 600.0, 900.0, 1800.0, 3600.0, + ]) + .with_unit("s") + .build(), + } + } + + pub(crate) fn record_change_applied(&self, source: EventSource, kind: ChangeKind) { + self.changes_applied_total.add( + 1, + &[ + KeyValue::new("source", source.as_str()), + KeyValue::new("type", kind.as_str()), + ], + ); + } + + pub(crate) fn record_event_processing_time( + &self, + source: EventSource, + kind: Option, + success: bool, + duration: Duration, + ) { + let result = if success { "ok" } else { "err" }; + self.event_processing_time.record( + duration.as_secs_f64(), + &[ + KeyValue::new("source", source.as_str()), + KeyValue::new("type", kind.map(ChangeKind::as_str).unwrap_or("unknown")), + KeyValue::new("result", result), + ], + ); + } + + pub(crate) fn record_events_received(&self, source: EventSource, count: usize) { + self.events_received_total + .add(count as u64, &[KeyValue::new("source", source.as_str())]); + } + + pub(crate) fn record_poll_error(&self, source: EventSource) { + self.poll_errors_total + .add(1, &[KeyValue::new("source", source.as_str())]); + } +} diff --git a/crates/lib/docs_rs_crates_io/Cargo.toml b/crates/lib/docs_rs_crates_io/Cargo.toml index 497536d66e..6ac6ab55d2 100644 --- a/crates/lib/docs_rs_crates_io/Cargo.toml +++ b/crates/lib/docs_rs_crates_io/Cargo.toml @@ -9,11 +9,12 @@ repository.workspace = true edition.workspace = true [dependencies] -chrono = { version = "0.4", features = ["serde"] } -serde = { version = "1", features = ["derive"] } +chrono = { workspace = true } +serde = { workspace = true } [dev-dependencies] -serde_json = "1.0" +serde_json = { workspace = true } +test-case = { workspace = true } [lints] workspace = true diff --git a/crates/lib/docs_rs_crates_io/src/events.rs b/crates/lib/docs_rs_crates_io/src/events.rs index f90484abc2..8ad77f665c 100644 --- a/crates/lib/docs_rs_crates_io/src/events.rs +++ b/crates/lib/docs_rs_crates_io/src/events.rs @@ -1,6 +1,35 @@ use chrono::{DateTime, Utc}; use std::fmt; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChangeKind { + Added, + AddedAndYanked, + Unyanked, + Yanked, + CrateDeleted, + VersionDeleted, +} + +impl ChangeKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Added => "added", + Self::AddedAndYanked => "added_and_yanked", + Self::Unyanked => "unyanked", + Self::Yanked => "yanked", + Self::CrateDeleted => "crate_deleted", + Self::VersionDeleted => "version_deleted", + } + } +} + +impl fmt::Display for ChangeKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + /// A change that can happen to a crate on our index. #[derive(Clone, serde::Serialize, serde::Deserialize, Eq, PartialEq, Debug)] #[serde(tag = "type", content = "payload", rename_all = "snake_case")] @@ -21,7 +50,7 @@ impl IndexChangeV1 { /// Return the added crate, if this is this kind of change. pub fn added(&self) -> Option<&CrateVersion> { match self { - IndexChangeV1::Added(v) => Some(v), + Self::Added(version) => Some(version), _ => None, } } @@ -29,7 +58,7 @@ impl IndexChangeV1 { /// Return the yanked crate, if this is this kind of change. pub fn yanked(&self) -> Option<&CrateVersion> { match self { - IndexChangeV1::Yanked(v) => Some(v), + Self::Yanked(version) => Some(version), _ => None, } } @@ -37,7 +66,7 @@ impl IndexChangeV1 { /// Return the unyanked crate, if this is this kind of change. pub fn unyanked(&self) -> Option<&CrateVersion> { match self { - IndexChangeV1::Unyanked(v) => Some(v), + Self::Unyanked(version) => Some(version), _ => None, } } @@ -45,7 +74,7 @@ impl IndexChangeV1 { /// Return the deleted crate, if this is this kind of change. pub fn crate_deleted(&self) -> Option<&str> { match self { - IndexChangeV1::CrateDeleted { name } => Some(name.as_str()), + Self::CrateDeleted { name } => Some(name), _ => None, } } @@ -53,21 +82,45 @@ impl IndexChangeV1 { /// Return the deleted version crate, if this is this kind of change. pub fn version_deleted(&self) -> Option<&CrateVersion> { match self { - IndexChangeV1::VersionDeleted(v) => Some(v), + Self::VersionDeleted(version) => Some(version), _ => None, } } + + pub fn name(&self) -> &str { + match self { + Self::Added(crate_version) + | Self::Unyanked(crate_version) + | Self::Yanked(crate_version) + | Self::VersionDeleted(crate_version) => &crate_version.name, + Self::CrateDeleted { name } => name, + } + } + + pub fn version(&self) -> Option<&str> { + match self { + Self::Added(crate_version) + | Self::Unyanked(crate_version) + | Self::Yanked(crate_version) + | Self::VersionDeleted(crate_version) => Some(&crate_version.version), + Self::CrateDeleted { .. } => None, + } + } + + pub fn kind(&self) -> ChangeKind { + match self { + Self::Added(_) => ChangeKind::Added, + Self::Unyanked(_) => ChangeKind::Unyanked, + Self::Yanked(_) => ChangeKind::Yanked, + Self::CrateDeleted { .. } => ChangeKind::CrateDeleted, + Self::VersionDeleted(_) => ChangeKind::VersionDeleted, + } + } } impl fmt::Display for IndexChangeV1 { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match *self { - IndexChangeV1::Added(_) => "added", - IndexChangeV1::Yanked(_) => "yanked", - IndexChangeV1::CrateDeleted { .. } => "crate deleted", - IndexChangeV1::VersionDeleted(_) => "version deleted", - IndexChangeV1::Unyanked(_) => "unyanked", - }) + self.kind().fmt(f) } } @@ -76,7 +129,7 @@ impl fmt::Display for IndexChangeV1 { pub struct Event { /// Unique event identifier for deduplication and tracing. pub id: String, - /// Timestamp when the event occured + /// Timestamp when the event occurred. pub occurred_at: DateTime, /// The typed payload. #[serde(flatten)] @@ -100,6 +153,7 @@ pub struct CrateVersion { mod tests { use super::*; use serde_json::json; + use test_case::test_case; fn crate_version() -> CrateVersion { CrateVersion { @@ -131,67 +185,81 @@ mod tests { ); } + #[test_case(ChangeKind::Added, "added"; "added")] + #[test_case(ChangeKind::AddedAndYanked, "added_and_yanked"; "added and yanked")] + #[test_case(ChangeKind::Unyanked, "unyanked"; "unyanked")] + #[test_case(ChangeKind::Yanked, "yanked"; "yanked")] + #[test_case(ChangeKind::CrateDeleted, "crate_deleted"; "crate deleted")] + #[test_case(ChangeKind::VersionDeleted, "version_deleted"; "version deleted")] + fn change_kind_formats_as_expected(kind: ChangeKind, expected: &str) { + assert_eq!(kind.as_str(), expected); + assert_eq!(kind.to_string(), expected); + } + + #[test_case(IndexChangeV1::Added(crate_version()), "added", json!({ "name": "clap", "vers": "4.5.0" }); "added")] + #[test_case(IndexChangeV1::Unyanked(crate_version()), "unyanked", json!({ "name": "clap", "vers": "4.5.0" }); "unyanked")] + #[test_case(IndexChangeV1::Yanked(crate_version()), "yanked", json!({ "name": "clap", "vers": "4.5.0" }); "yanked")] + #[test_case(IndexChangeV1::CrateDeleted { name: "old-crate".into() }, "crate_deleted", json!({ "name": "old-crate" }); "crate deleted")] + #[test_case(IndexChangeV1::VersionDeleted(crate_version()), "version_deleted", json!({ "name": "clap", "vers": "4.5.0" }); "version deleted")] + fn change_serializes_with_expected_variant_shape( + change: IndexChangeV1, + change_type: &str, + payload: serde_json::Value, + ) { + assert_eq!( + serde_json::to_value(change).unwrap(), + json!({ + "type": change_type, + "payload": payload, + }) + ); + } + + #[test_case(IndexChangeV1::Added(crate_version()), ChangeKind::Added, "clap", Some("4.5.0"); "added")] + #[test_case(IndexChangeV1::Unyanked(crate_version()), ChangeKind::Unyanked, "clap", Some("4.5.0"); "unyanked")] + #[test_case(IndexChangeV1::Yanked(crate_version()), ChangeKind::Yanked, "clap", Some("4.5.0"); "yanked")] + #[test_case(IndexChangeV1::CrateDeleted { name: "old-crate".into() }, ChangeKind::CrateDeleted, "old-crate", None; "crate deleted")] + #[test_case(IndexChangeV1::VersionDeleted(crate_version()), ChangeKind::VersionDeleted, "clap", Some("4.5.0"); "version deleted")] + fn change_metadata_matches_variant( + change: IndexChangeV1, + kind: ChangeKind, + name: &str, + version: Option<&str>, + ) { + assert_eq!(change.name(), name); + assert_eq!(change.version(), version); + assert_eq!(change.kind(), kind); + assert_eq!(change.to_string(), kind.as_str()); + } + #[test] - fn change_serializes_with_expected_variant_shapes() { - let crate_version = crate_version(); - - let cases = [ - ( - IndexChangeV1::Added(crate_version.clone()), - json!({ - "type": "added", - "payload": { - "name": "clap", - "vers": "4.5.0", - } - }), - ), - ( - IndexChangeV1::Unyanked(crate_version.clone()), - json!({ - "type": "unyanked", - "payload": { - "name": "clap", - "vers": "4.5.0", - } - }), - ), - ( - IndexChangeV1::Yanked(crate_version.clone()), - json!({ - "type": "yanked", - "payload": { - "name": "clap", - "vers": "4.5.0", - } - }), - ), - ( - IndexChangeV1::CrateDeleted { - name: "old-crate".into(), - }, - json!({ - "type": "crate_deleted", - "payload": { - "name": "old-crate" - } - }), - ), - ( - IndexChangeV1::VersionDeleted(crate_version), - json!({ - "type": "version_deleted", - "payload": { - "name": "clap", - "vers": "4.5.0", - } - }), - ), - ]; - - for (event, expected) in cases { - assert_eq!(serde_json::to_value(&event).unwrap(), expected); - } + fn variant_accessors_only_match_their_variant() { + let added = IndexChangeV1::Added(crate_version()); + assert_eq!(added.added(), Some(&crate_version())); + assert_eq!(added.yanked(), None); + assert_eq!(added.unyanked(), None); + assert_eq!(added.crate_deleted(), None); + assert_eq!(added.version_deleted(), None); + + assert_eq!( + IndexChangeV1::Yanked(crate_version()).yanked(), + Some(&crate_version()) + ); + assert_eq!( + IndexChangeV1::Unyanked(crate_version()).unyanked(), + Some(&crate_version()) + ); + assert_eq!( + IndexChangeV1::CrateDeleted { + name: "old-crate".into(), + } + .crate_deleted(), + Some("old-crate") + ); + assert_eq!( + IndexChangeV1::VersionDeleted(crate_version()).version_deleted(), + Some(&crate_version()) + ); } #[test] From 34f34df21a0ceb56e525ae5b349149e01837923a Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:01:12 +0200 Subject: [PATCH 02/13] Squashed commit of the following: commit d0a699590ebab673ecd67a06ec87bfd48c19012e Author: Denis Cornehl Date: Thu Aug 27 06:50:04 2026 +0200 kk commit 254154cc2d433b133cb124eea3d27e89454884ff Author: Denis Cornehl Date: Thu Aug 27 06:47:26 2026 +0200 kk commit 7d44010a18ddd8a792967abe7f4fe315126629ff Author: Denis Cornehl Date: Wed Aug 26 22:44:38 2026 +0200 kk commit 8ab10bdb83be6d5e9324fdc82c1269f75f59466c Author: Denis Cornehl Date: Wed Aug 26 22:36:30 2026 +0200 kk commit 6ab809e624517b8b085b30f7ada24b28aa3b636f Author: Denis Cornehl Date: Wed Aug 26 22:32:38 2026 +0200 kk commit 6557417747c006c5dd34d5dfc3cbd1a4473a4671 Author: Denis Cornehl Date: Wed Aug 26 22:31:51 2026 +0200 kk commit 5284aef6ec03d44404c629c343f367167571ef4b Author: Denis Cornehl Date: Wed Aug 26 22:29:42 2026 +0200 note commit 106f930a2380ed9196fac056655481f4c24c2678 Author: Denis Cornehl Date: Wed Aug 26 22:28:36 2026 +0200 kk commit 0de0661eb67c8062b301b49623eeaa677e6a7dd2 Author: Denis Cornehl Date: Wed Aug 26 22:27:12 2026 +0200 kk commit 13ab2e37496c2fea45bd9524a8b1a59a705e9504 Author: Denis Cornehl Date: Wed Aug 26 22:25:59 2026 +0200 kk commit ca54cec7bf8f0fd33987ebb9731c289a9296070f Author: Denis Cornehl Date: Wed Aug 26 22:21:02 2026 +0200 metrics commit d4a259f78365ab876c121a3cd630371611b20934 Author: Denis Cornehl Date: Wed Aug 26 22:13:28 2026 +0200 metri commit 09d1f686154813a1f44bd048f3e131b3525b3b90 Author: Denis Cornehl Date: Wed Aug 26 22:02:33 2026 +0200 kk commit 5c13b4895170f72f1892e7579986662490f86302 Author: Denis Cornehl Date: Wed Aug 26 22:00:55 2026 +0200 kk commit c4e9b3878648c940f16cba92a1d251ba2b1d0f70 Author: Denis Cornehl Date: Wed Aug 26 21:59:30 2026 +0200 kk commit 0be13cb441496ee5260333011646a2965ed74db3 Author: Denis Cornehl Date: Wed Aug 26 21:57:44 2026 +0200 kk commit 08217cd82d4fc0e8adc0a19fa673552ceb1380a8 Author: Denis Cornehl Date: Wed Aug 26 21:54:42 2026 +0200 kk commit 8162966c8da9fc6607797d15697f5a10fc6a1fee Author: Denis Cornehl Date: Wed Aug 26 21:52:24 2026 +0200 mertrics commit f3ab38cb395fe0c20d41ee443b0927420320403e Author: Denis Cornehl Date: Wed Aug 26 21:46:32 2026 +0200 kk commit 1870b93b31b0db440576e89338aa375aacd14981 Author: Denis Cornehl Date: Wed Aug 26 21:41:07 2026 +0200 kk commit 866ffa642c3a1dc6807076a8e8d5c0c8e4e90254 Author: Denis Cornehl Date: Wed Aug 26 21:19:10 2026 +0200 kk commit eb61bfe84ee1628af491d33a4f433c4ef2ddbe82 Author: Denis Cornehl Date: Wed Aug 26 21:16:07 2026 +0200 test commit 3520919823f6f353001f05552d65a555496750b9 Author: Denis Cornehl Date: Wed Aug 26 21:11:14 2026 +0200 kk commit 76582df856ab21b57716f74b69fb7c8d2d10da1a Author: Denis Cornehl Date: Wed Aug 26 21:08:12 2026 +0200 kk commit 21bf74e06669d3e819481a20d6afd33f700b1083 Author: Denis Cornehl Date: Wed Aug 26 20:56:57 2026 +0200 kk commit 7ef3f009a9bc5553c01e8346fa5d3cfa4e48412c Author: Denis Cornehl Date: Wed Aug 26 20:52:33 2026 +0200 ht commit 39e198584e0a4ed14c26e571a2e549540380c121 Author: Denis Cornehl Date: Wed Aug 26 20:46:26 2026 +0200 fix commit 6d32c4bbc83a7af3a391a7a71dd38d37420d4036 Merge: b4490bf4 79f294f1 Author: Denis Cornehl Date: Wed Aug 26 20:42:30 2026 +0200 Merge branch 'main' into crates-io-sqs commit b4490bf4b1b82276548ab9440cfc216db6421239 Merge: 51d00df5 2990aabc Author: Denis Cornehl Date: Mon Aug 17 12:45:15 2026 +0200 Merge branch 'main' into crates-io-sqs commit 51d00df57e09168b71cd5bb648969c118dadf3a1 Author: Denis Cornehl Date: Wed Aug 12 05:19:16 2026 +0200 kk commit d7d186239a8d0fb1edaa9cc108603cecf215b3c7 Merge: 5613a6d1 b67fb7d1 Author: Denis Cornehl Date: Wed Aug 12 05:10:10 2026 +0200 Merge branch 'main' into crates-io-sqs commit 5613a6d1c151034fed63b4d47573f92d0cd5883e Author: Denis Cornehl Date: Wed Aug 5 09:39:35 2026 +0200 fixes commit 5758369601ce71fceebf1cc928704011c0afc32b Author: Denis Cornehl Date: Wed Aug 5 09:17:41 2026 +0200 Revert "WIP" This reverts commit 3556734724b21563e624d7edb9739b4020dbf168. commit 221230ac8fb1d11686d21284ff7e0cfb288ec658 Author: Denis Cornehl Date: Wed Aug 5 09:17:03 2026 +0200 config commit 3556734724b21563e624d7edb9739b4020dbf168 Author: Denis Cornehl Date: Wed Aug 5 05:08:12 2026 +0200 WIP commit 0ce6c2a450c5a6e6588bc4b8a041f717f730dcdc Author: Denis Cornehl Date: Wed Aug 5 04:40:13 2026 +0200 tryt more commit a4150f0b4f4ebf46578f6734b7acc3a6e83098e2 Author: Denis Cornehl Date: Wed Aug 5 04:18:25 2026 +0200 kk commit fc47c253f76be654ebf2a872f0ca4f6e4df5183e Merge: c2302c2e c03c6d67 Author: Denis Cornehl Date: Wed Aug 5 04:11:17 2026 +0200 Merge branch 'main' into crates-io-sqs commit c2302c2e90e3831bf6248c5cf399e49b17d7c244 Author: Denis Cornehl Date: Fri Jul 17 18:09:54 2026 +0200 higher vis timeout commit dd9c9d59c8a1c2e1e3f846a7e608e5e1423eb0e5 Author: Denis Cornehl Date: Thu Jul 2 03:37:40 2026 +0200 sqlx commit 0644add289bab49f158b8312f007ff440a49f5d9 Author: Denis Cornehl Date: Thu Jul 2 03:33:14 2026 +0200 anme commit c6e40c0078e9939a0b5b3a322264788faded26c1 Author: Denis Cornehl Date: Thu Jul 2 03:32:16 2026 +0200 sql commit 00f0bca46ac861117d3c3d59bb368c6c997106e8 Author: Denis Cornehl Date: Thu Jul 2 03:29:51 2026 +0200 pub commit 307b678adcd740023688f130f22df34af6ed662d Author: Denis Cornehl Date: Thu Jul 2 03:28:14 2026 +0200 tryform commit 206d1ea2585b74ec1793372f01734d9c460cc70d Author: Denis Cornehl Date: Thu Jul 2 03:23:14 2026 +0200 fix env commit 089ba22a1fe187e2ba15fe25d03f203c1e4e2352 Author: Denis Cornehl Date: Thu Jul 2 03:22:25 2026 +0200 fix test commit 3b5a9b18bed32bb120597c67387369b0c0f0369a Author: Denis Cornehl Date: Thu Jul 2 03:16:30 2026 +0200 clean commit a8f480d812b4cd7059c641508ee655b8acc35a09 Author: Denis Cornehl Date: Thu Jul 2 03:15:11 2026 +0200 chore(watcher): refine SQS processing buckets Add buckets around the one-minute visibility timeout so slow message handlers are easier to spot in metrics. commit c0f5b0131da46f1ba97efd7e5a870fe69cc72c3c Author: Denis Cornehl Date: Thu Jul 2 03:11:22 2026 +0200 sort commit ecd7952b2a0dcee95d3263cb98283d230e1382a3 Author: Denis Cornehl Date: Thu Jul 2 03:10:38 2026 +0200 names commit 9fdefe37a807afc1f4e113624294f791e6464cda Author: Denis Cornehl Date: Thu Jul 2 03:03:09 2026 +0200 kk commit 1835ae7e972f7524583a6947bfefe65f29a4d27d Author: Denis Cornehl Date: Thu Jul 2 02:59:01 2026 +0200 small things commit 84e5739c5b691892e904e4faa41cfe55b1446b1e Author: Denis Cornehl Date: Thu Jul 2 02:40:29 2026 +0200 more instrument commit 1bc440c182bef17ca56e21556e4e30369784f68d Author: Denis Cornehl Date: Thu Jul 2 02:27:25 2026 +0200 metrics commit 79ba59df8a85c8e0fc4f26bbbc469fdc4f1e23d5 Author: Denis Cornehl Date: Thu Jul 2 02:17:17 2026 +0200 feat(watcher): instrument SQS subscriber metrics Add starter counters and histograms for SQS intake, failures, retries, processing time, event lag, and applied change types, and record them from the subscriber flow. commit ea9a7b3e499836341dd9ce7000ae2e8f57671136 Author: Denis Cornehl Date: Thu Jul 2 02:11:00 2026 +0200 added dummy commit 405ab70c29e6cb74b8f3f59220faef556a291a46 Author: Denis Cornehl Date: Thu Jul 2 02:02:18 2026 +0200 comments commit da0323006203b6ef7990990ce63c55335cd319e6 Author: Denis Cornehl Date: Thu Jul 2 01:59:22 2026 +0200 clean commit 5b1d257d274f1bf8f6be3e36a0330f26f36531f5 Author: Denis Cornehl Date: Thu Jul 2 01:53:41 2026 +0200 fix(watcher): keep registry watcher alive on SQS errors Supervise the SQS subscriber separately in mixed mode so SQS failures are logged and restarted without stopping the legacy registry watcher. commit 98caa4bce7e11da94cee99f5675057c2fd6a49b8 Author: Denis Cornehl Date: Thu Jul 2 01:50:57 2026 +0200 refactor(watcher): clarify subscriber naming Rename the SQS subscriber entrypoints to better reflect the transport, body-handling, and event-processing layers. commit 8f3bd6e7e630d31b006ee0e7f1cff95137745ac2 Author: Denis Cornehl Date: Thu Jul 2 01:48:00 2026 +0200 nnma commit 44c0dd658e0823ffa24edc5e5b55e3721e61ec1a Author: Denis Cornehl Date: Thu Jul 2 01:39:55 2026 +0200 fix test commit af73c65869affa8f5e38c59284e8feb2728a6ce3 Author: Denis Cornehl Date: Thu Jul 2 01:35:59 2026 +0200 config, fixes, comment commit 6c6d69437fdf7293203742bd0226c6311c41744a Author: Denis Cornehl Date: Thu Jul 2 01:24:32 2026 +0200 sqlx commit b7525670f0a1c595c962c52d12f3c0a89cf86029 Author: Denis Cornehl Date: Thu Jul 2 01:21:15 2026 +0200 config commit 11c8ecba158740bd5a270d8c82290e647165a87c Author: Denis Cornehl Date: Thu Jul 2 01:19:17 2026 +0200 no locks commit 28ebe7173c983bd2b4b3de960a5f13d00987316b Author: Denis Cornehl Date: Thu Jul 2 01:12:41 2026 +0200 sort commit 405014999a750e92a328bdd58a85c77b967e687e Author: Denis Cornehl Date: Thu Jul 2 01:04:27 2026 +0200 dummy commit fe8ed9d9e58ea0db43074b9e8068e57a394d9c45 Author: Denis Cornehl Date: Thu Jul 2 00:59:01 2026 +0200 fix wait time commit 72121335eefb62e03431023a67f444b73b79bfb1 Author: Denis Cornehl Date: Thu Jul 2 00:58:17 2026 +0200 logs commit 697c831ff86cd2667b444213559a778d58558135 Author: Denis Cornehl Date: Thu Jul 2 00:51:10 2026 +0200 no ui commit e70bafa7167e3e60fb75c509ecc3b9196da47f47 Author: Denis Cornehl Date: Thu Jul 2 00:46:49 2026 +0200 feat(watcher): add local ElasticMQ support Wire ElasticMQ into docker compose for watcher development and allow the watcher SQS client to target a custom endpoint URL. commit 28015cf964f63d3fbfbadd231a94499b723c6628 Author: Denis Cornehl Date: Thu Jul 2 00:42:41 2026 +0200 kk commit b6ef36653c03358a42c72dac7bc18eb781f775a2 Author: Denis Cornehl Date: Thu Jul 2 00:41:05 2026 +0200 kk commit f80a63408ad86eeeb4bc2e606acdd8eda9ce1365 Author: Denis Cornehl Date: Thu Jul 2 00:39:23 2026 +0200 kk commit 5a9ef5759646cab7800d902f5eea5e16488a6ea9 Author: Denis Cornehl Date: Thu Jul 2 00:36:06 2026 +0200 refactor(watcher): simplify subscriber flow Replace the transport trait and helper fan-out with a smaller listen/handle/process structure. Keep tests at the decision boundary with MessageOutcome and remove the recursion-limit workaround. commit c06a1e5f0f8fd31d4d63987046b80e8d787d34f0 Author: Denis Cornehl Date: Thu Jul 2 00:22:45 2026 +0200 rescursion commit 876f0836f6da44bd2783fcb63517d5e123d4de6a Author: Denis Cornehl Date: Thu Jul 2 00:15:07 2026 +0200 refactor commit 59fc37d41a5df3c55f2561515c7612d1974506b1 Author: Denis Cornehl Date: Thu Jul 2 00:05:05 2026 +0200 refactor commit 4955cbb9869ff2c752e4b7017b6da2428f2f597f Author: Denis Cornehl Date: Wed Jul 1 23:50:47 2026 +0200 refactor(watcher): isolate SQS subscriber transport Split the subscriber into poll, handle, and decode layers behind a small SQS client trait so transport behavior can be unit-tested without an emulator. commit 4ee1856cdb939caa4b03cdfa876c7a0cc5141e33 Author: Denis Cornehl Date: Wed Jul 1 23:46:03 2026 +0200 test(watcher): cover subscriber dispatch Add unit tests for process_change and process_message in the SQS subscriber and clean up small watcher warnings found during validation. commit 6ff066643c56c8b164c973ed13f6110283e5bd9f Author: Denis Cornehl Date: Wed Jul 1 23:36:08 2026 +0200 msg commit c82f60bcbf2fd2567b85e177426113563c6a38f2 Author: Denis Cornehl Date: Wed Jul 1 23:25:56 2026 +0200 err commit e5149b16a9cd7b7b7710543fa50217cf47406a1f Author: Denis Cornehl Date: Wed Jul 1 23:08:38 2026 +0200 simp commit 5f6dde2156ca3a6a6ff13049d07014a31af0f0a2 Author: Denis Cornehl Date: Wed Jul 1 22:51:00 2026 +0200 read commit 7d6c4c20115654248e30464c704c72a7e1086e22 Author: Denis Cornehl Date: Wed Jul 1 16:29:00 2026 +0200 wip commit 216b68d9e2e9cf7d2292d0c3fda3f94fa0ffec7d Author: Denis Cornehl Date: Wed Jul 1 16:08:31 2026 +0200 WIP commit b380d7e9ab54ceeea64c9f8b57c9a94f844d78fa Merge: c46d0686 f4344f77 Author: Denis Cornehl Date: Wed Jul 1 15:55:29 2026 +0200 Merge branch 'event-structs' into crates-io-sqs commit c46d0686b7c885e82669f74e2f180f295d53094a Merge: d927bc5e 964c90a4 Author: Denis Cornehl Date: Wed Jul 1 15:54:23 2026 +0200 Merge branch 'main' into crates-io-sqs commit f4344f77807c923ec00fc012e2d34ea5a980a82e Author: Denis Cornehl Date: Wed Jul 1 14:48:58 2026 +0200 use String for IndexChangeEvent -> version commit 6a862eef736eb5b5679ea7f68b8d411e1ae45a40 Author: Denis Cornehl Date: Wed Jul 1 14:46:18 2026 +0200 remove "yanked" from event payload commit 431662efc18f63d4c846d14b31f293c6ce70f786 Author: Denis Cornehl Date: Wed Jun 3 22:36:11 2026 +0200 add docs_rs_crates_io subcrate for interaction / shared types commit d927bc5e56e3b1e8565f467ba246541ef93544e5 Author: Denis Cornehl Date: Sun Jun 14 14:59:54 2026 +0200 prio commit 8f49598882325c8f25ac1b01fe1af83ff1149d7d Author: Denis Cornehl Date: Sat Jun 13 05:34:30 2026 +0200 todo commit bc3b16e3c067f4cb469e8fa08dfce1242906ca73 Author: Denis Cornehl Date: Sat Jun 13 05:33:48 2026 +0200 errs commit e1f4467e1a3eace5b2e0006983d6d34bec98a421 Author: Denis Cornehl Date: Sat Jun 13 05:31:14 2026 +0200 first version commit d0417990a4f10986d2ee4c04fa2e88240bc34dea Author: Denis Cornehl Date: Sat Jun 13 04:12:16 2026 +0200 make deletes repeatable commit f056b427b9ac842fe577fbe8df2d2372ad4a86a6 Author: Denis Cornehl Date: Sat Jun 13 04:03:26 2026 +0200 save commit fed546f2f27c1e98e47d03e25b22934a9a01b261 Author: Denis Cornehl Date: Sat Jun 13 03:55:39 2026 +0200 sort commit 3639af818d896cb719e95dca7fea8589cd0038d0 Merge: 26db891f 65af70e4 Author: Denis Cornehl Date: Sat Jun 13 03:55:19 2026 +0200 Merge branch 'event-structs' into crates-io-sqs commit 65af70e4b5519015b60215c419174f746f0f9699 Author: Denis Cornehl Date: Wed Jun 3 22:36:11 2026 +0200 add docs_rs_crates_io subcrate for interaction / shared types commit 26db891f57a1feca1f06b8a868286888f17a7a23 Merge: c0436b16 7a6c393e Author: Denis Cornehl Date: Wed Jun 3 22:28:35 2026 +0200 Merge branch 'main' into crates-io-sqs commit c0436b166914cf1326d6e7104961ea9a477e0378 Author: Denis Cornehl Date: Sat May 23 09:22:13 2026 +0200 no rustls commit bb4d8456ea0a98fa7151c98809d9b18f5bd89112 Author: Denis Cornehl Date: Sat May 23 08:22:02 2026 +0200 some cleanup commit 1e374cf4bfcfd5bdaf51ef8c8bc91e6e063f2b8b Author: Denis Cornehl Date: Sat May 23 08:17:50 2026 +0200 renames commit e84c343e18c6b8d7c9aaebe9c93d18800f44572c Author: Denis Cornehl Date: Sat May 23 08:15:59 2026 +0200 refactor(events): drop schema version Remove the redundant schema_version field from the crates.io event envelope and keep versioning in the typed payloads. commit b311a599d391230dc6d882de28b9089ec9b5bd6b Author: Denis Cornehl Date: Fri May 22 18:57:05 2026 +0200 chore(lockfile): record watcher url dep Update Cargo.lock after making docs_rs_watcher depend directly on url. commit de207e532af1fea82d5d78718a3e4723cba48930 Author: Denis Cornehl Date: Fri May 22 18:56:48 2026 +0200 refactor(watcher): parse SQS queue URL Use url::Url for the watcher SQS queue URL config so invalid values fail during config loading. commit cb7453f3948d00a19834c4e3f9ac2d25605ea2bc Author: Denis Cornehl Date: Fri May 22 18:56:01 2026 +0200 feat(watcher): add SQS config Add watcher config fields for an SQS queue URL and region to support an event-based path. commit 277ea22f63faa20ff8cf8e141da148913effdf24 Author: Denis Cornehl Date: Fri May 22 18:33:43 2026 +0200 fix(watcher): make version delete idempotent Treat duplicate version deletion events as a no-op so temporary event-based handling can safely replay them. commit 0b7dd8da0419b32bbc47c840b9802582e7dfd2df Author: Denis Cornehl Date: Fri May 22 17:03:56 2026 +0200 wider deps commit 6b053a37c239139f875a6f8a64dcc3e62947597d Author: Denis Cornehl Date: Fri May 22 16:35:11 2026 +0200 refactor(events): use chrono timestamps Replace time::OffsetDateTime with chrono::DateTime for RFC 3339 event timestamps. commit e52cae891fa7d5f1349f613d7b50faf81b5add6f Author: Denis Cornehl Date: Fri May 22 16:32:57 2026 +0200 refactor(events): use typed event timestamps Remove event source metadata and store occurred_at as an RFC 3339 OffsetDateTime. commit b7403da2686a3c44caf5ce4e0927f8bc2b63d8b8 Author: Denis Cornehl Date: Fri May 22 16:25:14 2026 +0200 refactor(events): version event payload types Rename the current wire payload to ChangeV1 and make the event envelope generic for future schema versions. commit 87c438435e9996c40292844b35e68f542eb2488f Author: Denis Cornehl Date: Fri May 22 16:14:48 2026 +0200 feat(events): add event envelope metadata Wrap typed change payloads in a conventional event envelope with id, occurred_at, source, and schema_version. commit fddeb9a511f039ab0412e83738d25d253cd5f1f5 Author: Denis Cornehl Date: Fri May 22 15:40:36 2026 +0200 add shared subcrate for event types commit 9d1b50bc2f8c64833833c805f5ecd80e7b7e5981 Author: Denis Cornehl Date: Fri May 22 15:16:04 2026 +0200 add aws-sdk-sqs --- .docker.env.sample | 5 + ...29e35750da2bea995fb0c433893addb253214.json | 15 - ...8420f7da0f0445c6e43d5d64617226c24fba1.json | 26 + ...86e6c7eed78a2d4f01f316772949f5d688f42.json | 14 + ...83bd73f73ecc1cf9b4fc24c457d5f26fd582b.json | 29 + ...f4254924db75fc7f76a6c78d17a3fc06d663.json} | 7 +- Cargo.lock | 125 +++- Cargo.toml | 5 + crates/bin/cratesfyi/src/daemon.rs | 4 +- crates/bin/cratesfyi/src/main.rs | 2 + crates/bin/docs_rs_watcher/Cargo.toml | 6 + crates/bin/docs_rs_watcher/src/config.rs | 66 +- crates/bin/docs_rs_watcher/src/db/delete.rs | 83 ++- .../bin/docs_rs_watcher/src/index_watcher.rs | 139 +++- crates/bin/docs_rs_watcher/src/lib.rs | 68 +- crates/bin/docs_rs_watcher/src/main.rs | 4 +- crates/bin/docs_rs_watcher/src/metrics.rs | 118 +++ crates/bin/docs_rs_watcher/src/subscriber.rs | 675 ++++++++++++++++++ crates/lib/docs_rs_crates_io/src/events.rs | 67 +- crates/lib/docs_rs_storage/Cargo.toml | 6 +- docker-compose.yml | 17 + dockerfiles/elasticmq.conf | 18 + justfiles/utils.just | 9 +- 23 files changed, 1403 insertions(+), 105 deletions(-) delete mode 100644 .sqlx/query-2dc065cc08f262c937c54f9cc8629e35750da2bea995fb0c433893addb253214.json create mode 100644 .sqlx/query-5f5fa0e89b4e13c690b1648a18e8420f7da0f0445c6e43d5d64617226c24fba1.json create mode 100644 .sqlx/query-66b0ba6978880b79ce7a179bbe986e6c7eed78a2d4f01f316772949f5d688f42.json create mode 100644 .sqlx/query-7b7dd5795cddcb66b140b57157983bd73f73ecc1cf9b4fc24c457d5f26fd582b.json rename .sqlx/{query-014a054d852f0937191e1a54f742d4b4c454361689fb3841cc12fd7dd1094948.json => query-fab139cabc0987a1f2ad706060a3f4254924db75fc7f76a6c78d17a3fc06d663.json} (65%) create mode 100644 crates/bin/docs_rs_watcher/src/metrics.rs create mode 100644 crates/bin/docs_rs_watcher/src/subscriber.rs create mode 100644 dockerfiles/elasticmq.conf diff --git a/.docker.env.sample b/.docker.env.sample index 743cc06133..995e1254af 100644 --- a/.docker.env.sample +++ b/.docker.env.sample @@ -10,3 +10,8 @@ DOCSRS_TOOLCHAIN=nightly # for the registry watcher, automatically queued reqbuidls. DOCSRS_MAX_QUEUED_REBUILDS: 10 +# optional overrides for local ElasticMQ testing +# DOCSRS_SQS_QUEUE_URL=http://elasticmq:9324/queue/docsrs-events +# DOCSRS_SQS_QUEUE_REGION=elasticmq +# DOCSRS_SQS_ENDPOINT_URL=http://elasticmq:9324 +# DOCSRS_SQS_ACTIVE=false diff --git a/.sqlx/query-2dc065cc08f262c937c54f9cc8629e35750da2bea995fb0c433893addb253214.json b/.sqlx/query-2dc065cc08f262c937c54f9cc8629e35750da2bea995fb0c433893addb253214.json deleted file mode 100644 index 2109f69cf3..0000000000 --- a/.sqlx/query-2dc065cc08f262c937c54f9cc8629e35750da2bea995fb0c433893addb253214.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM builds_logs bl\n USING builds b\n JOIN releases r ON b.rid = r.id\n WHERE bl.build_id = b.id AND r.crate_id = $1 AND r.version = $2;", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Text" - ] - }, - "nullable": [] - }, - "hash": "2dc065cc08f262c937c54f9cc8629e35750da2bea995fb0c433893addb253214" -} diff --git a/.sqlx/query-5f5fa0e89b4e13c690b1648a18e8420f7da0f0445c6e43d5d64617226c24fba1.json b/.sqlx/query-5f5fa0e89b4e13c690b1648a18e8420f7da0f0445c6e43d5d64617226c24fba1.json new file mode 100644 index 0000000000..5d451984e9 --- /dev/null +++ b/.sqlx/query-5f5fa0e89b4e13c690b1648a18e8420f7da0f0445c6e43d5d64617226c24fba1.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM releases", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4", + "origin": { + "Table": { + "table": "releases", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "5f5fa0e89b4e13c690b1648a18e8420f7da0f0445c6e43d5d64617226c24fba1" +} diff --git a/.sqlx/query-66b0ba6978880b79ce7a179bbe986e6c7eed78a2d4f01f316772949f5d688f42.json b/.sqlx/query-66b0ba6978880b79ce7a179bbe986e6c7eed78a2d4f01f316772949f5d688f42.json new file mode 100644 index 0000000000..89cbc239cc --- /dev/null +++ b/.sqlx/query-66b0ba6978880b79ce7a179bbe986e6c7eed78a2d4f01f316772949f5d688f42.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM builds_logs bl\n USING builds b\n WHERE bl.build_id = b.id AND b.rid = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "66b0ba6978880b79ce7a179bbe986e6c7eed78a2d4f01f316772949f5d688f42" +} diff --git a/.sqlx/query-7b7dd5795cddcb66b140b57157983bd73f73ecc1cf9b4fc24c457d5f26fd582b.json b/.sqlx/query-7b7dd5795cddcb66b140b57157983bd73f73ecc1cf9b4fc24c457d5f26fd582b.json new file mode 100644 index 0000000000..effb9f9ac3 --- /dev/null +++ b/.sqlx/query-7b7dd5795cddcb66b140b57157983bd73f73ecc1cf9b4fc24c457d5f26fd582b.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM releases WHERE crate_id = $1 AND version = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4", + "origin": { + "Table": { + "table": "releases", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int4", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7b7dd5795cddcb66b140b57157983bd73f73ecc1cf9b4fc24c457d5f26fd582b" +} diff --git a/.sqlx/query-014a054d852f0937191e1a54f742d4b4c454361689fb3841cc12fd7dd1094948.json b/.sqlx/query-fab139cabc0987a1f2ad706060a3f4254924db75fc7f76a6c78d17a3fc06d663.json similarity index 65% rename from .sqlx/query-014a054d852f0937191e1a54f742d4b4c454361689fb3841cc12fd7dd1094948.json rename to .sqlx/query-fab139cabc0987a1f2ad706060a3f4254924db75fc7f76a6c78d17a3fc06d663.json index 380bd9ea67..1f16abd551 100644 --- a/.sqlx/query-014a054d852f0937191e1a54f742d4b4c454361689fb3841cc12fd7dd1094948.json +++ b/.sqlx/query-fab139cabc0987a1f2ad706060a3f4254924db75fc7f76a6c78d17a3fc06d663.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM releases WHERE crate_id = $1 AND version = $2 RETURNING is_library", + "query": "DELETE FROM releases WHERE id = $1 RETURNING is_library", "describe": { "columns": [ { @@ -17,13 +17,12 @@ ], "parameters": { "Left": [ - "Int4", - "Text" + "Int4" ] }, "nullable": [ true ] }, - "hash": "014a054d852f0937191e1a54f742d4b4c454361689fb3841cc12fd7dd1094948" + "hash": "fab139cabc0987a1f2ad706060a3f4254924db75fc7f76a6c78d17a3fc06d663" } diff --git a/Cargo.lock b/Cargo.lock index 48b171579a..f3b6de17ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -485,11 +485,11 @@ dependencies = [ "aws-runtime", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-types", "bytes", @@ -546,7 +546,7 @@ dependencies = [ "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-http 0.64.0", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -577,12 +577,12 @@ dependencies = [ "aws-smithy-async", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -601,6 +601,31 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-sqs" +version = "1.102.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0246bf049cfc003ce44599dff955b9353758de3afa68a053da9b2c7de20a07d8" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.63.6", + "aws-smithy-json 0.62.7", + "aws-smithy-observability 0.2.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.5.0", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "1.112.0" @@ -611,13 +636,13 @@ dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-observability", + "aws-smithy-http 0.64.0", + "aws-smithy-json 0.63.0", + "aws-smithy-observability 0.3.0", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -636,7 +661,7 @@ checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-http 0.64.0", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -668,7 +693,7 @@ version = "0.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" dependencies = [ - "aws-smithy-http", + "aws-smithy-http 0.64.0", "aws-smithy-types", "bytes", "crc-fast", @@ -694,6 +719,27 @@ dependencies = [ "crc32fast", ] +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.5.0", + "http-body 1.1.0", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + [[package]] name = "aws-smithy-http" version = "0.64.0" @@ -740,6 +786,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-json" +version = "0.62.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema 0.1.0", + "aws-smithy-types", +] + [[package]] name = "aws-smithy-json" version = "0.63.0" @@ -747,10 +804,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", ] +[[package]] +name = "aws-smithy-observability" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +dependencies = [ + "aws-smithy-runtime-api", +] + [[package]] name = "aws-smithy-observability" version = "0.3.0" @@ -767,7 +833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "aws-smithy-xml", "urlencoding", @@ -780,11 +846,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" dependencies = [ "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.64.0", "aws-smithy-http-client", - "aws-smithy-observability", + "aws-smithy-observability 0.3.0", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "bytes", "fastrand", @@ -828,6 +894,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "aws-smithy-schema" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.5.0", +] + [[package]] name = "aws-smithy-schema" version = "0.2.0" @@ -882,7 +959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "xmlparser", ] @@ -896,7 +973,7 @@ dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-schema", + "aws-smithy-schema 0.2.0", "aws-smithy-types", "rustc_version", "tracing", @@ -2488,12 +2565,16 @@ name = "docs_rs_watcher" version = "0.6.0" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-sqs", + "chrono", "clap", "crates-index", "crates-index-diff", "docs_rs_build_queue", "docs_rs_config", "docs_rs_context", + "docs_rs_crates_io", "docs_rs_database", "docs_rs_env_vars", "docs_rs_fastly", @@ -2510,9 +2591,11 @@ dependencies = [ "opentelemetry", "pretty_assertions", "rayon", + "serde_json", "sqlx", "tokio", "tracing", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1684d19f5f..9e656de1d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,11 @@ edition = "2024" anyhow = { version = "1.0.42", features = ["backtrace"] } askama = "0.16.0" async-stream = "0.3.5" +# The default `rustls` feature pulls in the legacy hyper 0.14 + rustls 0.21 +# stack via `aws-smithy-runtime/tls-rustls`, which includes the vulnerable +# `rustls-webpki` v0.101.x. Using only `default-https-client` avoids this by +# using the modern rustls 0.23 + hyper 1.x stack instead. +aws-config = { version = "1.0.0", default-features = false, features = ["default-https-client", "rt-tokio"] } axum-extra = { version = "0.12.0", features = ["middleware", "routing", "typed-header"] } base64 = "0.23" bon = { version = "3.8.1", features = ["experimental-overwritable"] } diff --git a/crates/bin/cratesfyi/src/daemon.rs b/crates/bin/cratesfyi/src/daemon.rs index 2ea37bd6b4..f8b7f58082 100644 --- a/crates/bin/cratesfyi/src/daemon.rs +++ b/crates/bin/cratesfyi/src/daemon.rs @@ -4,7 +4,7 @@ use docs_rs_config::AppConfig as _; use docs_rs_context::Context; use docs_rs_watcher::{ start_background_queue_rebuild, start_background_repository_stats_updater, - start_background_service_metric_collector, watch_registry, + start_background_service_metric_collector, }; use docs_rs_web::run_web_server; use std::sync::Arc; @@ -21,7 +21,7 @@ fn start_registry_watcher( // space this out to prevent it from clashing against the queue-builder thread on launch tokio::time::sleep(Duration::from_secs(30)).await; - watch_registry(&config, &context).await + docs_rs_watcher::watch(&config, &context).await; }); Ok(()) diff --git a/crates/bin/cratesfyi/src/main.rs b/crates/bin/cratesfyi/src/main.rs index ccdb6c767c..e6f65026f4 100644 --- a/crates/bin/cratesfyi/src/main.rs +++ b/crates/bin/cratesfyi/src/main.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + use anyhow::Result; use clap::Parser; use cratesfyi::daemon::start_daemon; diff --git a/crates/bin/docs_rs_watcher/Cargo.toml b/crates/bin/docs_rs_watcher/Cargo.toml index 30f38a6e8b..7600139f82 100644 --- a/crates/bin/docs_rs_watcher/Cargo.toml +++ b/crates/bin/docs_rs_watcher/Cargo.toml @@ -8,6 +8,9 @@ edition.workspace = true [dependencies] anyhow = { workspace = true } +aws-config = { workspace = true } +aws-sdk-sqs = { version = "1.99.0", default-features = false, features = ["default-https-client", "rt-tokio"] } +chrono = { workspace = true } clap = { workspace = true } # NOTE: on the new infra, switch back from `git-https-reqwest` to `git-https` (curl) once the curl version is new enough crates-index = { version = "3.0.0", default-features = false, features = ["git", "git-https-reqwest", "git-performance", "parallel"] } @@ -16,6 +19,7 @@ crates-index-diff = { version = "31.0.0", default-features = false, features = [ docs_rs_build_queue = { path = "../../lib/docs_rs_build_queue" } docs_rs_config = { path = "../../lib/docs_rs_config" } docs_rs_context = { path = "../../lib/docs_rs_context" } +docs_rs_crates_io = { path = "../../lib/docs_rs_crates_io" } docs_rs_database = { path = "../../lib/docs_rs_database" } docs_rs_env_vars = { path = "../../lib/docs_rs_env_vars" } docs_rs_fastly = { path = "../../lib/docs_rs_fastly" } @@ -29,9 +33,11 @@ futures-util = { workspace = true } itertools = { workspace = true } opentelemetry = { workspace = true } rayon = "1.6.1" +serde_json = { workspace = true } sqlx = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +url = { workspace = true } [dev-dependencies] docs_rs_config = { path = "../../lib/docs_rs_config", features = ["testing"] } diff --git a/crates/bin/docs_rs_watcher/src/config.rs b/crates/bin/docs_rs_watcher/src/config.rs index 7b5f179760..5f7f105fcd 100644 --- a/crates/bin/docs_rs_watcher/src/config.rs +++ b/crates/bin/docs_rs_watcher/src/config.rs @@ -2,15 +2,54 @@ use anyhow::Result; use docs_rs_config::AppConfig; use docs_rs_env_vars::{env, maybe_env, require_env}; use std::{path::PathBuf, time::Duration}; +use url::Url; + +const SQS_QUEUE_URL: &str = "DOCSRS_SQS_QUEUE_URL"; +const SQS_QUEUE_REGION: &str = "DOCSRS_SQS_QUEUE_REGION"; + +#[derive(Debug)] +pub struct SqsConfig { + pub queue_url: Url, + pub region: String, + pub endpoint_url: Option, + pub max_retries: u32, + /// temporary, to switch between the sources for the index (git index vs SQS). + /// true = only use SQS, don't even fetch git + /// false = fetch both sqs & git, use git, just log sqs. + pub active: bool, +} + +impl SqsConfig { + pub(crate) fn if_configured() -> Result> { + if maybe_env::(SQS_QUEUE_URL)?.is_some() + && maybe_env::(SQS_QUEUE_REGION)?.is_some() + { + SqsConfig::from_environment().map(Some) + } else { + Ok(None) + } + } +} + +impl AppConfig for SqsConfig { + fn from_environment() -> Result { + Ok(Self { + queue_url: require_env(SQS_QUEUE_URL)?, + region: require_env(SQS_QUEUE_REGION)?, + endpoint_url: maybe_env("DOCSRS_SQS_ENDPOINT_URL")?, + active: env("DOCSRS_SQS_ACTIVE", false)?, + max_retries: env("DOCSRS_SQS_MAX_RETRIES", 6u32)?, + }) + } +} #[derive(Debug)] pub struct Config { + /// registry watching config. Also used for database-synchonize pub registry_index_path: PathBuf, pub registry_url: Option, - /// How long to wait between registry checks pub delay_between_registry_fetches: Duration, - // Time between 'git gc --auto' calls in seconds pub registry_gc_interval: u64, @@ -20,15 +59,29 @@ pub struct Config { /// Maximum time to wait for queue row locks when deleting crates/releases. pub delete_lock_timeout: Duration, + pub crates_io_events: Option, + pub repository: docs_rs_repository_stats::Config, } +impl Config { + pub fn crates_io_events_active(&self) -> bool { + self.crates_io_events + .as_ref() + .map(|config| config.active) + .unwrap_or(false) + } +} + impl AppConfig for Config { fn from_environment() -> Result { let prefix: PathBuf = require_env("DOCSRS_PREFIX")?; Ok(Self { registry_index_path: env("REGISTRY_INDEX_PATH", prefix.join("crates.io-index"))?, registry_url: maybe_env("REGISTRY_URL")?, + + crates_io_events: SqsConfig::if_configured()?, + delay_between_registry_fetches: Duration::from_secs(env::( "DOCSRS_DELAY_BETWEEN_REGISTRY_FETCHES", 60, @@ -42,4 +95,13 @@ impl AppConfig for Config { repository: docs_rs_repository_stats::Config::from_environment()?, }) } + + #[cfg(test)] + fn test_config() -> Result { + let mut config = Self::from_environment()?; + if let Some(sqs_config) = &mut config.crates_io_events { + sqs_config.active = false; + } + Ok(config) + } } diff --git a/crates/bin/docs_rs_watcher/src/db/delete.rs b/crates/bin/docs_rs_watcher/src/db/delete.rs index 7a5a3fb448..e0b2ce52bf 100644 --- a/crates/bin/docs_rs_watcher/src/db/delete.rs +++ b/crates/bin/docs_rs_watcher/src/db/delete.rs @@ -67,7 +67,13 @@ pub async fn delete_version( return Ok(()); }; - let is_library = delete_version_from_database(conn, config, name, crate_id, version).await?; + let Some(is_library) = + delete_version_from_database(conn, config, name, crate_id, version).await? + else { + // release doesn't exist + return Ok(()); + }; + let paths = if is_library { LIBRARY_STORAGE_PATHS_TO_DELETE } else { @@ -133,7 +139,18 @@ async fn delete_version_from_database( name: &KrateName, crate_id: CrateId, version: &Version, -) -> Result { +) -> Result> { + let Some(release_id) = sqlx::query_scalar!( + "SELECT id FROM releases WHERE crate_id = $1 AND version = $2", + crate_id as _, + version as _ + ) + .fetch_optional(&mut *conn) + .await? + else { + return Ok(None); + }; + let mut transaction = conn.begin().await?; let delete_lock_timeout = format!("{}ms", config.delete_lock_timeout.as_millis()); @@ -157,23 +174,23 @@ async fn delete_version_from_database( sqlx::query!( "DELETE FROM builds_logs bl USING builds b - JOIN releases r ON b.rid = r.id - WHERE bl.build_id = b.id AND r.crate_id = $1 AND r.version = $2;", - crate_id as _, - version as _ + WHERE bl.build_id = b.id AND b.rid = $1;", + release_id as _, ) .execute(&mut *transaction) .await?; for &(table, column) in METADATA { - sqlx::query(sqlx::AssertSqlSafe( - format!("DELETE FROM {table} WHERE {column} IN (SELECT id FROM releases WHERE crate_id = $1 AND version = $2)"))) - .bind(crate_id).bind(version).execute(&mut *transaction).await?; + sqlx::query(sqlx::AssertSqlSafe(format!( + "DELETE FROM {table} WHERE {column} = $1" + ))) + .bind(release_id) + .execute(&mut *transaction) + .await?; } let is_library: bool = sqlx::query_scalar!( - "DELETE FROM releases WHERE crate_id = $1 AND version = $2 RETURNING is_library", - crate_id.0, - version as _, + "DELETE FROM releases WHERE id = $1 RETURNING is_library", + release_id as _, ) .fetch_one(&mut *transaction) .await? @@ -190,7 +207,7 @@ async fn delete_version_from_database( update_latest_version_id(&mut transaction, crate_id).await?; transaction.commit().await?; - Ok(is_library) + Ok(Some(is_library)) } /// Returns whether any release in this crate was a library @@ -406,6 +423,13 @@ mod tests { assert!(!storage.exists(&rustdoc_archive_path(&FOO, &V1)).await?); assert!(!storage.exists(&rustdoc_archive_path(&FOO, &V2)).await?); + // running delete-crate again doesn't error. + assert!( + delete_crate(&mut conn, storage, env.config(), &FOO) + .await + .is_ok() + ); + Ok(()) } @@ -534,6 +558,13 @@ mod tests { vec!["Peter Rabbit".to_string()] ); + // running delete-version again doesn't fail. + assert!( + delete_version(&mut conn, storage, env.config(), &KRATE, &V1) + .await + .is_ok() + ); + // FIXME: remove for now until test frontend is async // let web = env.frontend(); // assert_success("/a/2.0.0/a/", web)?; @@ -612,6 +643,32 @@ mod tests { Ok(()) } + #[tokio::test(flavor = "multi_thread")] + async fn test_delete_already_deleted_version_doesnt_error() -> Result<()> { + let env = TestEnvironment::new().await?; + let mut conn = env.async_conn().await?; + + env.fake_release() + .await + .name(&KRATE) + .version(V1) + .create() + .await?; + env.fake_release() + .await + .name(&KRATE) + .version(V2) + .create() + .await?; + + delete_version(&mut conn, env.storage()?, env.config(), &KRATE, &V1).await?; + delete_version(&mut conn, env.storage()?, env.config(), &KRATE, &V1).await?; + + assert!(crate_exists(&mut conn, &KRATE).await?); + + Ok(()) + } + #[tokio::test(flavor = "multi_thread")] async fn test_delete_version_waits_for_locked_queue_rows() -> Result<()> { let env = TestEnvironment::new().await?; diff --git a/crates/bin/docs_rs_watcher/src/index_watcher.rs b/crates/bin/docs_rs_watcher/src/index_watcher.rs index 3d03d3ce94..8488fc2a7d 100644 --- a/crates/bin/docs_rs_watcher/src/index_watcher.rs +++ b/crates/bin/docs_rs_watcher/src/index_watcher.rs @@ -2,24 +2,26 @@ use crate::{ Config, db::{delete_crate, delete_version}, index::Index, + metrics::{EventSource, WatcherMetrics}, }; use anyhow::{Context as _, Result}; use crates_index_diff::Change; use docs_rs_build_queue::PRIORITY_MANUAL_FROM_CRATES_IO; use docs_rs_context::Context; +use docs_rs_crates_io::events::ChangeKind; use docs_rs_database::{ crate_details::update_latest_version_id, service_config::{ConfigName, get_config, set_config}, }; use docs_rs_fastly::{Cdn, CdnBehaviour as _}; use docs_rs_types::{CrateId, KrateName, Version}; +use std::time::Instant; use tracing::{debug, error, info, warn}; #[derive(Debug)] pub(crate) struct CrateVersion { pub name: KrateName, pub version: Version, - pub yanked: bool, } #[cfg(test)] @@ -28,19 +30,28 @@ impl Default for CrateVersion { Self { name: docs_rs_types::testing::KRATE, version: docs_rs_types::testing::V1, - yanked: false, } } } -impl TryFrom for CrateVersion { +impl TryFrom<&crates_index_diff::CrateVersion> for CrateVersion { type Error = anyhow::Error; - fn try_from(value: crates_index_diff::CrateVersion) -> Result { + fn try_from(value: &crates_index_diff::CrateVersion) -> Result { + Ok(Self { + name: value.name.parse()?, + version: value.version.parse()?, + }) + } +} + +impl TryFrom<&docs_rs_crates_io::events::CrateVersion> for CrateVersion { + type Error = anyhow::Error; + + fn try_from(value: &docs_rs_crates_io::events::CrateVersion) -> Result { Ok(Self { name: value.name.parse()?, version: value.version.parse()?, - yanked: value.yanked, }) } } @@ -51,7 +62,6 @@ impl From for crates_index_diff::CrateVersion { Self { name: value.name.to_string().into(), version: value.version.to_string().into(), - yanked: value.yanked, ..Default::default() } } @@ -94,6 +104,7 @@ pub(crate) async fn get_new_crates( context: &Context, index: &Index, config: &Config, + metrics: &WatcherMetrics, ) -> Result { let mut conn = context.pool()?.get_async().await?; @@ -115,7 +126,8 @@ pub(crate) async fn get_new_crates( debug!(last_seen_reference=%last_seen_reference, new_reference=%new_reference, "queueing changes"); - let crates_added = process_changes(context, &changes, config).await; + metrics.record_events_received(EventSource::Git, changes.len()); + let crates_added = process_changes(context, &changes, config, metrics).await; if let Err(err) = context.build_queue()?.reevaluate_priorities().await { error!(?err, "error reevaluating queued release priorities"); @@ -129,41 +141,109 @@ pub(crate) async fn get_new_crates( Ok(crates_added) } -async fn process_changes(context: &Context, changes: &Vec, config: &Config) -> usize { +async fn process_changes( + context: &Context, + changes: &Vec, + config: &Config, + metrics: &WatcherMetrics, +) -> usize { let mut crates_added = 0; for change in changes { - match process_change(context, change, config).await { + let start = Instant::now(); + // temporarily log all changes, so we can compare them with the SQS changes we see. + // They share the same log-target, and most tracing fields. + let (change_type, crate_name, crate_version) = match change { + Change::Added(version) => ( + ChangeKind::Added, + version.name.as_str(), + version.version.as_str(), + ), + Change::AddedAndYanked(version) => ( + ChangeKind::AddedAndYanked, + version.name.as_str(), + version.version.as_str(), + ), + Change::Unyanked(version) => ( + ChangeKind::Unyanked, + version.name.as_str(), + version.version.as_str(), + ), + Change::Yanked(version) => ( + ChangeKind::Yanked, + version.name.as_str(), + version.version.as_str(), + ), + Change::CrateDeleted { name, .. } => (ChangeKind::CrateDeleted, name.as_str(), ""), + Change::VersionDeleted(version) => ( + ChangeKind::VersionDeleted, + version.name.as_str(), + version.version.as_str(), + ), + }; + debug!( + target: "docs_rs_watcher::index_event", + source = %EventSource::Git, + change_type = %change_type, + crate_name, + crate_version, + "crates.io index event" + ); + + if config.crates_io_events_active() { + // just to be safe. + // Generally we don't even start the git-index-watcher when + // SQS is active. + // Will be removed with the git index watcher code when SQS is stable. + continue; + } + + let success = match process_change(context, change, config).await { Ok(added) => { + metrics.record_change_applied(EventSource::Git, change_type); if added { crates_added += 1; } + true } Err(err) => { error!(?change, ?err, "failed to process change"); + false } - } + }; + metrics.record_event_processing_time( + EventSource::Git, + Some(change_type), + success, + start.elapsed(), + ); } crates_added } /// Process a crate change, returning whether the change was a crate addition or not. -async fn process_change(context: &Context, change: &Change, config: &Config) -> Result { +pub(crate) async fn process_change( + context: &Context, + change: &Change, + config: &Config, +) -> Result { let crate_version: CrateVersion = change .versions() .first() .expect("always exists") - .clone() .try_into()?; match change { Change::Added(_release) => process_version_added(context, &crate_version).await?, Change::AddedAndYanked(_release) => { process_version_added(context, &crate_version).await?; - process_version_yank_status(context, &crate_version).await?; + process_version_yank_status(context, &crate_version, true).await?; } - Change::Unyanked(_release) | Change::Yanked(_release) => { - process_version_yank_status(context, &crate_version).await? + Change::Unyanked(_release) => { + process_version_yank_status(context, &crate_version, false).await? + } + Change::Yanked(_release) => { + process_version_yank_status(context, &crate_version, true).await? } Change::CrateDeleted { name, .. } => { let name: KrateName = name.parse()?; @@ -177,15 +257,19 @@ async fn process_change(context: &Context, change: &Change, config: &Config) -> } /// Processes crate changes, whether they got yanked or unyanked. -async fn process_version_yank_status(context: &Context, release: &CrateVersion) -> Result<()> { +pub(crate) async fn process_version_yank_status( + context: &Context, + release: &CrateVersion, + yanked: bool, +) -> Result<()> { // FIXME: delay yanks of crates that have not yet finished building // https://github.com/rust-lang/docs.rs/issues/1934 - set_yanked(context, &release.name, &release.version, release.yanked).await?; + set_yanked(context, &release.name, &release.version, yanked).await?; queue_crate_invalidation(&release.name, context.cdn.as_deref()).await; Ok(()) } -async fn process_version_added(context: &Context, release: &CrateVersion) -> Result<()> { +pub(crate) async fn process_version_added(context: &Context, release: &CrateVersion) -> Result<()> { let build_queue = context.build_queue()?; let priority = build_queue.find_priority(&release.name).await?; @@ -217,7 +301,7 @@ async fn process_version_added(context: &Context, release: &CrateVersion) -> Res Ok(()) } -async fn process_version_deleted( +pub(crate) async fn process_version_deleted( context: &Context, config: &Config, release: &CrateVersion, @@ -251,7 +335,7 @@ async fn process_version_deleted( Ok(()) } -async fn process_crate_deleted( +pub(crate) async fn process_crate_deleted( context: &Context, config: &Config, krate: &KrateName, @@ -343,7 +427,6 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V1, - ..Default::default() }; process_version_added(&env, &krate).await?; @@ -354,7 +437,6 @@ mod tests { let krate = CrateVersion { name: "krate".parse()?, version: V2.to_string().parse()?, - ..Default::default() }; process_version_added(&env, &krate).await?; @@ -387,9 +469,8 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V1, - yanked: true, }; - process_version_yank_status(&env, &krate).await?; + process_version_yank_status(&env, &krate, true).await?; // And verify it's actually marked as yanked let row = sqlx::query!( @@ -406,9 +487,8 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V1, - yanked: false, }; - process_version_yank_status(&env, &krate).await?; + process_version_yank_status(&env, &krate, false).await?; let row = sqlx::query!( "SELECT yanked @@ -471,7 +551,6 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V2, - ..Default::default() }; process_version_deleted(&env, env.config(), &krate).await?; @@ -501,23 +580,20 @@ mod tests { let krate1 = CrateVersion { name: KRATE, version: V1, - ..Default::default() }; let krate2 = CrateVersion { name: "krate2".parse()?, version: V1, - ..Default::default() }; let krate_already_present = CrateVersion { name: "krate_already_present".parse()?, version: V1, - ..Default::default() }; let non_existing_version = CrateVersion { name: "krate_already_present".parse()?, version: V2, - ..Default::default() }; + let metrics = WatcherMetrics::new(&env.context().meter_provider); let added = process_changes( &env, &vec![ @@ -531,6 +607,7 @@ mod tests { Change::VersionDeleted(non_existing_version.into()), ], env.config(), + &metrics, ) .await; diff --git a/crates/bin/docs_rs_watcher/src/lib.rs b/crates/bin/docs_rs_watcher/src/lib.rs index 833a6c6885..327048bc11 100644 --- a/crates/bin/docs_rs_watcher/src/lib.rs +++ b/crates/bin/docs_rs_watcher/src/lib.rs @@ -1,10 +1,14 @@ +#![recursion_limit = "256"] + mod config; pub mod consistency; mod db; mod index; pub mod index_watcher; +mod metrics; mod rebuilds; mod service_metrics; +mod subscriber; #[cfg(test)] mod testing; @@ -13,7 +17,9 @@ pub use db::{delete_crate, delete_version}; pub use index::Index; pub use rebuilds::queue_rebuilds; -use crate::{index_watcher::get_new_crates, service_metrics::OtelServiceMetrics}; +use crate::{ + index_watcher::get_new_crates, metrics::WatcherMetrics, service_metrics::OtelServiceMetrics, +}; use anyhow::Result; use docs_rs_context::Context; use docs_rs_utils::start_async_cron; @@ -21,10 +27,65 @@ use std::{sync::Arc, time::Duration}; use tokio::time::{self, Instant}; use tracing::{debug, error, info, trace}; +/// main index-watcher / subscriber loop. +/// mostly wraps either the git index watcher loop, or the sqs subscriber loop. +/// Only here so unexpected errors lead to a sentry report & restart instead of +/// the daemon / watcher just stopping. +pub async fn watch(config: &Config, context: &Context) { + let metrics = WatcherMetrics::new(context.meter_provider()); + + // NOTE: for now we don't have a graceful shutdown. + // Since we currently always lock the queue & builds before deploys, that's + // not a problem. + // But I assume with the new AWS infra we need to solve this at some point so we don't loose + // events. + + loop { + if config.crates_io_events_active() { + if let Err(err) = crate::subscriber::run_sqs_subscriber(config, context, &metrics).await + { + error!(?err, "unexpected error watching SQS, will retry"); + time::sleep(Duration::from_secs(10)).await; + } + } else { + // intermediate mode: + // - still fetch from git for events + // - listen so SQS, and log the events so we can test SQS connection, and compare events + // + // We don't retry on unespected SQS errors yet. + + let registry_watcher = crate::watch_registry(config, context, &metrics); + tokio::pin!(registry_watcher); + + let registry_result = tokio::select! { + result = &mut registry_watcher => result, + sqs_result = crate::subscriber::run_sqs_subscriber(config, context, &metrics) => { + // Unexpected SQS errors stop the test subscriber, but the registry watcher + // remains the authoritative source and must keep running. + if let Err(err) = sqs_result { + error!(?err, "error setting up SQS test subscriber"); + } + registry_watcher.await + } + }; + + if let Err(err) = registry_result { + // unexpected index watcher errors lead to a report & retry. + error!(?err, "unexpected error watching registry, will retry"); + time::sleep(Duration::from_secs(10)).await; + } + } + } +} + /// Run the registry watcher /// NOTE: this should only be run once, otherwise crates would be added /// to the queue multiple times. -pub async fn watch_registry(config: &Config, context: &Context) -> Result<()> { +async fn watch_registry( + config: &Config, + context: &Context, + metrics: &WatcherMetrics, +) -> Result<()> { let mut last_gc = Instant::now(); let queue = context.build_queue()?; @@ -36,9 +97,10 @@ pub async fn watch_registry(config: &Config, context: &Context) -> Result<()> { debug!("Checking new crates"); let index = Index::from_config(config).await?; - match get_new_crates(context, &index, config).await { + match get_new_crates(context, &index, config, metrics).await { Ok(n) => debug!("{} crates added to queue", n), Err(e) => { + metrics.record_poll_error(crate::metrics::EventSource::Git); error!(?e, "Failed to get new crates"); } } diff --git a/crates/bin/docs_rs_watcher/src/main.rs b/crates/bin/docs_rs_watcher/src/main.rs index ebc4f728f9..e5a276a9bd 100644 --- a/crates/bin/docs_rs_watcher/src/main.rs +++ b/crates/bin/docs_rs_watcher/src/main.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + use anyhow::{Context as _, Result}; use clap::{Parser, Subcommand}; use docs_rs_config::AppConfig as _; @@ -82,7 +84,7 @@ impl CommandLine { // which should only run once, and all the time. docs_rs_watcher::start_background_service_metric_collector(&ctx).await?; - docs_rs_watcher::watch_registry(&config, &ctx).await?; + docs_rs_watcher::watch(&config, &ctx).await; } Self::Queue { subcommand } => subcommand.handle_args(config, ctx).await?, Self::Database { subcommand } => subcommand.handle_args(config, ctx).await?, diff --git a/crates/bin/docs_rs_watcher/src/metrics.rs b/crates/bin/docs_rs_watcher/src/metrics.rs new file mode 100644 index 0000000000..56fad5fa4d --- /dev/null +++ b/crates/bin/docs_rs_watcher/src/metrics.rs @@ -0,0 +1,118 @@ +use docs_rs_crates_io::events::ChangeKind; +use docs_rs_opentelemetry::AnyMeterProvider; +use opentelemetry::{ + KeyValue, + metrics::{Counter, Histogram}, +}; +use std::{fmt, time::Duration}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum EventSource { + Git, + Sqs, +} + +impl EventSource { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Git => "git", + Self::Sqs => "sqs", + } + } +} + +impl fmt::Display for EventSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug)] +pub(crate) struct WatcherMetrics { + events_received_total: Counter, + poll_errors_total: Counter, + changes_applied_total: Counter, + event_processing_time: Histogram, + event_lag: Histogram, +} + +impl WatcherMetrics { + pub(crate) fn new(meter_provider: &AnyMeterProvider) -> Self { + let meter = meter_provider.meter("watcher"); + const PREFIX: &str = "docsrs.watcher"; + Self { + events_received_total: meter + .u64_counter(format!("{PREFIX}.events_received_total")) + .with_unit("1") + .build(), + poll_errors_total: meter + .u64_counter(format!("{PREFIX}.poll_errors_total")) + .with_unit("1") + .build(), + changes_applied_total: meter + .u64_counter(format!("{PREFIX}.changes_applied_total")) + .with_unit("1") + .build(), + event_processing_time: meter + .f64_histogram(format!("{PREFIX}.event_processing_time")) + .with_boundaries(vec![ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, + 45.0, 55.0, 60.0, 65.0, 90.0, 120.0, + ]) + .with_unit("s") + .build(), + event_lag: meter + .f64_histogram(format!("{PREFIX}.event_lag")) + .with_boundaries(vec![ + 0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0, 900.0, 3600.0, + ]) + .with_unit("s") + .build(), + } + } + + pub(crate) fn record_change_applied(&self, source: EventSource, kind: ChangeKind) { + self.changes_applied_total.add( + 1, + &[ + KeyValue::new("source", source.as_str()), + KeyValue::new("type", kind.as_str()), + ], + ); + } + + pub(crate) fn record_event_lag(&self, source: EventSource, duration: Duration) { + self.event_lag.record( + duration.as_secs_f64(), + &[KeyValue::new("source", source.as_str())], + ); + } + + pub(crate) fn record_event_processing_time( + &self, + source: EventSource, + kind: Option, + success: bool, + duration: Duration, + ) { + let result = if success { "ok" } else { "err" }; + self.event_processing_time.record( + duration.as_secs_f64(), + &[ + KeyValue::new("source", source.as_str()), + KeyValue::new("type", kind.map(ChangeKind::as_str).unwrap_or("unknown")), + KeyValue::new("result", result), + ], + ); + } + + pub(crate) fn record_events_received(&self, source: EventSource, count: usize) { + self.events_received_total + .add(count as u64, &[KeyValue::new("source", source.as_str())]); + } + + pub(crate) fn record_poll_error(&self, source: EventSource) { + self.poll_errors_total + .add(1, &[KeyValue::new("source", source.as_str())]); + } +} diff --git a/crates/bin/docs_rs_watcher/src/subscriber.rs b/crates/bin/docs_rs_watcher/src/subscriber.rs new file mode 100644 index 0000000000..15b044d75c --- /dev/null +++ b/crates/bin/docs_rs_watcher/src/subscriber.rs @@ -0,0 +1,675 @@ +use crate::{ + Config, + index_watcher::{ + process_crate_deleted, process_version_added, process_version_deleted, + process_version_yank_status, + }, + metrics::{EventSource, WatcherMetrics}, +}; +use anyhow::{Context as _, Result}; +use aws_config::{BehaviorVersion, Region, retry::RetryConfig}; +use aws_sdk_sqs::{Client, types::Message}; +use chrono::Utc; +use docs_rs_context::Context; +use docs_rs_crates_io::events::{IndexChangeEventV1, IndexChangeV1}; +use docs_rs_types::KrateName; +use docs_rs_utils::retry_async; +use std::time::{Duration, Instant}; +use tokio::time; +use tracing::{debug, error, instrument, warn}; + +/// wait-time (long polling): +/// +/// How long should the request be kept open when there are no messages. +/// SQS only accepts values in the range 0..=20 seconds. +const WAIT_TIME: Duration = Duration::from_secs(20); + +/// when one long-polling request is finished, how long to sleep before starting the next? +const SLEEP_BETWEEN_REQUESTS: Duration = Duration::from_secs(1); + +/// How regularly to recheck the priorities of queued crates. +/// Right now only runs `deprioritize_workspaces`. +const DELAY_BETWEEN_PRIORITY_RECHECK: Duration = Duration::from_secs(60); + +/// visibility timeout: +/// SQS visibility timeout is the period after a consumer receives a message during +/// which that message is hidden from other consumers, and if it is not deleted before +/// the timeout expires, it becomes visible again for redelivery. +/// +/// Should be longer than the longest time our server takes to handle a message. +const VISIBILITY_TIMEOUT: Duration = Duration::from_secs(600); + +trait SqsActions { + async fn delete_message(&self, queue_url: &str, receipt_handle: &str) -> Result<()>; +} + +impl SqsActions for Client { + async fn delete_message(&self, queue_url: &str, receipt_handle: &str) -> Result<()> { + self.delete_message() + .queue_url(queue_url) + .receipt_handle(receipt_handle) + .send() + .await + .context("error deleting SQS message")?; + Ok(()) + } +} + +pub(crate) async fn run_sqs_subscriber( + config: &Config, + context: &Context, + metrics: &WatcherMetrics, +) -> Result<()> { + let Some(sqs_config) = &config.crates_io_events else { + warn!("missing sqs config, disabling crates.io SQS subscriber"); + return Ok(()); + }; + let mut last_priority_recheck = Instant::now(); + let queue = context.build_queue()?; + + debug!("creating SQS client..."); + let shared_config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let mut client_config = aws_sdk_sqs::config::Builder::from(&shared_config) + .retry_config(RetryConfig::standard().with_max_attempts(sqs_config.max_retries)) + .region(Region::new(sqs_config.region.to_string())); + if let Some(endpoint_url) = &sqs_config.endpoint_url { + client_config = client_config.endpoint_url(endpoint_url.to_string()); + } + let client = Client::from_conf(client_config.build()); + + let queue_url = sqs_config.queue_url.to_string(); + + loop { + if queue.is_locked().await? { + debug!("Queue is locked, skipping checking new crates"); + time::sleep(WAIT_TIME).await; + continue; + } + + debug!("receiving messages..."); + let messages = match client + .receive_message() + .queue_url(&queue_url) + // confirm that we want to do batches. + // important because it's a FIFO queue: + // NOTE: when we start retrying tasks with a FIFO queute. + // important: return on on the first erroring message, don't + // handle the rest of the batch. + .max_number_of_messages(10) + .wait_time_seconds(WAIT_TIME.as_secs() as i32) + .visibility_timeout(VISIBILITY_TIMEOUT.as_secs() as i32) + .send() + .await + { + Ok(response) => response.messages().to_vec(), + Err(err) => { + // NOTE: right now we handle the change-events like the old + // git index: on error just skip over the event, handle the next. + // Future improvement: retry the task for retryable errors. + metrics.record_poll_error(EventSource::Sqs); + error!(?err, queue_url, "error receiving messages from sqs"); + time::sleep(WAIT_TIME).await; + continue; + } + }; + process_messages(&client, &queue_url, context, config, metrics, messages).await; + + if last_priority_recheck.elapsed() >= DELAY_BETWEEN_PRIORITY_RECHECK { + if let Err(err) = queue.reevaluate_priorities().await { + error!(?err, "error reevaluating queued release priorities"); + } + + last_priority_recheck = Instant::now(); + } + + time::sleep(SLEEP_BETWEEN_REQUESTS).await; + } +} + +async fn process_messages( + client: &impl SqsActions, + queue_url: &str, + context: &Context, + config: &Config, + metrics: &WatcherMetrics, + messages: Vec, +) { + for message in messages { + handle_message_body(context, config, metrics, message.body.as_deref()).await; + if let Some(receipt_handle) = message.receipt_handle.as_deref() + && let Err(err) = client.delete_message(queue_url, receipt_handle).await + { + error!(?err, receipt_handle, "error deleting message from queue"); + } + } +} + +async fn handle_message_body( + context: &Context, + config: &Config, + metrics: &WatcherMetrics, + body: Option<&str>, +) { + let Some(body) = body else { + return; + }; + if let Err(err) = process_sqs_event(context, config, metrics, body).await { + // Match the git-index watcher behavior for the initial rollout: record and skip + // failed events instead of letting one event block the FIFO queue indefinitely. + error!(?err, body, "error handling message, skipping event"); + } +} + +#[instrument(skip_all)] +async fn process_sqs_event( + context: &Context, + config: &Config, + metrics: &WatcherMetrics, + body: &str, +) -> Result<()> { + metrics.record_events_received(EventSource::Sqs, 1); + + let start = Instant::now(); + let event: IndexChangeEventV1 = match serde_json::from_str(body) { + Ok(event) => event, + Err(err) => { + metrics.record_event_processing_time(EventSource::Sqs, None, false, start.elapsed()); + return Err(err).context("error parsing event from json"); + } + }; + + debug!( + target: "docs_rs_watcher::index_event", + source = %EventSource::Sqs, + event_id = %event.id, + occurred_at = %event.occurred_at, + change_type = %event.change.kind(), + crate_name = event.change.name(), + crate_version = event.change.version().unwrap_or_default(), + "crates.io index event" + ); + + if let Ok(lag) = (Utc::now() - event.occurred_at).to_std() { + metrics.record_event_lag(EventSource::Sqs, lag); + } + + let processing_result = if config.crates_io_events_active() { + retry_async( + || { + let change = event.change.clone(); + async move { process_change(context, &change, config).await } + }, + 3, + ) + .await + .context("error processing change") + } else { + Ok(()) + }; + + metrics.record_event_processing_time( + EventSource::Sqs, + Some(event.change.kind()), + processing_result.is_ok(), + start.elapsed(), + ); + processing_result?; + + if config.crates_io_events_active() { + metrics.record_change_applied(EventSource::Sqs, event.change.kind()); + } + + Ok(()) +} + +/// Process a crate change +#[instrument(skip(context, config))] +pub(crate) async fn process_change( + context: &Context, + change: &IndexChangeV1, + config: &Config, +) -> Result<()> { + match change { + IndexChangeV1::Added(crate_version) => { + process_version_added(context, &crate_version.try_into()?).await? + } + IndexChangeV1::Yanked(crate_version) => { + process_version_yank_status(context, &crate_version.try_into()?, true).await? + } + IndexChangeV1::Unyanked(crate_version) => { + process_version_yank_status(context, &crate_version.try_into()?, false).await? + } + IndexChangeV1::CrateDeleted { name, .. } => { + let name: KrateName = name.parse()?; + process_crate_deleted(context, config, &name).await? + } + IndexChangeV1::VersionDeleted(crate_version) => { + process_version_deleted(context, config, &crate_version.try_into()?).await? + } + }; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::TestEnvironment; + use docs_rs_config::AppConfig as _; + use docs_rs_crates_io::events::CrateVersion; + use docs_rs_types::{ + Version, + testing::{KRATE, V1, V2}, + }; + use pretty_assertions::assert_eq; + use std::sync::Mutex; + + #[derive(Default)] + struct FakeSqsActions { + deleted: Mutex>, + } + + impl SqsActions for FakeSqsActions { + async fn delete_message(&self, _queue_url: &str, receipt_handle: &str) -> Result<()> { + self.deleted.lock().unwrap().push(receipt_handle.into()); + Ok(()) + } + } + + fn added_event_json(name: &KrateName, version: &Version) -> String { + serde_json::to_string(&serde_json::json!({ + "id":"evt_123", + "occurred_at":"2026-06-01T12:00:00Z", + "type":"added", + "payload":{ + "name": name.to_string(), + "vers": version.to_string(), + } + })) + .unwrap() + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_added_queues_crate() -> Result<()> { + let env = TestEnvironment::new().await?; + + process_change( + &env, + &IndexChangeV1::Added(CrateVersion { + name: KRATE.to_string(), + version: V1.to_string(), + }), + env.config(), + ) + .await?; + + let queue = env.build_queue()?.queued_crates().await?; + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].name, KRATE); + assert_eq!(queue[0].version, V1); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_yanked_updates_release() -> Result<()> { + let env = TestEnvironment::new().await?; + let mut conn = env.async_conn().await?; + + let id = env + .fake_release() + .await + .name(KRATE) + .version(V1) + .create() + .await?; + + process_change( + &env, + &IndexChangeV1::Yanked(CrateVersion { + name: KRATE.to_string(), + version: V1.to_string(), + }), + env.config(), + ) + .await?; + + let yanked = sqlx::query_scalar!( + "SELECT yanked + FROM releases + WHERE id = $1", + id.0 + ) + .fetch_one(&mut *conn) + .await?; + assert_eq!(yanked, Some(true)); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_unyanked_updates_release() -> Result<()> { + let env = TestEnvironment::new().await?; + let mut conn = env.async_conn().await?; + + let id = env + .fake_release() + .await + .name(KRATE) + .version(V1) + .yanked(true) + .create() + .await?; + + process_change( + &env, + &IndexChangeV1::Unyanked(CrateVersion { + name: KRATE.to_string(), + version: V1.to_string(), + }), + env.config(), + ) + .await?; + + let row = sqlx::query!( + "SELECT yanked + FROM releases + WHERE id = $1", + id.0 + ) + .fetch_one(&mut *conn) + .await?; + assert_eq!(row.yanked, Some(false)); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_crate_deleted_removes_crate() -> Result<()> { + let env = TestEnvironment::new().await?; + let mut conn = env.async_conn().await?; + + env.fake_release() + .await + .name(KRATE) + .version(V1) + .create() + .await?; + + process_change( + &env, + &IndexChangeV1::CrateDeleted { + name: KRATE.to_string(), + }, + env.config(), + ) + .await?; + + let row = sqlx::query!( + "SELECT id + FROM crates + WHERE name = $1", + KRATE as _ + ) + .fetch_optional(&mut *conn) + .await?; + assert!(row.is_none()); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_added_is_idempotent() -> Result<()> { + let env = TestEnvironment::new().await?; + let change = IndexChangeV1::Added(CrateVersion { + name: KRATE.to_string(), + version: V1.to_string(), + }); + + process_change(&env, &change, env.config()).await?; + process_change(&env, &change, env.config()).await?; + + let queue = env.build_queue()?.queued_crates().await?; + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].name, KRATE); + assert_eq!(queue[0].version, V1); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_version_deleted_removes_release() -> Result<()> { + let env = TestEnvironment::new().await?; + let mut conn = env.async_conn().await?; + + let rid_1 = env + .fake_release() + .await + .name(KRATE) + .version(V1) + .create() + .await?; + env.fake_release() + .await + .name(KRATE) + .version(V2) + .create() + .await?; + + process_change( + &env, + &IndexChangeV1::VersionDeleted(CrateVersion { + name: KRATE.to_string(), + version: V2.to_string(), + }), + env.config(), + ) + .await?; + + assert_eq!( + sqlx::query_scalar!("SELECT id FROM releases") + .fetch_all(&mut *conn) + .await?, + vec![rid_1.0] + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_change_version_deleted_is_idempotent() -> Result<()> { + let env = TestEnvironment::new().await?; + env.fake_release() + .await + .name(KRATE) + .version(V1) + .create() + .await?; + let change = IndexChangeV1::VersionDeleted(CrateVersion { + name: KRATE.to_string(), + version: V1.to_string(), + }); + + process_change(&env, &change, env.config()).await?; + process_change(&env, &change, env.config()).await?; + + assert!( + sqlx::query_scalar!("SELECT id FROM releases") + .fetch_all(&mut *env.async_conn().await?) + .await? + .is_empty() + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_sqs_event_dispatches_added_event() -> Result<()> { + let mut config = Config::test_config()?; + if let Some(sqs_config) = &mut config.crates_io_events { + sqs_config.active = true; + } + let env = TestEnvironment::builder().config(config).build().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + + process_sqs_event(&env, env.config(), &metrics, &added_event_json(&KRATE, &V1)).await?; + + let queue = env.build_queue()?.queued_crates().await?; + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].name, KRATE); + assert_eq!(queue[0].version, V1); + let collected = env.collected_metrics(); + let applied_metric = + collected.get_metric("watcher", "docsrs.watcher.changes_applied_total")?; + let applied = applied_metric.get_u64_counter(); + let change_type = applied + .attributes() + .find(|kv| kv.key.as_str() == "type") + .unwrap() + .value + .to_string(); + assert_eq!(change_type, "added"); + assert_eq!(applied.value(), 1); + let lag_metric = collected.get_metric("watcher", "docsrs.watcher.event_lag")?; + assert_eq!(lag_metric.get_f64_histogram().count(), 1); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_sqs_event_respects_sqs_active() -> Result<()> { + let mut config = Config::test_config()?; + if let Some(sqs_config) = &mut config.crates_io_events { + sqs_config.active = false; + } + let env = TestEnvironment::builder().config(config).build().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + + process_sqs_event(&env, env.config(), &metrics, &added_event_json(&KRATE, &V1)).await?; + + assert!(env.build_queue()?.queued_crates().await?.is_empty()); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_sqs_event_rejects_invalid_json() -> Result<()> { + let env = TestEnvironment::new().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + + let err = process_sqs_event(&env, env.config(), &metrics, "{not json").await; + + assert!(err.is_err()); + let err = format!("{:?}", err.unwrap_err()); + assert!( + err.contains("error parsing event from json"), + "unexpected error: {err}" + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_handle_message_body_acknowledges_success() -> Result<()> { + let config = Config::test_config()?; + let env = TestEnvironment::builder().config(config).build().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + + handle_message_body( + &env, + env.config(), + &metrics, + Some(&added_event_json(&KRATE, &V1)), + ) + .await; + let collected = env.collected_metrics(); + let processing_metric = + collected.get_metric("watcher", "docsrs.watcher.event_processing_time")?; + assert_eq!(processing_metric.get_f64_histogram().count(), 1); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_handle_message_body_records_failed_processing() -> Result<()> { + let env = TestEnvironment::new().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + + handle_message_body(&env, env.config(), &metrics, Some("{bad json")).await; + let collected = env.collected_metrics(); + let processing_metric = + collected.get_metric("watcher", "docsrs.watcher.event_processing_time")?; + let processing = processing_metric.get_f64_histogram(); + assert_eq!(processing.count(), 1); + assert!(processing.attributes().any(|attribute| { + attribute.key.as_str() == "result" && attribute.value.to_string() == "err" + })); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_handle_message_body_acknowledges_missing_body() -> Result<()> { + let env = TestEnvironment::new().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + + handle_message_body(&env, env.config(), &metrics, None).await; + assert!(env.build_queue()?.queued_crates().await?.is_empty()); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_messages_skips_errors_and_continues_batch() -> Result<()> { + let config = Config::test_config()?; + let env = TestEnvironment::builder().config(config).build().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + let client = FakeSqsActions::default(); + let messages = vec![ + Message::builder() + .body(added_event_json(&KRATE, &V1)) + .receipt_handle("success-1") + .build(), + Message::builder() + .body("{bad json") + .receipt_handle("failure") + .build(), + Message::builder() + .body(added_event_json(&KRATE, &V2)) + .receipt_handle("success-2") + .build(), + ]; + + process_messages(&client, "queue-url", &env, env.config(), &metrics, messages).await; + + assert_eq!( + *client.deleted.lock().unwrap(), + vec!["success-1", "failure", "success-2"] + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_process_messages_without_body_is_acknowledged() -> Result<()> { + let config = Config::test_config()?; + let env = TestEnvironment::builder().config(config).build().await?; + let metrics = WatcherMetrics::new(&env.context().meter_provider); + let client = FakeSqsActions::default(); + + process_messages( + &client, + "queue-url", + &env, + env.config(), + &metrics, + vec![Message::builder().receipt_handle("missing-body").build()], + ) + .await; + + assert_eq!( + *client.deleted.lock().unwrap(), + vec!["missing-body".to_string()] + ); + Ok(()) + } +} diff --git a/crates/lib/docs_rs_crates_io/src/events.rs b/crates/lib/docs_rs_crates_io/src/events.rs index f90484abc2..0169f02972 100644 --- a/crates/lib/docs_rs_crates_io/src/events.rs +++ b/crates/lib/docs_rs_crates_io/src/events.rs @@ -1,6 +1,35 @@ use chrono::{DateTime, Utc}; use std::fmt; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ChangeKind { + Added, + AddedAndYanked, + Unyanked, + Yanked, + CrateDeleted, + VersionDeleted, +} + +impl ChangeKind { + pub const fn as_str(self) -> &'static str { + match self { + Self::Added => "added", + Self::AddedAndYanked => "added_and_yanked", + Self::Unyanked => "unyanked", + Self::Yanked => "yanked", + Self::CrateDeleted => "crate_deleted", + Self::VersionDeleted => "version_deleted", + } + } +} + +impl fmt::Display for ChangeKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + /// A change that can happen to a crate on our index. #[derive(Clone, serde::Serialize, serde::Deserialize, Eq, PartialEq, Debug)] #[serde(tag = "type", content = "payload", rename_all = "snake_case")] @@ -57,17 +86,41 @@ impl IndexChangeV1 { _ => None, } } + + pub fn name(&self) -> &str { + match self { + IndexChangeV1::Added(crate_version) => &crate_version.name, + IndexChangeV1::Unyanked(crate_version) => &crate_version.name, + IndexChangeV1::Yanked(crate_version) => &crate_version.name, + IndexChangeV1::CrateDeleted { name } => name, + IndexChangeV1::VersionDeleted(crate_version) => &crate_version.name, + } + } + + pub fn version(&self) -> Option<&str> { + match self { + IndexChangeV1::Added(crate_version) => Some(&crate_version.version), + IndexChangeV1::Unyanked(crate_version) => Some(&crate_version.version), + IndexChangeV1::Yanked(crate_version) => Some(&crate_version.version), + IndexChangeV1::CrateDeleted { .. } => None, + IndexChangeV1::VersionDeleted(crate_version) => Some(&crate_version.version), + } + } + + pub fn kind(&self) -> ChangeKind { + match *self { + IndexChangeV1::Added(_) => ChangeKind::Added, + IndexChangeV1::Yanked(_) => ChangeKind::Yanked, + IndexChangeV1::CrateDeleted { .. } => ChangeKind::CrateDeleted, + IndexChangeV1::VersionDeleted(_) => ChangeKind::VersionDeleted, + IndexChangeV1::Unyanked(_) => ChangeKind::Unyanked, + } + } } impl fmt::Display for IndexChangeV1 { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match *self { - IndexChangeV1::Added(_) => "added", - IndexChangeV1::Yanked(_) => "yanked", - IndexChangeV1::CrateDeleted { .. } => "crate deleted", - IndexChangeV1::VersionDeleted(_) => "version deleted", - IndexChangeV1::Unyanked(_) => "unyanked", - }) + self.kind().fmt(f) } } diff --git a/crates/lib/docs_rs_storage/Cargo.toml b/crates/lib/docs_rs_storage/Cargo.toml index 1980060474..ba7060c8bd 100644 --- a/crates/lib/docs_rs_storage/Cargo.toml +++ b/crates/lib/docs_rs_storage/Cargo.toml @@ -16,11 +16,7 @@ testing = [ anyhow = { workspace = true } async-compression = { version = "0.4.32", features = ["bzip2", "deflate", "gzip", "tokio", "zstd"] } async-stream = { workspace = true } -# The default `rustls` feature pulls in the legacy hyper 0.14 + rustls 0.21 -# stack via `aws-smithy-runtime/tls-rustls`, which includes the vulnerable -# `rustls-webpki` v0.101.x. Using only `default-https-client` avoids this by -# using the modern rustls 0.23 + hyper 1.x stack instead. -aws-config = { version = "1.0.0", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-config = { workspace = true } aws-sdk-s3 = { version = "1.3.0", default-features = false, features = ["default-https-client", "rt-tokio"] } aws-smithy-types-convert = { version = "0.61.0", features = ["convert-chrono"] } base64 = { workspace = true } diff --git a/docker-compose.yml b/docker-compose.yml index 87f605d7fc..6e695ba457 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ # - repo-stats updater # - cdn invalidator # - release-rebuild-enqueuer +# * `elasticmq` -> local SQS-compatible queue for watcher testing # # optional profile: `metrics`: # * `opentelemetry` -> a debug opentelemetry receiver @@ -120,6 +121,7 @@ x-registry-watcher: ®istry-watcher depends_on: - db - s3 + - elasticmq volumes: - "./ignored/docker-registry-watcher/prefix:/opt/docsrs/prefix" - crates-io-index:/opt/docsrs/crates.io-index @@ -132,6 +134,10 @@ x-registry-watcher: ®istry-watcher REGISTRY_INDEX_PATH: /opt/docsrs/crates.io-index # configure the rebuild-queuer, DOCSRS_MAX_QUEUED_REBUILDS: ${DOCSRS_MAX_QUEUED_REBUILDS:-10} + DOCSRS_SQS_QUEUE_URL: ${DOCSRS_SQS_QUEUE_URL:-http://elasticmq:9324/queue/docsrs-events} + DOCSRS_SQS_QUEUE_REGION: ${DOCSRS_SQS_QUEUE_REGION:-elasticmq} + DOCSRS_SQS_ENDPOINT_URL: ${DOCSRS_SQS_ENDPOINT_URL:-http://elasticmq:9324} + DOCSRS_SQS_ACTIVE: ${DOCSRS_SQS_ACTIVE:-false} env_file: - .docker.env @@ -169,6 +175,17 @@ services: # watcher-CLI should not be run as background daemon, just manually - manual + elasticmq: + image: softwaremill/elasticmq + ports: + - "127.0.0.1:9324:9324" + volumes: + - "./dockerfiles/elasticmq.conf:/opt/elasticmq.conf:ro" + command: ["-Dconfig.file=/opt/elasticmq.conf"] + healthcheck: + <<: *healthcheck-interval + test: curl --silent --fail http://localhost:9324/health + builder-a: <<: *builder volumes: diff --git a/dockerfiles/elasticmq.conf b/dockerfiles/elasticmq.conf new file mode 100644 index 0000000000..fb77fac15d --- /dev/null +++ b/dockerfiles/elasticmq.conf @@ -0,0 +1,18 @@ +include classpath("application.conf") + +node-address { + protocol = http + host = "*" + port = 9324 + context-path = "" +} + +rest-sqs { + enabled = true + bind-port = 9324 + bind-hostname = "0.0.0.0" +} + +queues { + docsrs-events { } +} diff --git a/justfiles/utils.just b/justfiles/utils.just index db9458e2cc..65ac996e0c 100644 --- a/justfiles/utils.just +++ b/justfiles/utils.just @@ -2,11 +2,18 @@ _ensure_db_and_s3_are_running: _touch-docker-env # dependencies in the docker-cli file are ignored # here. Instead we explicitly start any dependent services first. - docker compose up -d db s3 --wait + docker compose up -d db s3 elasticmq --wait _touch-docker-env: touch .docker.env +send-sqs-payload: + aws sqs send-message \ + --endpoint-url $DOCSRS_SQS_ENDPOINT_URL \ + --region elasticmq \ + --queue-url $DOCSRS_SQS_QUEUE_URL \ + --message-body '{"id":"evt_1","occurred_at":"2026-07-02T12:00:00Z","type":"added","payload":{"name":"demo-crate","vers":"1.2.3"}}' + # helper recipe to ensure a CLI tool is installed. # * Accepts multiple names # * uses `cargo binstall` if it exists. From 5b881e5b8961f9d13f5f9838adba3883e8904f69 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:06:01 +0200 Subject: [PATCH 03/13] kk --- .../bin/docs_rs_watcher/src/index_watcher.rs | 169 +++++++++--------- 1 file changed, 81 insertions(+), 88 deletions(-) diff --git a/crates/bin/docs_rs_watcher/src/index_watcher.rs b/crates/bin/docs_rs_watcher/src/index_watcher.rs index eba5b926bf..8488fc2a7d 100644 --- a/crates/bin/docs_rs_watcher/src/index_watcher.rs +++ b/crates/bin/docs_rs_watcher/src/index_watcher.rs @@ -16,49 +16,12 @@ use docs_rs_database::{ use docs_rs_fastly::{Cdn, CdnBehaviour as _}; use docs_rs_types::{CrateId, KrateName, Version}; use std::time::Instant; -use tracing::{debug, error, info, instrument, warn}; - -trait ChangeExt { - fn name(&self) -> &str; - fn version(&self) -> Option<&str>; - fn kind(&self) -> ChangeKind; - fn first_crate_version(&self) -> &crates_index_diff::CrateVersion; -} - -impl ChangeExt for Change { - fn first_crate_version(&self) -> &crates_index_diff::CrateVersion { - self.versions().first().expect("always exists") - } - - fn name(&self) -> &str { - self.first_crate_version().name.as_str() - } - - fn version(&self) -> Option<&str> { - if let Change::CrateDeleted { .. } = self { - None - } else { - Some(self.first_crate_version().version.as_str()) - } - } - - fn kind(&self) -> ChangeKind { - match *self { - Change::Added(_) => ChangeKind::Added, - Change::Yanked(_) => ChangeKind::Yanked, - Change::CrateDeleted { .. } => ChangeKind::CrateDeleted, - Change::VersionDeleted(_) => ChangeKind::VersionDeleted, - Change::Unyanked(_) => ChangeKind::Unyanked, - Change::AddedAndYanked(_) => ChangeKind::AddedAndYanked, - } - } -} +use tracing::{debug, error, info, warn}; #[derive(Debug)] pub(crate) struct CrateVersion { pub name: KrateName, pub version: Version, - pub yanked: bool, } #[cfg(test)] @@ -67,19 +30,28 @@ impl Default for CrateVersion { Self { name: docs_rs_types::testing::KRATE, version: docs_rs_types::testing::V1, - yanked: false, } } } -impl TryFrom for CrateVersion { +impl TryFrom<&crates_index_diff::CrateVersion> for CrateVersion { + type Error = anyhow::Error; + + fn try_from(value: &crates_index_diff::CrateVersion) -> Result { + Ok(Self { + name: value.name.parse()?, + version: value.version.parse()?, + }) + } +} + +impl TryFrom<&docs_rs_crates_io::events::CrateVersion> for CrateVersion { type Error = anyhow::Error; - fn try_from(value: crates_index_diff::CrateVersion) -> Result { + fn try_from(value: &docs_rs_crates_io::events::CrateVersion) -> Result { Ok(Self { name: value.name.parse()?, version: value.version.parse()?, - yanked: value.yanked, }) } } @@ -90,7 +62,6 @@ impl From for crates_index_diff::CrateVersion { Self { name: value.name.to_string().into(), version: value.version.to_string().into(), - yanked: value.yanked, ..Default::default() } } @@ -156,9 +127,7 @@ pub(crate) async fn get_new_crates( debug!(last_seen_reference=%last_seen_reference, new_reference=%new_reference, "queueing changes"); metrics.record_events_received(EventSource::Git, changes.len()); - // NOTE: `Box::pin` to type-erase this future, otherwise we'll run into `recursion_limit` - // errors. - let crates_added = Box::pin(process_changes(context, &changes, config, metrics)).await; + let crates_added = process_changes(context, &changes, config, metrics).await; if let Err(err) = context.build_queue()?.reevaluate_priorities().await { error!(?err, "error reevaluating queued release priorities"); @@ -182,20 +151,53 @@ async fn process_changes( for change in changes { let start = Instant::now(); - let crate_name = change.name(); - let crate_version = change.version(); - let change_type = change.kind(); - - // Start temporarily loging all changes, as preparation for the SQS event migration. + // temporarily log all changes, so we can compare them with the SQS changes we see. + // They share the same log-target, and most tracing fields. + let (change_type, crate_name, crate_version) = match change { + Change::Added(version) => ( + ChangeKind::Added, + version.name.as_str(), + version.version.as_str(), + ), + Change::AddedAndYanked(version) => ( + ChangeKind::AddedAndYanked, + version.name.as_str(), + version.version.as_str(), + ), + Change::Unyanked(version) => ( + ChangeKind::Unyanked, + version.name.as_str(), + version.version.as_str(), + ), + Change::Yanked(version) => ( + ChangeKind::Yanked, + version.name.as_str(), + version.version.as_str(), + ), + Change::CrateDeleted { name, .. } => (ChangeKind::CrateDeleted, name.as_str(), ""), + Change::VersionDeleted(version) => ( + ChangeKind::VersionDeleted, + version.name.as_str(), + version.version.as_str(), + ), + }; debug!( target: "docs_rs_watcher::index_event", source = %EventSource::Git, - %change_type, + change_type = %change_type, crate_name, crate_version, "crates.io index event" ); + if config.crates_io_events_active() { + // just to be safe. + // Generally we don't even start the git-index-watcher when + // SQS is active. + // Will be removed with the git index watcher code when SQS is stable. + continue; + } + let success = match process_change(context, change, config).await { Ok(added) => { metrics.record_change_applied(EventSource::Git, change_type); @@ -220,27 +222,28 @@ async fn process_changes( } /// Process a crate change, returning whether the change was a crate addition or not. -#[instrument(skip_all, fields(name, version))] -async fn process_change(context: &Context, change: &Change, config: &Config) -> Result { - // 1: use the `CrateVersion` from `crates-index-diff`. - let crate_version = change.first_crate_version(); - - // record name & version on the tracing span for performance instrumentation. - tracing::Span::current() - .record("name", crate_version.name.as_str()) - .record("version", crate_version.version.as_str()); - - // 2: now, convert to our own internal `CrateVersion.` - let crate_version: CrateVersion = crate_version.clone().try_into()?; +pub(crate) async fn process_change( + context: &Context, + change: &Change, + config: &Config, +) -> Result { + let crate_version: CrateVersion = change + .versions() + .first() + .expect("always exists") + .try_into()?; match change { Change::Added(_release) => process_version_added(context, &crate_version).await?, Change::AddedAndYanked(_release) => { process_version_added(context, &crate_version).await?; - process_version_yank_status(context, &crate_version).await?; + process_version_yank_status(context, &crate_version, true).await?; } - Change::Unyanked(_release) | Change::Yanked(_release) => { - process_version_yank_status(context, &crate_version).await? + Change::Unyanked(_release) => { + process_version_yank_status(context, &crate_version, false).await? + } + Change::Yanked(_release) => { + process_version_yank_status(context, &crate_version, true).await? } Change::CrateDeleted { name, .. } => { let name: KrateName = name.parse()?; @@ -254,17 +257,19 @@ async fn process_change(context: &Context, change: &Change, config: &Config) -> } /// Processes crate changes, whether they got yanked or unyanked. -#[instrument(skip_all)] -async fn process_version_yank_status(context: &Context, release: &CrateVersion) -> Result<()> { +pub(crate) async fn process_version_yank_status( + context: &Context, + release: &CrateVersion, + yanked: bool, +) -> Result<()> { // FIXME: delay yanks of crates that have not yet finished building // https://github.com/rust-lang/docs.rs/issues/1934 - set_yanked(context, &release.name, &release.version, release.yanked).await?; + set_yanked(context, &release.name, &release.version, yanked).await?; queue_crate_invalidation(&release.name, context.cdn.as_deref()).await; Ok(()) } -#[instrument(skip_all)] -async fn process_version_added(context: &Context, release: &CrateVersion) -> Result<()> { +pub(crate) async fn process_version_added(context: &Context, release: &CrateVersion) -> Result<()> { let build_queue = context.build_queue()?; let priority = build_queue.find_priority(&release.name).await?; @@ -296,8 +301,7 @@ async fn process_version_added(context: &Context, release: &CrateVersion) -> Res Ok(()) } -#[instrument(skip_all)] -async fn process_version_deleted( +pub(crate) async fn process_version_deleted( context: &Context, config: &Config, release: &CrateVersion, @@ -331,8 +335,7 @@ async fn process_version_deleted( Ok(()) } -#[instrument(skip_all)] -async fn process_crate_deleted( +pub(crate) async fn process_crate_deleted( context: &Context, config: &Config, krate: &KrateName, @@ -351,7 +354,6 @@ async fn process_crate_deleted( context.build_queue()?.remove_crate_from_queue(krate).await } -#[instrument(skip_all, fields(name=%name, version=%version, yanked=%yanked))] pub(crate) async fn set_yanked( context: &Context, name: &KrateName, @@ -425,7 +427,6 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V1, - ..Default::default() }; process_version_added(&env, &krate).await?; @@ -436,7 +437,6 @@ mod tests { let krate = CrateVersion { name: "krate".parse()?, version: V2.to_string().parse()?, - ..Default::default() }; process_version_added(&env, &krate).await?; @@ -469,9 +469,8 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V1, - yanked: true, }; - process_version_yank_status(&env, &krate).await?; + process_version_yank_status(&env, &krate, true).await?; // And verify it's actually marked as yanked let row = sqlx::query!( @@ -488,9 +487,8 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V1, - yanked: false, }; - process_version_yank_status(&env, &krate).await?; + process_version_yank_status(&env, &krate, false).await?; let row = sqlx::query!( "SELECT yanked @@ -553,7 +551,6 @@ mod tests { let krate = CrateVersion { name: KRATE, version: V2, - ..Default::default() }; process_version_deleted(&env, env.config(), &krate).await?; @@ -583,22 +580,18 @@ mod tests { let krate1 = CrateVersion { name: KRATE, version: V1, - ..Default::default() }; let krate2 = CrateVersion { name: "krate2".parse()?, version: V1, - ..Default::default() }; let krate_already_present = CrateVersion { name: "krate_already_present".parse()?, version: V1, - ..Default::default() }; let non_existing_version = CrateVersion { name: "krate_already_present".parse()?, version: V2, - ..Default::default() }; let metrics = WatcherMetrics::new(&env.context().meter_provider); let added = process_changes( From 864872b4672f39ac892039f3bca52e33d6556642 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:09:34 +0200 Subject: [PATCH 04/13] kk --- .../bin/docs_rs_watcher/src/index_watcher.rs | 90 +++++++++++-------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/crates/bin/docs_rs_watcher/src/index_watcher.rs b/crates/bin/docs_rs_watcher/src/index_watcher.rs index 8488fc2a7d..8f7359f8ba 100644 --- a/crates/bin/docs_rs_watcher/src/index_watcher.rs +++ b/crates/bin/docs_rs_watcher/src/index_watcher.rs @@ -16,7 +16,43 @@ use docs_rs_database::{ use docs_rs_fastly::{Cdn, CdnBehaviour as _}; use docs_rs_types::{CrateId, KrateName, Version}; use std::time::Instant; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info, instrument, warn}; + +trait ChangeExt { + fn name(&self) -> &str; + fn version(&self) -> Option<&str>; + fn kind(&self) -> ChangeKind; + fn first_crate_version(&self) -> &crates_index_diff::CrateVersion; +} + +impl ChangeExt for Change { + fn first_crate_version(&self) -> &crates_index_diff::CrateVersion { + self.versions().first().expect("always exists") + } + + fn name(&self) -> &str { + self.first_crate_version().name.as_str() + } + + fn version(&self) -> Option<&str> { + if let Change::CrateDeleted { .. } = self { + None + } else { + Some(self.first_crate_version().version.as_str()) + } + } + + fn kind(&self) -> ChangeKind { + match *self { + Change::Added(_) => ChangeKind::Added, + Change::Yanked(_) => ChangeKind::Yanked, + Change::CrateDeleted { .. } => ChangeKind::CrateDeleted, + Change::VersionDeleted(_) => ChangeKind::VersionDeleted, + Change::Unyanked(_) => ChangeKind::Unyanked, + Change::AddedAndYanked(_) => ChangeKind::AddedAndYanked, + } + } +} #[derive(Debug)] pub(crate) struct CrateVersion { @@ -151,40 +187,14 @@ async fn process_changes( for change in changes { let start = Instant::now(); - // temporarily log all changes, so we can compare them with the SQS changes we see. - // They share the same log-target, and most tracing fields. - let (change_type, crate_name, crate_version) = match change { - Change::Added(version) => ( - ChangeKind::Added, - version.name.as_str(), - version.version.as_str(), - ), - Change::AddedAndYanked(version) => ( - ChangeKind::AddedAndYanked, - version.name.as_str(), - version.version.as_str(), - ), - Change::Unyanked(version) => ( - ChangeKind::Unyanked, - version.name.as_str(), - version.version.as_str(), - ), - Change::Yanked(version) => ( - ChangeKind::Yanked, - version.name.as_str(), - version.version.as_str(), - ), - Change::CrateDeleted { name, .. } => (ChangeKind::CrateDeleted, name.as_str(), ""), - Change::VersionDeleted(version) => ( - ChangeKind::VersionDeleted, - version.name.as_str(), - version.version.as_str(), - ), - }; + let crate_name = change.name(); + let crate_version = change.version(); + let change_type = change.kind(); + debug!( target: "docs_rs_watcher::index_event", source = %EventSource::Git, - change_type = %change_type, + %change_type, crate_name, crate_version, "crates.io index event" @@ -222,16 +232,22 @@ async fn process_changes( } /// Process a crate change, returning whether the change was a crate addition or not. +#[instrument(skip_all, fields(name, version))] pub(crate) async fn process_change( context: &Context, change: &Change, config: &Config, ) -> Result { - let crate_version: CrateVersion = change - .versions() - .first() - .expect("always exists") - .try_into()?; + // 1: use the `CrateVersion` from `crates-index-diff`. + let crate_version = change.first_crate_version(); + + // record name & version on the tracing span for performance instrumentation. + tracing::Span::current() + .record("name", crate_version.name.as_str()) + .record("version", crate_version.version.as_str()); + + // 2: now, convert to our own internal `CrateVersion.` + let crate_version: CrateVersion = crate_version.clone().try_into()?; match change { Change::Added(_release) => process_version_added(context, &crate_version).await?, From 19b67307790e7463d9939c7d17c9c25aecc46c8b Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:10:26 +0200 Subject: [PATCH 05/13] kk --- crates/bin/docs_rs_watcher/src/index_watcher.rs | 4 ++-- crates/bin/docs_rs_watcher/src/lib.rs | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/bin/docs_rs_watcher/src/index_watcher.rs b/crates/bin/docs_rs_watcher/src/index_watcher.rs index 8f7359f8ba..ef5f2fceee 100644 --- a/crates/bin/docs_rs_watcher/src/index_watcher.rs +++ b/crates/bin/docs_rs_watcher/src/index_watcher.rs @@ -70,10 +70,10 @@ impl Default for CrateVersion { } } -impl TryFrom<&crates_index_diff::CrateVersion> for CrateVersion { +impl TryFrom for CrateVersion { type Error = anyhow::Error; - fn try_from(value: &crates_index_diff::CrateVersion) -> Result { + fn try_from(value: crates_index_diff::CrateVersion) -> Result { Ok(Self { name: value.name.parse()?, version: value.version.parse()?, diff --git a/crates/bin/docs_rs_watcher/src/lib.rs b/crates/bin/docs_rs_watcher/src/lib.rs index f46b2ab045..f7ed5b8db0 100644 --- a/crates/bin/docs_rs_watcher/src/lib.rs +++ b/crates/bin/docs_rs_watcher/src/lib.rs @@ -89,9 +89,7 @@ async fn watch_registry( metrics: &WatcherMetrics, ) -> Result<()> { let mut last_gc = Instant::now(); - let queue = context.build_queue()?; - let metrics = WatcherMetrics::new(context.meter_provider()); loop { if queue.is_locked().await? { @@ -100,7 +98,7 @@ async fn watch_registry( debug!("Checking new crates"); let index = Index::from_config(config).await?; - match get_new_crates(context, &index, config, &metrics).await { + match get_new_crates(context, &index, config, metrics).await { Ok(n) => debug!("{} crates added to queue", n), Err(e) => { metrics.record_poll_error(EventSource::Git); From 6ce4c25fbc500a8989bf59570f87de427da24411 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:14:16 +0200 Subject: [PATCH 06/13] try --- crates/bin/cratesfyi/src/main.rs | 2 -- crates/bin/docs_rs_watcher/src/main.rs | 2 -- crates/bin/docs_rs_web/src/lib.rs | 1 - 3 files changed, 5 deletions(-) diff --git a/crates/bin/cratesfyi/src/main.rs b/crates/bin/cratesfyi/src/main.rs index e6f65026f4..ccdb6c767c 100644 --- a/crates/bin/cratesfyi/src/main.rs +++ b/crates/bin/cratesfyi/src/main.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "256"] - use anyhow::Result; use clap::Parser; use cratesfyi::daemon::start_daemon; diff --git a/crates/bin/docs_rs_watcher/src/main.rs b/crates/bin/docs_rs_watcher/src/main.rs index e5a276a9bd..d4138a3758 100644 --- a/crates/bin/docs_rs_watcher/src/main.rs +++ b/crates/bin/docs_rs_watcher/src/main.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "256"] - use anyhow::{Context as _, Result}; use clap::{Parser, Subcommand}; use docs_rs_config::AppConfig as _; diff --git a/crates/bin/docs_rs_web/src/lib.rs b/crates/bin/docs_rs_web/src/lib.rs index edd28d7c30..41fd32eaa2 100644 --- a/crates/bin/docs_rs_web/src/lib.rs +++ b/crates/bin/docs_rs_web/src/lib.rs @@ -1,4 +1,3 @@ -#![recursion_limit = "256"] #![allow( // clippy::cognitive_complexity, // TODO: `AxumNope::Redirect(EscapedURI, CachePolicy)` is too big. From b0e2b39f43d0045b3bdad3cc735c6e65f07662eb Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:23:21 +0200 Subject: [PATCH 07/13] log error when timeout is too short --- crates/bin/docs_rs_watcher/src/subscriber.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/bin/docs_rs_watcher/src/subscriber.rs b/crates/bin/docs_rs_watcher/src/subscriber.rs index 15b044d75c..0ad57a980d 100644 --- a/crates/bin/docs_rs_watcher/src/subscriber.rs +++ b/crates/bin/docs_rs_watcher/src/subscriber.rs @@ -134,6 +134,8 @@ async fn process_messages( metrics: &WatcherMetrics, messages: Vec, ) { + let batch_start = Instant::now(); + for message in messages { handle_message_body(context, config, metrics, message.body.as_deref()).await; if let Some(receipt_handle) = message.receipt_handle.as_deref() @@ -141,6 +143,18 @@ async fn process_messages( { error!(?err, receipt_handle, "error deleting message from queue"); } + + if batch_start.elapsed() >= VISIBILITY_TIMEOUT { + // NOTE: When the message is still in the queue ( not deleted here) after + // the visibility-timeout is reached, SQS will redeliver it, assuming that + // something went wrong. + // So in these cases we'll get duplicate messages. + error!( + ?messages, + VISIBILITY_TIMEOUT, + "handling message batch took longer than the visibility timeout!" + ); + } } } From 351f3f4feb38395d60dbd59dc849f73b161c70d0 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:29:05 +0200 Subject: [PATCH 08/13] kk --- crates/bin/docs_rs_watcher/src/lib.rs | 2 -- crates/bin/docs_rs_watcher/src/subscriber.rs | 10 ++++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/bin/docs_rs_watcher/src/lib.rs b/crates/bin/docs_rs_watcher/src/lib.rs index f7ed5b8db0..337388d2b3 100644 --- a/crates/bin/docs_rs_watcher/src/lib.rs +++ b/crates/bin/docs_rs_watcher/src/lib.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "256"] - mod config; pub mod consistency; mod db; diff --git a/crates/bin/docs_rs_watcher/src/subscriber.rs b/crates/bin/docs_rs_watcher/src/subscriber.rs index 0ad57a980d..a18c2b78c9 100644 --- a/crates/bin/docs_rs_watcher/src/subscriber.rs +++ b/crates/bin/docs_rs_watcher/src/subscriber.rs @@ -135,8 +135,9 @@ async fn process_messages( messages: Vec, ) { let batch_start = Instant::now(); + let mut error_logged = false; - for message in messages { + for message in &messages { handle_message_body(context, config, metrics, message.body.as_deref()).await; if let Some(receipt_handle) = message.receipt_handle.as_deref() && let Err(err) = client.delete_message(queue_url, receipt_handle).await @@ -144,16 +145,17 @@ async fn process_messages( error!(?err, receipt_handle, "error deleting message from queue"); } - if batch_start.elapsed() >= VISIBILITY_TIMEOUT { + if !error_logged && batch_start.elapsed() >= VISIBILITY_TIMEOUT { // NOTE: When the message is still in the queue ( not deleted here) after // the visibility-timeout is reached, SQS will redeliver it, assuming that // something went wrong. // So in these cases we'll get duplicate messages. error!( - ?messages, - VISIBILITY_TIMEOUT, + messages = ?messages, + visibility_timetout = VISIBILITY_TIMEOUT.as_secs_f64(), "handling message batch took longer than the visibility timeout!" ); + error_logged = true; } } } From 2f1e13e8c145795d44873bf31115124bcf78ca65 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:32:11 +0200 Subject: [PATCH 09/13] try fix recurse --- crates/bin/docs_rs_watcher/src/subscriber.rs | 50 ++++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/crates/bin/docs_rs_watcher/src/subscriber.rs b/crates/bin/docs_rs_watcher/src/subscriber.rs index a18c2b78c9..c58149174b 100644 --- a/crates/bin/docs_rs_watcher/src/subscriber.rs +++ b/crates/bin/docs_rs_watcher/src/subscriber.rs @@ -14,7 +14,11 @@ use docs_rs_context::Context; use docs_rs_crates_io::events::{IndexChangeEventV1, IndexChangeV1}; use docs_rs_types::KrateName; use docs_rs_utils::retry_async; -use std::time::{Duration, Instant}; +use std::{ + future::Future, + pin::Pin, + time::{Duration, Instant}, +}; use tokio::time; use tracing::{debug, error, instrument, warn}; @@ -39,19 +43,29 @@ const DELAY_BETWEEN_PRIORITY_RECHECK: Duration = Duration::from_secs(60); /// Should be longer than the longest time our server takes to handle a message. const VISIBILITY_TIMEOUT: Duration = Duration::from_secs(600); -trait SqsActions { - async fn delete_message(&self, queue_url: &str, receipt_handle: &str) -> Result<()>; +trait SqsActions: Sync { + fn delete_message<'a>( + &'a self, + queue_url: &'a str, + receipt_handle: &'a str, + ) -> Pin> + Send + 'a>>; } impl SqsActions for Client { - async fn delete_message(&self, queue_url: &str, receipt_handle: &str) -> Result<()> { - self.delete_message() - .queue_url(queue_url) - .receipt_handle(receipt_handle) - .send() - .await - .context("error deleting SQS message")?; - Ok(()) + fn delete_message<'a>( + &'a self, + queue_url: &'a str, + receipt_handle: &'a str, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.delete_message() + .queue_url(queue_url) + .receipt_handle(receipt_handle) + .send() + .await + .context("error deleting SQS message")?; + Ok(()) + }) } } @@ -127,7 +141,7 @@ pub(crate) async fn run_sqs_subscriber( } async fn process_messages( - client: &impl SqsActions, + client: &dyn SqsActions, queue_url: &str, context: &Context, config: &Config, @@ -285,9 +299,15 @@ mod tests { } impl SqsActions for FakeSqsActions { - async fn delete_message(&self, _queue_url: &str, receipt_handle: &str) -> Result<()> { - self.deleted.lock().unwrap().push(receipt_handle.into()); - Ok(()) + fn delete_message<'a>( + &'a self, + _queue_url: &'a str, + receipt_handle: &'a str, + ) -> Pin> + Send + 'a>> { + Box::pin(async move { + self.deleted.lock().unwrap().push(receipt_handle.into()); + Ok(()) + }) } } From 12d73937cf3fba5345c63a6d6f96fed71f0d0bcc Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:33:31 +0200 Subject: [PATCH 10/13] kk --- Cargo.lock | 1 + crates/bin/docs_rs_watcher/Cargo.toml | 1 + crates/bin/docs_rs_watcher/src/subscriber.rs | 50 +++++++------------- 3 files changed, 19 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82f674a7f3..372915df99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2564,6 +2564,7 @@ name = "docs_rs_watcher" version = "0.6.0" dependencies = [ "anyhow", + "async-trait", "aws-config", "aws-sdk-sqs", "chrono", diff --git a/crates/bin/docs_rs_watcher/Cargo.toml b/crates/bin/docs_rs_watcher/Cargo.toml index 7600139f82..31da9b4f28 100644 --- a/crates/bin/docs_rs_watcher/Cargo.toml +++ b/crates/bin/docs_rs_watcher/Cargo.toml @@ -8,6 +8,7 @@ edition.workspace = true [dependencies] anyhow = { workspace = true } +async-trait = "0.1.89" aws-config = { workspace = true } aws-sdk-sqs = { version = "1.99.0", default-features = false, features = ["default-https-client", "rt-tokio"] } chrono = { workspace = true } diff --git a/crates/bin/docs_rs_watcher/src/subscriber.rs b/crates/bin/docs_rs_watcher/src/subscriber.rs index c58149174b..977d2e02c6 100644 --- a/crates/bin/docs_rs_watcher/src/subscriber.rs +++ b/crates/bin/docs_rs_watcher/src/subscriber.rs @@ -7,6 +7,7 @@ use crate::{ metrics::{EventSource, WatcherMetrics}, }; use anyhow::{Context as _, Result}; +use async_trait::async_trait; use aws_config::{BehaviorVersion, Region, retry::RetryConfig}; use aws_sdk_sqs::{Client, types::Message}; use chrono::Utc; @@ -14,11 +15,7 @@ use docs_rs_context::Context; use docs_rs_crates_io::events::{IndexChangeEventV1, IndexChangeV1}; use docs_rs_types::KrateName; use docs_rs_utils::retry_async; -use std::{ - future::Future, - pin::Pin, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use tokio::time; use tracing::{debug, error, instrument, warn}; @@ -43,29 +40,21 @@ const DELAY_BETWEEN_PRIORITY_RECHECK: Duration = Duration::from_secs(60); /// Should be longer than the longest time our server takes to handle a message. const VISIBILITY_TIMEOUT: Duration = Duration::from_secs(600); +#[async_trait] trait SqsActions: Sync { - fn delete_message<'a>( - &'a self, - queue_url: &'a str, - receipt_handle: &'a str, - ) -> Pin> + Send + 'a>>; + async fn delete_message(&self, queue_url: &str, receipt_handle: &str) -> Result<()>; } +#[async_trait] impl SqsActions for Client { - fn delete_message<'a>( - &'a self, - queue_url: &'a str, - receipt_handle: &'a str, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - self.delete_message() - .queue_url(queue_url) - .receipt_handle(receipt_handle) - .send() - .await - .context("error deleting SQS message")?; - Ok(()) - }) + async fn delete_message(&self, queue_url: &str, receipt_handle: &str) -> Result<()> { + self.delete_message() + .queue_url(queue_url) + .receipt_handle(receipt_handle) + .send() + .await + .context("error deleting SQS message")?; + Ok(()) } } @@ -298,16 +287,11 @@ mod tests { deleted: Mutex>, } + #[async_trait] impl SqsActions for FakeSqsActions { - fn delete_message<'a>( - &'a self, - _queue_url: &'a str, - receipt_handle: &'a str, - ) -> Pin> + Send + 'a>> { - Box::pin(async move { - self.deleted.lock().unwrap().push(receipt_handle.into()); - Ok(()) - }) + async fn delete_message(&self, _queue_url: &str, receipt_handle: &str) -> Result<()> { + self.deleted.lock().unwrap().push(receipt_handle.into()); + Ok(()) } } From e43461fae2f21d7c8853275dcb59b5e15410ad5f Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:39:26 +0200 Subject: [PATCH 11/13] kk --- .env.sample | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index b6974b702c..33f0ac110b 100644 --- a/.env.sample +++ b/.env.sample @@ -20,5 +20,11 @@ SENTRY_ENVIRONMENT=dev # https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds DOCSRS_TOOLCHAIN=nightly -# NOTE: when running services in docker-compose, you can override the settings in +# NOTE: when running services in docker-compose, you can override the settings in # `.docker.env`, you'll fine an example in `.docker.env.sample`. + +# optional overrides for local ElasticMQ testing +DOCSRS_SQS_QUEUE_URL=http://elasticmq:9324/queue/docsrs-events +DOCSRS_SQS_QUEUE_REGION=elasticmq +DOCSRS_SQS_ENDPOINT_URL=http://elasticmq:9324 +DOCSRS_SQS_ACTIVE=false From 1d112a94347b6fa5a4fbe4aa6b09e05e91473089 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:40:00 +0200 Subject: [PATCH 12/13] fix env --- .env.sample | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.sample b/.env.sample index 33f0ac110b..b67400de0c 100644 --- a/.env.sample +++ b/.env.sample @@ -24,7 +24,7 @@ DOCSRS_TOOLCHAIN=nightly # `.docker.env`, you'll fine an example in `.docker.env.sample`. # optional overrides for local ElasticMQ testing -DOCSRS_SQS_QUEUE_URL=http://elasticmq:9324/queue/docsrs-events -DOCSRS_SQS_QUEUE_REGION=elasticmq -DOCSRS_SQS_ENDPOINT_URL=http://elasticmq:9324 +DOCSRS_SQS_QUEUE_URL=http://localhost:9324/queue/docsrs-events +DOCSRS_SQS_QUEUE_REGION=localhost +DOCSRS_SQS_ENDPOINT_URL=http://localhost:9324 DOCSRS_SQS_ACTIVE=false From 163bb713a8fec9ad4d6f791152bc1cc9d1e8a384 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Thu, 27 Aug 2026 07:42:16 +0200 Subject: [PATCH 13/13] kk --- justfiles/utils.just | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/justfiles/utils.just b/justfiles/utils.just index 65ac996e0c..fa8663e82b 100644 --- a/justfiles/utils.just +++ b/justfiles/utils.just @@ -8,11 +8,24 @@ _touch-docker-env: touch .docker.env send-sqs-payload: + #!/usr/bin/env bash + set -euo pipefail + + payload='{ + "id": "evt_1", + "occurred_at": "2026-07-02T12:00:00Z", + "type": "added", + "payload": { + "name": "demo-crate", + "vers": "1.2.3" + } + }' + aws sqs send-message \ --endpoint-url $DOCSRS_SQS_ENDPOINT_URL \ --region elasticmq \ --queue-url $DOCSRS_SQS_QUEUE_URL \ - --message-body '{"id":"evt_1","occurred_at":"2026-07-02T12:00:00Z","type":"added","payload":{"name":"demo-crate","vers":"1.2.3"}}' + --message-body "$payload" # helper recipe to ensure a CLI tool is installed. # * Accepts multiple names