From 94cacfb39b42e7afa3b447ef0f186038c659b20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 29 Aug 2026 18:56:33 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(pypi):=20honor=20PEP=20792?= =?UTF-8?q?=20status?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move project status to the top-level JSON object required by PEP 792. Parse and validate it before files can stream so quarantine remains independent of member order. --- .../src/cache/page_stream/live.rs | 15 +- .../src/cache/page_stream/mod.rs | 2 +- crates/peryx-ecosystem-pypi/src/metadata.rs | 12 +- .../src/openapi/simple.rs | 3 +- .../peryx-ecosystem-pypi/src/simple/meta.rs | 43 +++- crates/peryx-ecosystem-pypi/src/simple/mod.rs | 4 + .../peryx-ecosystem-pypi/src/simple/parse.rs | 46 +++- .../src/stream/transformer.rs | 200 +++++++++++++----- .../tests/unit/cache_coverage/live.rs | 2 +- .../tests/unit/serving/web/command_tests.rs | 3 +- .../tests/unit/tests/conformance_tests.rs | 11 +- .../tests/unit/tests/http/download.rs | 2 +- .../tests/unit/tests/http/mirror.rs | 35 ++- .../tests/unit/tests/http/support/request.rs | 5 +- .../tests/unit/tests/metadata_tests.rs | 11 +- .../tests/unit/tests/serve/resolve.rs | 8 +- .../tests/unit/tests/serve/streaming.rs | 99 +++++---- .../tests/unit/tests/simple/meta_tests.rs | 4 +- .../tests/unit/tests/simple/parse_tests.rs | 111 +++++++++- ...sts__simple__parse_tests__detail_json.snap | 2 +- .../unit/tests/stream/transformer_tests.rs | 65 ++++-- .../tests/unit/tests/virtual_tests.rs | 7 +- 22 files changed, 506 insertions(+), 184 deletions(-) diff --git a/crates/peryx-ecosystem-pypi/src/cache/page_stream/live.rs b/crates/peryx-ecosystem-pypi/src/cache/page_stream/live.rs index 5dbfa3561..b583ba9f5 100644 --- a/crates/peryx-ecosystem-pypi/src/cache/page_stream/live.rs +++ b/crates/peryx-ecosystem-pypi/src/cache/page_stream/live.rs @@ -60,7 +60,7 @@ impl FreshJsonStream { let preflight = match preflight { JsonPreflight::Streaming { body, transformer, raw, .. - } if !transformer.project_status_known() => match buffer_whole_page(body, raw, buffered_context).await { + } if !transformer.headers_known() => match buffer_whole_page(body, raw, buffered_context).await { Ok((raw, served, summary)) => JsonPreflight::Complete { raw, served, summary }, Err(err) => { release_flight(&self.state, &self.key, self.guard); @@ -147,7 +147,7 @@ fn build_record( } } -/// Drain the rest of a page whose `files` preceded `meta`, then transform it whole with the project +/// Drain the rest of a page whose `files` preceded `project-status`, then transform it whole with the project /// status seeded so a quarantined project withholds its files regardless of key order. async fn buffer_whole_page( mut body: futures_util::stream::BoxStream<'static, Result>, @@ -155,7 +155,7 @@ async fn buffer_whole_page( context: crate::stream::PageContext, ) -> Result<(Vec, Vec, PageSummary), CacheError> { use futures_util::StreamExt as _; - // Bound the buffer as the transformer does: a `files`-before-`meta` page whose whole body would + // Bound the buffer as the transformer does: a `files`-before-status page whose whole body would // pass `MAX_PAGE_BYTES` must fail here, before `raw` grows unbounded, rather than after the second // transformer pass allocates a matching output vector from the oversized input. while let Some(chunk) = body.next().await { @@ -165,12 +165,9 @@ async fn buffer_whole_page( } raw.extend_from_slice(&chunk); } - let status = crate::parse_detail(&raw) - .map_err(CacheError::Simple)? - .meta - .project_status; let mut transformer = PageTransformer::new(context); - transformer.seed_project_status(status); + let parsed = crate::parse_detail(&raw).map_err(CacheError::Simple)?; + transformer.seed_project_status(parsed.meta.project_status, parsed.meta.project_status_reason); let mut served = Vec::with_capacity(raw.len()); transformer.push_into(&raw, &mut served).map_err(transform_error)?; let summary = transformer.finish().map_err(transform_error)?; @@ -217,7 +214,7 @@ async fn preflight_json_stream( transformer .push_into(&chunk[position..=position], &mut outgoing) .map_err(transform_error)?; - if transformer.meta_preflight_done() || raw.len() >= JSON_META_PREFLIGHT_BYTES { + if transformer.header_preflight_done() || raw.len() >= JSON_META_PREFLIGHT_BYTES { pending.push_back(Bytes::copy_from_slice(&outgoing)); body = prepend_chunk(body, chunk.slice(position + 1..)); return Ok(JsonPreflight::Streaming { diff --git a/crates/peryx-ecosystem-pypi/src/cache/page_stream/mod.rs b/crates/peryx-ecosystem-pypi/src/cache/page_stream/mod.rs index 9009cf72c..39596ca2c 100644 --- a/crates/peryx-ecosystem-pypi/src/cache/page_stream/mod.rs +++ b/crates/peryx-ecosystem-pypi/src/cache/page_stream/mod.rs @@ -424,7 +424,7 @@ fn transform_whole( let mut transformer = PageTransformer::new(context); // Seed the status so a quarantined page withholds its files whether `meta` precedes or follows // `files`; the whole-page pass otherwise learns the status only once it reaches `meta`. - transformer.seed_project_status(detail.meta.project_status); + transformer.seed_project_status(detail.meta.project_status, detail.meta.project_status_reason); let mut out = Vec::with_capacity(record.body.len()); transformer.push_into(&record.body, &mut out).map_err(transform_error)?; transformer.finish().map_err(transform_error)?; diff --git a/crates/peryx-ecosystem-pypi/src/metadata.rs b/crates/peryx-ecosystem-pypi/src/metadata.rs index 88208a776..35e564a11 100644 --- a/crates/peryx-ecosystem-pypi/src/metadata.rs +++ b/crates/peryx-ecosystem-pypi/src/metadata.rs @@ -462,19 +462,19 @@ pub fn ui_project_from_detail(value: &serde_json::Value) -> crate::view::Project } } -/// The publish status the page flags, drawn from the PEP 792 `project-status` marker in the detail -/// document's `meta`. Active, absent, and unrecognized markers carry `None`, so the page flags only a -/// state that departs from serving the project as usual. +/// The publish status the page flags, drawn from the PEP 792 `project-status` object. Active, absent, +/// and unrecognized markers carry `None`, so the page flags only a state that departs from serving +/// the project as usual. #[cfg(feature = "serving")] fn project_status(value: &serde_json::Value) -> Option> { - let meta = &value["meta"]; - let status = meta["project-status"] + let project_status = &value["project-status"]; + let status = project_status["status"] .as_str() .and_then(crate::ProjectStatus::from_marker) .filter(|status| *status != crate::ProjectStatus::Active)?; Some(Box::new(crate::view::ProjectStatusView { marker: status.marker().to_owned(), - reason: meta["project-status-reason"] + reason: project_status["reason"] .as_str() .filter(|reason| !reason.is_empty()) .map(str::to_owned), diff --git a/crates/peryx-ecosystem-pypi/src/openapi/simple.rs b/crates/peryx-ecosystem-pypi/src/openapi/simple.rs index 89f5f33a5..f2f79199e 100644 --- a/crates/peryx-ecosystem-pypi/src/openapi/simple.rs +++ b/crates/peryx-ecosystem-pypi/src/openapi/simple.rs @@ -44,7 +44,8 @@ pub(super) fn project_detail() -> OperationBuilder { json_response( "The project detail page", json!({ - "meta": {"api-version": "1.4", "project-status": "active"}, + "meta": {"api-version": "1.4"}, + "project-status": {"status": "active"}, "name": "peryxpkg", "versions": ["1.0"], "files": [{ diff --git a/crates/peryx-ecosystem-pypi/src/simple/meta.rs b/crates/peryx-ecosystem-pypi/src/simple/meta.rs index fe743cd97..bab4faf46 100644 --- a/crates/peryx-ecosystem-pypi/src/simple/meta.rs +++ b/crates/peryx-ecosystem-pypi/src/simple/meta.rs @@ -16,9 +16,9 @@ const PEP700_MINOR: u64 = 1; pub struct Meta { #[serde(rename = "api-version")] pub api_version: &'static str, - #[serde(rename = "project-status", skip_serializing_if = "Option::is_none")] + #[serde(skip)] pub project_status: Option, - #[serde(rename = "project-status-reason", skip_serializing_if = "Option::is_none")] + #[serde(skip)] pub project_status_reason: Option, } @@ -60,6 +60,35 @@ impl Meta { .and_then(ProjectStatus::from_marker) .unwrap_or(ProjectStatus::Active) } + + pub(super) fn project_status_object(&self) -> Option> { + (self.project_status.is_some() || self.project_status_reason.is_some()).then_some(ProjectStatusObject { + status: self.project_status.as_deref(), + reason: self.project_status_reason.as_deref(), + }) + } +} + +#[derive(Serialize)] +pub struct ProjectStatusObject<'a> { + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) status: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) reason: Option<&'a str>, +} + +#[derive(Deserialize)] +pub(super) struct IncomingProjectStatus { + #[serde(default)] + status: Option, + #[serde(default)] + reason: Option, +} + +impl IncomingProjectStatus { + pub(super) fn into_parts(self) -> (Option, Option) { + (self.status, self.reason) + } } #[derive(Default, Deserialize)] @@ -80,6 +109,14 @@ impl IncomingMeta { self.project_status_reason, ) } + + pub(super) fn into_detail_meta(self, project_status: Option) -> Result { + let (project_status, project_status_reason) = project_status.map_or( + (self.project_status, self.project_status_reason), + IncomingProjectStatus::into_parts, + ); + Meta::from_upstream(self.api_version.as_deref(), project_status, project_status_reason) + } } /// The version peryx advertises for an upstream that declared `version`: its own ceiling once the @@ -108,7 +145,7 @@ fn served_version(version: Option<&str>) -> Result<&'static str, SimpleError> { }) } -fn validate_project_status(status: &str) -> Result<(), SimpleError> { +pub(super) fn validate_project_status(status: &str) -> Result<(), SimpleError> { ProjectStatus::from_marker(status) .map(|_| ()) .ok_or_else(|| SimpleError::InvalidProjectStatus(status.to_owned())) diff --git a/crates/peryx-ecosystem-pypi/src/simple/mod.rs b/crates/peryx-ecosystem-pypi/src/simple/mod.rs index eec6816ac..de4f290f2 100644 --- a/crates/peryx-ecosystem-pypi/src/simple/mod.rs +++ b/crates/peryx-ecosystem-pypi/src/simple/mod.rs @@ -11,9 +11,13 @@ mod render; pub use error::SimpleError; pub use file::{CoreMetadata, File, Provenance, Yanked}; +#[cfg(feature = "serving")] +pub use meta::ProjectStatusObject; pub use meta::{API_VERSION, API_VERSION_BASE, Meta, ProjectStatus}; #[cfg(feature = "serving")] pub use parse::absolutize; +#[cfg(feature = "serving")] +pub use parse::parse_project_status; pub use parse::{ DetailSink, ParsedDetail, ProjectDetail, ProjectList, ProjectListEntry, StreamedDetail, parse_detail, parse_index, parse_meta, stream_detail_json, to_json, diff --git a/crates/peryx-ecosystem-pypi/src/simple/parse.rs b/crates/peryx-ecosystem-pypi/src/simple/parse.rs index 3075ca8ea..262617126 100644 --- a/crates/peryx-ecosystem-pypi/src/simple/parse.rs +++ b/crates/peryx-ecosystem-pypi/src/simple/parse.rs @@ -4,10 +4,11 @@ use std::fmt; use std::io::Read; use serde::de::{DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::ser::SerializeMap as _; use serde::{Deserialize, Serialize}; use url::Url; -use super::meta::IncomingMeta; +use super::meta::{IncomingMeta, IncomingProjectStatus}; use super::{File, Meta, SimpleError}; /// Resolve `url` in place against `base`, turning a relative, root-relative, or protocol-relative @@ -36,6 +37,8 @@ pub struct ParsedDetail { #[derive(Deserialize)] struct IncomingDetail { meta: IncomingMeta, + #[serde(rename = "project-status", default, deserialize_with = "deserialize_project_status")] + project_status: Option, name: String, #[serde(default)] versions: Vec, @@ -50,7 +53,7 @@ struct IncomingDetail { pub fn parse_detail(bytes: &[u8]) -> Result { let detail: IncomingDetail = serde_json::from_slice(bytes)?; Ok(ParsedDetail { - meta: detail.meta.into_meta()?, + meta: detail.meta.into_detail_meta(detail.project_status)?, name: detail.name, versions: detail.versions, files: detail.files, @@ -94,7 +97,7 @@ pub fn stream_detail_json( let header = DetailSeed { base, sink }.deserialize(&mut deserializer)?; deserializer.end()?; Ok(StreamedDetail { - meta: header.meta.into_meta()?, + meta: header.meta.into_detail_meta(header.project_status)?, name: header.name, versions: header.versions, }) @@ -102,6 +105,7 @@ pub fn stream_detail_json( struct IncomingStreamedDetail { meta: IncomingMeta, + project_status: Option, name: String, versions: Vec, } @@ -136,12 +140,14 @@ impl<'de, S: DetailSink> Visitor<'de> for DetailVisitor<'_, S> { fn visit_map>(self, mut map: A) -> Result { let mut meta = None; + let mut project_status = None; let mut name = None; let mut versions = Vec::new(); let mut files = false; while let Some(key) = map.next_key::()? { match key.as_str() { "meta" => meta = Some(map.next_value()?), + "project-status" => project_status = Some(map.next_value()?), "name" => name = Some(map.next_value()?), "versions" => versions = map.next_value()?, "files" => { @@ -161,6 +167,7 @@ impl<'de, S: DetailSink> Visitor<'de> for DetailVisitor<'_, S> { } Ok(IncomingStreamedDetail { meta: meta.ok_or_else(|| serde::de::Error::missing_field("meta"))?, + project_status, name: name.ok_or_else(|| serde::de::Error::missing_field("name"))?, versions, }) @@ -214,6 +221,22 @@ pub fn parse_meta(bytes: &[u8]) -> Result { meta.into_meta() } +#[cfg(feature = "serving")] +pub fn parse_project_status(bytes: &[u8]) -> Result<(Option, Option), SimpleError> { + let status: IncomingProjectStatus = serde_json::from_slice(bytes)?; + let (status, reason) = status.into_parts(); + if let Some(status) = status.as_deref() { + super::meta::validate_project_status(status)?; + } + Ok((status, reason)) +} + +fn deserialize_project_status<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + IncomingProjectStatus::deserialize(deserializer).map(Some) +} + #[derive(Deserialize)] struct IncomingProjectListEntry { name: String, @@ -245,7 +268,7 @@ pub fn parse_index(bytes: &[u8]) -> Result { } /// A project's detail response (`/simple//`). -#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ProjectDetail { pub meta: Meta, pub name: String, @@ -253,6 +276,21 @@ pub struct ProjectDetail { pub files: Vec, } +impl Serialize for ProjectDetail { + fn serialize(&self, serializer: S) -> Result { + let project_status = self.meta.project_status_object(); + let mut map = serializer.serialize_map(Some(4 + usize::from(project_status.is_some())))?; + map.serialize_entry("meta", &self.meta)?; + if let Some(project_status) = project_status { + map.serialize_entry("project-status", &project_status)?; + } + map.serialize_entry("name", &self.name)?; + map.serialize_entry("versions", &self.versions)?; + map.serialize_entry("files", &self.files)?; + map.end() + } +} + /// One entry in the root project list. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct ProjectListEntry { diff --git a/crates/peryx-ecosystem-pypi/src/stream/transformer.rs b/crates/peryx-ecosystem-pypi/src/stream/transformer.rs index c15e8a915..ed8350c3c 100644 --- a/crates/peryx-ecosystem-pypi/src/stream/transformer.rs +++ b/crates/peryx-ecosystem-pypi/src/stream/transformer.rs @@ -9,7 +9,7 @@ use serde::Serialize; use super::validator::JsonValidator; use super::{PageContext, PageSummary, Registration, TransformError, is_json_whitespace}; use crate::policy::{PypiPolicy, RemoteMetadataMode}; -use crate::simple::absolutize; +use crate::simple::{ProjectStatusObject, absolutize, parse_project_status}; use crate::{CoreMetadata, File, parse_meta}; /// The most raw bytes peryx will read from one upstream Simple page before refusing it. The biggest @@ -21,7 +21,7 @@ pub const MAX_PAGE_BYTES: usize = 64 * 1024 * 1024; /// per-element work: a page of many tiny file objects stays small yet still forces a parse, a policy /// check, and a registration each, so the element count is capped on its own. const MAX_PAGE_FILES: usize = 500_000; -const MAX_KEY_BYTES: usize = b"versions".len(); +const MAX_KEY_BYTES: usize = b"project-status".len(); /// The transformer's lexer state, kept across chunk boundaries. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -30,6 +30,7 @@ enum Mode { Passthrough, /// Capturing the top-level `meta` object so peryx can advertise its supported version. Meta, + ProjectStatus, /// Between `files[` and its matching `]`: elements are captured and rewritten one by one. Files, /// Between `versions[` and its matching `]`: the whole (small) array is buffered and merged. @@ -64,14 +65,20 @@ struct StringState { } #[derive(Clone, Copy, PartialEq, Eq)] -enum MetaState { +enum StatusState { Unseen, - FilesFirst, Seen, + Emitted, +} + +struct HeaderState { + meta_seen: bool, + status: StatusState, + files_precede_headers: bool, } struct DocumentState { - meta: MetaState, + headers: HeaderState, closed: bool, trailing: bool, emitted_in_array: bool, @@ -90,7 +97,6 @@ pub struct PageTransformer { key_decode: KeyDecode, /// The page's top-level `name`, captured in flight so persistence needs no re-parse. name: Vec, - /// The top-level `meta` object has been checked. project_status: Option, project_status_reason: Option, document: DocumentState, @@ -126,7 +132,11 @@ impl PageTransformer { key_decode: KeyDecode::Literal, name: Vec::new(), document: DocumentState { - meta: MetaState::Unseen, + headers: HeaderState { + meta_seen: false, + status: StatusState::Unseen, + files_precede_headers: false, + }, closed: false, trailing: false, emitted_in_array: false, @@ -142,35 +152,31 @@ impl PageTransformer { } } - /// Whether a bounded preflight has enough information to leave the streaming loop: either `meta` - /// was seen (status known, safe to stream) or `files` opened first (status not yet known, so the - /// caller buffers the whole page before emitting any file). + /// Whether preflight validated both headers or reached `files` before it could. #[must_use] - pub const fn meta_preflight_done(&self) -> bool { - matches!(self.document.meta, MetaState::FilesFirst | MetaState::Seen) + pub const fn header_preflight_done(&self) -> bool { + self.document.headers.files_precede_headers || self.headers_known() } - /// Whether streaming reached the `files` array before `meta`, which leaves the project status - /// unknown; the caller then buffers the whole page and transforms it with the status seeded, so - /// a quarantined project withholds its files regardless of key order. + /// Whether streaming reached `files` before validating `meta` and resolving project status. #[must_use] - pub const fn files_precede_meta(&self) -> bool { - matches!(self.document.meta, MetaState::FilesFirst) + pub const fn files_precede_headers(&self) -> bool { + self.document.headers.files_precede_headers } - /// Whether the streaming preflight resolved the project status. Only `meta` carries the status, - /// so until it is parsed the caller cannot know whether a project is quarantined and must not - /// emit any file: a preflight that hit its byte cap before either `meta` or `files` leaves the - /// status unknown just as `files`-before-`meta` does, and both take the buffer-whole-page path. + /// Whether preflight validated the API version and resolved project status. #[must_use] - pub const fn project_status_known(&self) -> bool { - matches!(self.document.meta, MetaState::Seen) + pub const fn headers_known(&self) -> bool { + self.document.headers.meta_seen + && matches!(self.document.headers.status, StatusState::Seen | StatusState::Emitted) } - /// Seed the project status before a whole-page pass so a quarantined page withholds its files - /// even when `files` precedes `meta` in the document. - pub fn seed_project_status(&mut self, status: Option) { + /// Seed a whole-page pass after parsing established the explicit or absent status. + pub fn seed_project_status(&mut self, status: Option, reason: Option) { self.project_status = status; + self.project_status_reason = reason; + self.document.headers.meta_seen = true; + self.document.headers.status = StatusState::Seen; } /// # Errors @@ -225,6 +231,7 @@ impl PageTransformer { Ok(()) } Mode::Meta => self.step_meta(byte, out), + Mode::ProjectStatus => self.step_project_status(byte, out), Mode::Files => self.step_files(byte, out), Mode::Versions => self.step_versions(byte, out), } @@ -232,24 +239,7 @@ impl PageTransformer { fn step_passthrough(&mut self, byte: u8, out: &mut Vec) { if self.string.active { - out.push(byte); - if self.string.capture == StringCapture::Key && (self.string.escaped || byte != b'"') { - self.decode_key_byte(byte); - } - if self.string.capture == StringCapture::Name { - self.name.push(byte); - } - if self.string.escaped { - self.string.escaped = false; - } else if byte == b'\\' { - self.string.escaped = true; - } else if byte == b'"' { - self.string.active = false; - if self.string.capture == StringCapture::Name { - self.name.pop(); - } - self.string.capture = StringCapture::None; - } + self.step_passthrough_string(byte, out); return; } // Anything but whitespace once the root has closed is trailing garbage, whatever its kind. @@ -279,12 +269,21 @@ impl PageTransformer { self.depth += 1; // `"files": [` or `"versions": [` at the top level switches modes; the bracket is // emitted (files) or captured (versions merges into one emission). - if byte == b'{' && self.depth == 2 && self.key() == b"meta" { - self.mode = Mode::Meta; - self.array_depth = self.depth; - self.capture.clear(); - self.capture.push(byte); - return; + if byte == b'{' && self.depth == 2 { + if self.key() == b"meta" { + self.mode = Mode::Meta; + self.array_depth = self.depth; + self.capture.clear(); + self.capture.push(byte); + return; + } + if self.key() == b"project-status" { + self.mode = Mode::ProjectStatus; + self.array_depth = self.depth; + self.capture.clear(); + self.capture.push(byte); + return; + } } if byte == b'[' && self.depth == 2 { if self.key() == b"files" { @@ -306,6 +305,9 @@ impl PageTransformer { out.push(byte); } b'}' | b']' => { + if byte == b'}' && self.depth == 1 { + self.emit_seeded_project_status(out); + } self.depth = self.depth.saturating_sub(1); if self.depth == 0 { self.document.closed = true; @@ -314,8 +316,8 @@ impl PageTransformer { } b':' if self.depth == 1 => { self.string.expect_name = self.key() == b"name"; - if self.document.meta == MetaState::Unseen && self.key() == b"files" { - self.document.meta = MetaState::FilesFirst; + if !self.headers_known() && self.key() == b"files" { + self.document.headers.files_precede_headers = true; } out.push(byte); } @@ -329,6 +331,27 @@ impl PageTransformer { } } + fn step_passthrough_string(&mut self, byte: u8, out: &mut Vec) { + out.push(byte); + if self.string.capture == StringCapture::Key && (self.string.escaped || byte != b'"') { + self.decode_key_byte(byte); + } + if self.string.capture == StringCapture::Name { + self.name.push(byte); + } + if self.string.escaped { + self.string.escaped = false; + } else if byte == b'\\' { + self.string.escaped = true; + } else if byte == b'"' { + self.string.active = false; + if self.string.capture == StringCapture::Name { + self.name.pop(); + } + self.string.capture = StringCapture::None; + } + } + /// Feed one raw key byte through the JSON string-escape grammar, appending decoded bytes to the /// key buffer so dispatch matches the member name's value, not its spelling. fn decode_key_byte(&mut self, byte: u8) { @@ -438,6 +461,45 @@ impl PageTransformer { Ok(()) } + fn step_project_status(&mut self, byte: u8, out: &mut Vec) -> Result<(), TransformError> { + if self.string.active { + self.capture.push(byte); + if self.string.escaped { + self.string.escaped = false; + } else if byte == b'\\' { + self.string.escaped = true; + } else if byte == b'"' { + self.string.active = false; + } + return Ok(()); + } + match byte { + b'"' => { + self.string.active = true; + self.capture.push(byte); + } + b'{' | b'[' => { + self.depth += 1; + self.capture.push(byte); + } + b'}' => { + self.depth = self.depth.saturating_sub(1); + self.capture.push(byte); + if self.depth == self.array_depth - 1 { + self.emit_project_status(out)?; + self.capture.clear(); + self.mode = Mode::Passthrough; + } + } + b']' => { + self.depth = self.depth.saturating_sub(1); + self.capture.push(byte); + } + _ => self.capture.push(byte), + } + Ok(()) + } + fn step_files(&mut self, byte: u8, out: &mut Vec) -> Result<(), TransformError> { if self.string.active { self.capture.push(byte); @@ -651,12 +713,40 @@ impl PageTransformer { fn emit_meta(&mut self, out: &mut Vec) -> Result<(), TransformError> { let meta = parse_meta(&self.capture)?; write_json(out, &meta); - self.project_status = meta.project_status; - self.project_status_reason = meta.project_status_reason; - self.document.meta = MetaState::Seen; + self.document.headers.meta_seen = true; Ok(()) } + fn emit_project_status(&mut self, out: &mut Vec) -> Result<(), TransformError> { + (self.project_status, self.project_status_reason) = parse_project_status(&self.capture)?; + write_json( + out, + &ProjectStatusObject { + status: self.project_status.as_deref(), + reason: self.project_status_reason.as_deref(), + }, + ); + self.document.headers.status = StatusState::Emitted; + Ok(()) + } + + fn emit_seeded_project_status(&mut self, out: &mut Vec) { + if self.document.headers.status == StatusState::Emitted + || (self.project_status.is_none() && self.project_status_reason.is_none()) + { + return; + } + out.extend_from_slice(b",\"project-status\":"); + write_json( + out, + &ProjectStatusObject { + status: self.project_status.as_deref(), + reason: self.project_status_reason.as_deref(), + }, + ); + self.document.headers.status = StatusState::Emitted; + } + fn emit_versions(&self, out: &mut Vec) -> Result<(), TransformError> { let upstream: Vec = serde_json::from_slice(&self.capture)?; let merged: BTreeSet<&str> = upstream diff --git a/crates/peryx-ecosystem-pypi/tests/unit/cache_coverage/live.rs b/crates/peryx-ecosystem-pypi/tests/unit/cache_coverage/live.rs index 751a09eb1..c43299644 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/cache_coverage/live.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/cache_coverage/live.rs @@ -34,7 +34,7 @@ async fn test_live_stream_survives_a_persistence_error() { crate::tests::install_global_subscriber(); let dir = tempfile::tempdir().unwrap(); let server = MockServer::start().await; - let page = r#"{"meta":{"api-version":"1.4"},"name":"flask","versions":["1.0"],"files":[]}"#; + let page = r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","versions":["1.0"],"files":[]}"#; mount_page(&server, page).await; let state = read_only_state(&dir, &server); let representation_key = state.representation_key("pypi", "flask", crate::cache::SIMPLE_JSON); diff --git a/crates/peryx-ecosystem-pypi/tests/unit/serving/web/command_tests.rs b/crates/peryx-ecosystem-pypi/tests/unit/serving/web/command_tests.rs index 4b99f573a..4d99d2ddb 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/serving/web/command_tests.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/serving/web/command_tests.rs @@ -94,9 +94,8 @@ fn cached_project(display: &str, versions: &[&str]) -> (tempfile::TempDir, Arc"#)); + assert!(html.contains(r#""#)); } #[tokio::test] async fn test_mirror_detail_html() { @@ -204,8 +223,10 @@ async fn test_mirror_detail_from_html_keeps_fields_but_drops_gpg_sig() { let detail: serde_json::Value = serde_json::from_str(&body).unwrap(); let file = &detail["files"][0]; assert_eq!(status, StatusCode::OK); - assert_eq!(detail["meta"]["project-status"], "archived"); - assert_eq!(detail["meta"]["project-status-reason"], "read only"); + assert_eq!( + detail["project-status"], + serde_json::json!({"status": "archived", "reason": "read only"}) + ); assert_eq!(file["core-metadata"]["sha256"], metadata.as_str()); assert_eq!(file["dist-info-metadata"]["sha256"], metadata.as_str()); // The URL is now peryx's route, which cannot serve the `.asc`, so the marker must be gone. diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/http/support/request.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/http/support/request.rs index f770e51db..7c735edb8 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/http/support/request.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/http/support/request.rs @@ -119,8 +119,9 @@ pub async fn mount_status_detail( file_url: &str, ) { let body = format!( - "{{\"meta\":{{\"api-version\":\"1.4\",\"project-status\":\"{status}\",\ - \"project-status-reason\":\"{reason}\"}},\"name\":\"{project}\",\"versions\":[\"1.0\"],\ + "{{\"meta\":{{\"api-version\":\"1.4\"}},\ + \"project-status\":{{\"status\":\"{status}\",\"reason\":\"{reason}\"}},\ + \"name\":\"{project}\",\"versions\":[\"1.0\"],\ \"files\":[{{\"filename\":\"{project}-1.0-py3-none-any.whl\",\"url\":\"{file_url}\",\ \"hashes\":{{\"sha256\":\"{digest}\"}}}}]}}" ); diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/metadata_tests.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/metadata_tests.rs index 2853c0200..0589cc0e8 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/metadata_tests.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/metadata_tests.rs @@ -215,14 +215,17 @@ fn test_ui_project_from_detail_carries_project_status( #[case] reason: Option<&str>, #[case] expected: Option<(&str, Option<&str>)>, ) { - let mut meta = serde_json::json!({"api-version": "1.4"}); + let mut project_status = serde_json::json!({}); if let Some(marker) = marker { - meta["project-status"] = marker.into(); + project_status["status"] = marker.into(); } if let Some(reason) = reason { - meta["project-status-reason"] = reason.into(); + project_status["reason"] = reason.into(); + } + let mut value = serde_json::json!({"name": "veloxdemo", "files": []}); + if marker.is_some() || reason.is_some() { + value["project-status"] = project_status; } - let value = serde_json::json!({"name": "veloxdemo", "meta": meta, "files": []}); assert_eq!( ui_project_from_detail(&value).status, expected.map(|(marker, reason)| Box::new(ProjectStatusView { diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/resolve.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/resolve.rs index 5bd92546a..55bcf28ab 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/resolve.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/resolve.rs @@ -376,8 +376,9 @@ async fn test_overlay_with_two_mirrors_serves_buffered() { let digest = Digest::of(b"wheel"); let file_url = format!("{}/files/flask.whl", server.uri()); let page = format!( - "{{\"meta\":{{\"api-version\":\"1.4\",\"project-status\":\"archived\",\ - \"project-status-reason\":\"read only\"}},\"name\":\"flask\",\"versions\":[\"1.0\"],\ + "{{\"meta\":{{\"api-version\":\"1.4\"}},\ + \"project-status\":{{\"status\":\"archived\",\"reason\":\"read only\"}},\ + \"name\":\"flask\",\"versions\":[\"1.0\"],\ \"files\":[{{\"filename\":\"flask-1.0-py3-none-any.whl\",\"url\":\"{file_url}\",\ \"hashes\":{{\"sha256\":\"{digest}\"}}}}]}}", digest = digest.as_str(), @@ -421,8 +422,7 @@ async fn test_overlay_with_two_mirrors_serves_buffered() { let (status, _, body) = get(&state, "/both/simple/flask/", Some("application/json")).await; assert_eq!(status, StatusCode::OK); assert!(body.contains(digest.as_str())); - assert!(body.contains(r#""project-status":"archived""#)); - assert!(body.contains(r#""project-status-reason":"read only""#)); + assert!(body.contains(r#""project-status":{"status":"archived","reason":"read only"}"#)); } #[tokio::test] async fn test_overlay_nesting_an_overlay_serves_buffered() { diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/streaming.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/streaming.rs index 5d752b918..87f6e5bd9 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/streaming.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/serve/streaming.rs @@ -4,7 +4,7 @@ use futures_util::TryStreamExt as _; use peryx_identity::IndexAcl; use rstest::rstest; -fn padded_files_before_meta_page(file_url: &str, digest: &str, len: usize) -> String { +fn padded_files_without_status_page(file_url: &str, digest: &str, len: usize) -> String { let head = format!( "{{\"name\":\"flask\",\"versions\":[\"1.0\"],\ \"files\":[{{\"filename\":\"flask-1.0-py3-none-any.whl\",\"url\":\"{file_url}\",\ @@ -15,11 +15,14 @@ fn padded_files_before_meta_page(file_url: &str, digest: &str, len: usize) -> St format!("{head}{pad}{tail}", pad = " ".repeat(pad)) } -fn files_before_meta_page(file_url: &str, digest: &str, meta: &str) -> String { +fn files_before_status_page(file_url: &str, digest: &str, status: Option<&str>) -> String { + let project_status = status.map_or_else(String::new, |status| { + format!(r#","project-status":{{"status":"{status}"}}"#) + }); format!( "{{\"name\":\"flask\",\"versions\":[\"1.0\"],\ \"files\":[{{\"filename\":\"flask-1.0-py3-none-any.whl\",\"url\":\"{file_url}\",\ - \"hashes\":{{\"sha256\":\"{digest}\"}}}}],\"meta\":{meta}}}" + \"hashes\":{{\"sha256\":\"{digest}\"}}}}],\"meta\":{{\"api-version\":\"1.4\"}}{project_status}}}" ) } @@ -38,7 +41,8 @@ fn versions_outrun_preflight_page(file_url: &str, digest: &str) -> String { "{{\"name\":\"flask\",\"versions\":[{versions}],\ \"files\":[{{\"filename\":\"flask-1.0-py3-none-any.whl\",\"url\":\"{file_url}\",\ \"hashes\":{{\"sha256\":\"{digest}\"}}}}],\ - \"meta\":{{\"api-version\":\"1.4\",\"project-status\":\"quarantined\"}}}}" + \"meta\":{{\"api-version\":\"1.4\"}},\ + \"project-status\":{{\"status\":\"quarantined\"}}}}" ) } @@ -76,12 +80,13 @@ async fn test_small_json_page_without_meta_completes_during_preflight() { assert!(h.state.serving.meta.get_index("pypi/flask").unwrap().is_some()); } #[tokio::test] -async fn test_json_meta_preflight_streams_remainder() { +async fn test_json_status_preflight_streams_remainder() { let h = harness().await; let digest = Digest::of(b"wheel"); let file_url = format!("{}/files/flask.whl", h.server.uri()); let page = format!( - "{{\"meta\":{{\"api-version\":\"1.4\"}},\"name\":\"flask\",\"versions\":[\"1.0\"],\ + "{{\"meta\":{{\"api-version\":\"1.4\"}},\"project-status\":{{}},\ + \"name\":\"flask\",\"versions\":[\"1.0\"],\ \"files\":[{{\"filename\":\"flask-1.0-py3-none-any.whl\",\"url\":\"{file_url}\",\ \"hashes\":{{\"sha256\":\"{digest}\"}}}}]}}", digest = digest.as_str(), @@ -99,7 +104,7 @@ async fn test_json_meta_preflight_streams_remainder() { .unwrap() .concat(); let expected = concat!( - r#"{"meta":{"api-version":"1.4"},"name":"flask","versions":["1.0"],"files":[{"filename":"flask-1.0-py3-none-any.whl","url":"/pypi/files/"#, + r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","versions":["1.0"],"files":[{"filename":"flask-1.0-py3-none-any.whl","url":"/pypi/files/"#, r#"DIGEST/flask-1.0-py3-none-any.whl","hashes":{"sha256":"DIGEST"},"yanked":false,"core-metadata":false}]}"#, ) .replace("DIGEST", digest.as_str()); @@ -110,11 +115,12 @@ async fn test_json_meta_preflight_streams_remainder() { async fn test_live_stream_records_the_routed_upstream() { let server = MockServer::start().await; let digest = Digest::of(b"wheel"); - mount_json_page( - &server, - &detail_json(digest.as_str(), "https://example.invalid/files/flask.whl"), - ) - .await; + let page = detail_json(digest.as_str(), "https://example.invalid/files/flask.whl").replacen( + ",\"name\":", + ",\"project-status\":{},\"name\":", + 1, + ); + mount_json_page(&server, &page).await; let client = UpstreamClient::new(&format!("{}/simple/", server.uri())).unwrap(); let router = UpstreamRouter::new(vec![NamedUpstream::new("mirror", client.clone())]).unwrap(); let dir = tempfile::tempdir().unwrap(); @@ -146,8 +152,11 @@ async fn test_live_stream_records_the_routed_upstream() { } #[tokio::test] -async fn test_json_meta_preflight_streams_without_remainder() { - let upstream = split_project_upstream(br#"{"meta":{"api-version":"1.4"}"#.to_vec(), br"}".to_vec()); +async fn test_json_status_preflight_streams_without_remainder() { + let upstream = split_project_upstream( + br#"{"meta":{"api-version":"1.4"},"project-status":{}"#.to_vec(), + br"}".to_vec(), + ); let dir = tempfile::tempdir().unwrap(); let state = custom_state(&dir, &upstream.upstream, |client| { vec![Index { @@ -175,7 +184,7 @@ async fn test_json_meta_preflight_streams_without_remainder() { body.extend_from_slice(&chunk); body }); - assert_eq!(body, br#"{"meta":{"api-version":"1.4"}}"#); + assert_eq!(body, br#"{"meta":{"api-version":"1.4"},"project-status":{}}"#); } #[tokio::test] async fn test_materialize_detail_fetches_and_reuses_cached_page() { @@ -210,7 +219,7 @@ async fn test_materialize_detail_returns_stream_errors() { let h = harness().await; mount_json_page( &h.server, - r#"{"meta":{"api-version":"1.4"},"name":"flask","files":[{"bad": }]}"#, + r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","files":[{"bad": }]}"#, ) .await; @@ -225,7 +234,7 @@ async fn test_live_stream_surfaces_malformed_file_objects() { let h = harness().await; mount_json_page( &h.server, - r#"{"meta":{"api-version":"1.4"},"name":"flask","files":[{"bad": }]}"#, + r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","files":[{"bad": }]}"#, ) .await; let outcome = cache::stream_detail(h.state.serving.clone(), 0, "flask".to_owned()) @@ -237,7 +246,11 @@ async fn test_live_stream_surfaces_malformed_file_objects() { #[tokio::test] async fn test_live_stream_surfaces_truncated_pages() { let h = harness().await; - mount_json_page(&h.server, r#"{"meta":{"api-version":"1.4"},"name":"flask","files":["#).await; + mount_json_page( + &h.server, + r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","files":["#, + ) + .await; let outcome = cache::stream_detail(h.state.serving.clone(), 0, "flask".to_owned()) .await .unwrap(); @@ -245,10 +258,12 @@ async fn test_live_stream_surfaces_truncated_pages() { assert!(items.last().is_some_and(Result::is_err)); } #[rstest] -#[case(r#"{"meta":{"api-version":"1.4"},"name":"flask","versions":["1.0"],"files":[]}trailing"#)] -#[case(r#"{"meta":{"api-version":"1.4"},"name":"flask","versions":["1.0"],"files":[],"unknown":,}"#)] -#[case("{\"meta\":{\"api-version\":\"1.4\"},\"name\":\"flask\"\u{000b},\"files\":[]}")] -#[case("{\"meta\":{\"api-version\":\"1.4\"},\"name\":\"flask\"\u{000c},\"files\":[]}")] +#[case(r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","versions":["1.0"],"files":[]}trailing"#)] +#[case( + r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","versions":["1.0"],"files":[],"unknown":,}"# +)] +#[case("{\"meta\":{\"api-version\":\"1.4\"},\"project-status\":{},\"name\":\"flask\"\u{000b},\"files\":[]}")] +#[case("{\"meta\":{\"api-version\":\"1.4\"},\"project-status\":{},\"name\":\"flask\"\u{000c},\"files\":[]}")] #[tokio::test] async fn test_live_stream_invalid_document_errors_and_never_persists(#[case] page: &str) { let h = harness().await; @@ -278,7 +293,7 @@ async fn test_live_stream_error_releases_the_inflight_entry() { let h = harness().await; mount_json_page( &h.server, - r#"{"meta":{"api-version":"1.4"},"name":"flask","files":[{"bad": }]}"#, + r#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","files":[{"bad": }]}"#, ) .await; let outcome = cache::stream_detail(h.state.serving.clone(), 0, "flask".to_owned()) @@ -295,7 +310,7 @@ async fn test_live_stream_error_releases_the_inflight_entry() { #[tokio::test] async fn test_client_disconnect_releases_the_inflight_entry() { let upstream = split_project_upstream( - br#"{"meta":{"api-version":"1.4"},"name":"flask","files":["#.to_vec(), + br#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"flask","files":["#.to_vec(), br"]}".to_vec(), ); let dir = tempfile::tempdir().unwrap(); @@ -329,7 +344,7 @@ async fn test_client_disconnect_releases_the_inflight_entry() { async fn test_live_stream_forwards_a_broken_upstream_transfer() { let server = response_server( b"HTTP/1.1 200 OK\r\ncontent-type: application/vnd.pypi.simple.v1+json\r\n\ - content-length: 500\r\n\r\n{\"meta\":{\"api-version\":\"1.4\"},\"name\":\"flask\",\"files\":[", + content-length: 500\r\n\r\n{\"meta\":{\"api-version\":\"1.4\"},\"project-status\":{},\"name\":\"flask\",\"files\":[", ); let dir = tempfile::tempdir().unwrap(); let state = custom_state(&dir, &server.upstream, |client| { @@ -349,7 +364,7 @@ async fn test_live_stream_forwards_a_broken_upstream_transfer() { assert!(items.last().is_some_and(Result::is_err)); } #[tokio::test] -async fn test_buffered_files_before_meta_surfaces_a_broken_transfer() { +async fn test_buffered_files_before_status_surfaces_a_broken_transfer() { let server = response_server( b"HTTP/1.1 200 OK\r\ncontent-type: application/vnd.pypi.simple.v1+json\r\n\ content-length: 500\r\n\r\n{\"name\":\"flask\",\"files\":[{\"filename\":\"a\",", @@ -375,12 +390,12 @@ async fn test_buffered_files_before_meta_surfaces_a_broken_transfer() { ); } #[tokio::test] -async fn test_live_stream_buffers_quarantined_files_before_meta() { +async fn test_live_stream_buffers_quarantined_files_before_status() { let h = harness().await; - let page = files_before_meta_page( + let page = files_before_status_page( &format!("{}/files/flask.whl", h.server.uri()), Digest::of(b"wheel").as_str(), - r#"{"api-version":"1.4","project-status":"quarantined"}"#, + Some("quarantined"), ); mount_json_page(&h.server, &page).await; @@ -396,10 +411,10 @@ async fn test_live_stream_buffers_quarantined_files_before_meta() { )); } #[tokio::test] -async fn test_live_stream_buffers_files_before_meta_exactly_at_the_byte_cap() { +async fn test_live_stream_buffers_files_without_status_exactly_at_the_byte_cap() { let h = harness().await; let digest = Digest::of(b"wheel"); - let page = padded_files_before_meta_page( + let page = padded_files_without_status_page( &format!("{}/files/flask.whl", h.server.uri()), digest.as_str(), MAX_PAGE_BYTES, @@ -418,9 +433,9 @@ async fn test_live_stream_buffers_files_before_meta_exactly_at_the_byte_cap() { assert!(h.state.serving.meta.get_index("pypi/flask").unwrap().is_some()); } #[tokio::test] -async fn test_live_stream_rejects_files_before_meta_past_the_byte_cap() { +async fn test_live_stream_rejects_files_without_status_past_the_byte_cap() { let h = harness().await; - let page = padded_files_before_meta_page( + let page = padded_files_without_status_page( &format!("{}/files/flask.whl", h.server.uri()), Digest::of(b"wheel").as_str(), MAX_PAGE_BYTES + 1024, @@ -444,7 +459,7 @@ async fn test_live_stream_withholds_quarantined_files_when_versions_outrun_the_p let page = versions_outrun_preflight_page(&format!("{}/files/flask.whl", h.server.uri()), digest.as_str()); mount_json_page(&h.server, &page).await; - // Preflight exhaustion must not leak quarantined files before `meta` arrives. + // Preflight exhaustion must not leak quarantined files before `project-status` arrives. let outcome = cache::stream_detail(h.state.serving.clone(), 0, "flask".to_owned()) .await .unwrap(); @@ -457,7 +472,7 @@ async fn test_live_stream_withholds_quarantined_files_when_versions_outrun_the_p )); } #[tokio::test] -async fn test_buffered_files_before_meta_surfaces_parse_errors() { +async fn test_buffered_files_before_status_surfaces_parse_errors() { let h = harness().await; // Buffered parse failures must release the shared flight. mount_json_page(&h.server, r#"{"name":"flask","files":[{"bad": }]}"#).await; @@ -470,14 +485,10 @@ async fn test_buffered_files_before_meta_surfaces_parse_errors() { ); } #[tokio::test] -async fn test_live_stream_buffers_downloadable_files_before_meta() { +async fn test_live_stream_buffers_downloadable_files_without_status() { let h = harness().await; let digest = Digest::of(b"wheel"); - let page = files_before_meta_page( - &format!("{}/files/flask.whl", h.server.uri()), - digest.as_str(), - r#"{"api-version":"1.4"}"#, - ); + let page = files_before_status_page(&format!("{}/files/flask.whl", h.server.uri()), digest.as_str(), None); mount_json_page(&h.server, &page).await; let outcome = cache::stream_detail(h.state.serving.clone(), 0, "flask".to_owned()) @@ -490,7 +501,7 @@ async fn test_live_stream_buffers_downloadable_files_before_meta() { )); } #[tokio::test] -async fn test_transform_whole_withholds_quarantined_files_before_meta() { +async fn test_transform_whole_withholds_quarantined_files_before_status() { let dir = tempfile::tempdir().unwrap(); let state = custom_state(&dir, "https://example.invalid/simple/", |client| { vec![Index { @@ -502,10 +513,10 @@ async fn test_transform_whole_withholds_quarantined_files_before_meta() { acl: IndexAcl::default(), }] }); - let page = files_before_meta_page( + let page = files_before_status_page( "https://example.invalid/files/flask.whl", Digest::of(b"wheel").as_str(), - r#"{"api-version":"1.4","project-status":"quarantined"}"#, + Some("quarantined"), ); state .serving diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/meta_tests.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/meta_tests.rs index e42a9cd32..d036600ad 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/meta_tests.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/meta_tests.rs @@ -3,7 +3,7 @@ use std::error::Error as _; use crate::{ProjectStatus, SimpleError, parse_index}; #[test] -fn test_parse_meta_reads_project_status() { +fn test_parse_meta_reads_legacy_project_status() { let meta = crate::parse_meta(br#"{"api-version":"1.4","project-status":"archived","project-status-reason":"read only"}"#) .unwrap(); @@ -28,7 +28,7 @@ fn test_parse_meta_advertises_ceiling_only_for_pep700_upstreams() { } #[test] -fn test_parse_meta_rejects_invalid_project_status() { +fn test_parse_meta_rejects_invalid_legacy_project_status() { let err = crate::parse_meta(br#"{"api-version":"1.4","project-status":"frozen"}"#).unwrap_err(); assert!(matches!(&err, SimpleError::InvalidProjectStatus(status) if status == "frozen")); assert_eq!(err.to_string(), "invalid upstream project status marker \"frozen\""); diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/parse_tests.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/parse_tests.rs index f79c02426..5b148a8ba 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/parse_tests.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/parse_tests.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use super::{sample_detail, sample_list, sha256}; use crate::{ - CoreMetadata, File, Meta, ProjectDetail, ProjectList, ProjectListEntry, Provenance, Yanked, parse_index, - render_legacy_json, to_json, + CoreMetadata, File, Meta, ProjectDetail, ProjectList, ProjectListEntry, ProjectStatus, Provenance, SimpleError, + Yanked, parse_index, render_legacy_json, to_json, }; #[test] @@ -103,17 +103,14 @@ fn test_parse_detail_reads_legacy_only_metadata_key() { } #[test] -fn test_parse_detail_reads_project_status_provenance_gpg_size_upload_time_and_versions() { - let json = r#"{"meta":{"api-version":"1.4","project-status":"archived", - "project-status-reason":"read only"},"name":"x","versions":["1.0"], +fn test_parse_detail_reads_provenance_gpg_size_upload_time_and_versions() { + let json = r#"{"meta":{"api-version":"1.4"},"name":"x","versions":["1.0"], "files":[{"filename":"x-1.whl","url":"u","hashes":{},"size":42, "upload-time":"2024-01-01T00:00:00Z","gpg-sig":false, "provenance":"https://example.test/x-1.whl.provenance"}]}"#; let parsed = crate::parse_detail(json.as_bytes()).unwrap(); assert_eq!( ( - parsed.meta.project_status.as_deref(), - parsed.meta.project_status_reason.as_deref(), parsed.versions.as_slice(), parsed.files[0].size, parsed.files[0].upload_time.as_deref(), @@ -121,8 +118,6 @@ fn test_parse_detail_reads_project_status_provenance_gpg_size_upload_time_and_ve &parsed.files[0].provenance, ), ( - Some("archived"), - Some("read only"), ["1.0".to_owned()].as_slice(), Some(42), Some("2024-01-01T00:00:00Z"), @@ -132,6 +127,75 @@ fn test_parse_detail_reads_project_status_provenance_gpg_size_upload_time_and_ve ); } +#[test] +fn test_parse_detail_reads_published_pep_792_example() { + let parsed = crate::parse_detail( + br#"{ + "meta": {"api-version": "1.4"}, + "project-status": {"status": "quarantined", "reason": "the project is haunted"}, + "alternate-locations": [], + "files": [], + "name": "sampleproject", + "versions": ["1.2.0", "1.3.0", "1.3.1", "2.0.0", "3.0.0", "4.0.0"] + }"#, + ) + .unwrap(); + assert_eq!( + ( + parsed.meta.status(), + parsed.meta.project_status_reason.as_deref(), + parsed.name.as_str(), + parsed.versions.as_slice(), + parsed.files.as_slice(), + ), + ( + ProjectStatus::Quarantined, + Some("the project is haunted"), + "sampleproject", + ["1.2.0", "1.3.0", "1.3.1", "2.0.0", "3.0.0", "4.0.0"] + .map(str::to_owned) + .as_slice(), + [].as_slice(), + ) + ); +} + +#[rstest::rstest] +#[case::active(Some("active"), ProjectStatus::Active)] +#[case::archived(Some("archived"), ProjectStatus::Archived)] +#[case::quarantined(Some("quarantined"), ProjectStatus::Quarantined)] +#[case::deprecated(Some("deprecated"), ProjectStatus::Deprecated)] +#[case::absent(None, ProjectStatus::Active)] +fn test_parse_detail_maps_project_status(#[case] marker: Option<&str>, #[case] expected: ProjectStatus) { + let mut page = serde_json::json!({"meta": {"api-version": "1.4"}, "name": "demo", "files": []}); + if let Some(marker) = marker { + page["project-status"] = serde_json::json!({"status": marker}); + } + assert_eq!( + crate::parse_detail(page.to_string().as_bytes()).unwrap().meta.status(), + expected + ); +} + +#[test] +fn test_parse_detail_rejects_unknown_project_status() { + let error = crate::parse_detail( + br#"{"meta":{"api-version":"1.4"},"project-status":{"status":"frozen"},"name":"demo","files":[]}"#, + ) + .unwrap_err(); + assert!(matches!(&error, SimpleError::InvalidProjectStatus(status) if status == "frozen")); +} + +#[test] +fn test_parse_detail_prefers_top_level_status_over_legacy_cache_shape() { + let parsed = crate::parse_detail( + br#"{"meta":{"api-version":"1.4","project-status":"archived"}, + "project-status":{"status":"deprecated"},"name":"demo","files":[]}"#, + ) + .unwrap(); + assert_eq!(parsed.meta.status(), ProjectStatus::Deprecated); +} + #[test] fn test_legacy_project_json_maps_simple_fields() { let detail = sample_detail(); @@ -448,6 +512,35 @@ fn test_stream_detail_json_collects_files_and_absolutizes_urls() { assert_eq!(sink.0[0].url, "https://pypi.org/files/flask-1.0.tar.gz"); } +#[rstest::rstest] +#[case::active(Some("active"), ProjectStatus::Active)] +#[case::archived(Some("archived"), ProjectStatus::Archived)] +#[case::quarantined(Some("quarantined"), ProjectStatus::Quarantined)] +#[case::deprecated(Some("deprecated"), ProjectStatus::Deprecated)] +#[case::absent(None, ProjectStatus::Active)] +fn test_stream_detail_json_maps_project_status(#[case] marker: Option<&str>, #[case] expected: ProjectStatus) { + let mut page = serde_json::json!({"meta": {"api-version": "1.4"}, "name": "demo", "files": []}); + if let Some(marker) = marker { + page["project-status"] = serde_json::json!({"status": marker, "reason": "upstream reason"}); + } + let mut sink = Collect::default(); + let detail = + crate::simple::stream_detail_json(std::io::Cursor::new(page.to_string()), &detail_base(), &mut sink).unwrap(); + assert_eq!( + (detail.meta.status(), detail.meta.project_status_reason.as_deref()), + (expected, marker.map(|_| "upstream reason")) + ); +} + +#[test] +fn test_stream_detail_json_rejects_unknown_project_status() { + let body = br#"{"meta":{"api-version":"1.4"},"project-status":{"status":"frozen"}, + "name":"demo","files":[]}"#; + let mut sink = Collect::default(); + let error = crate::simple::stream_detail_json(std::io::Cursor::new(body), &detail_base(), &mut sink).unwrap_err(); + assert!(matches!(&error, SimpleError::InvalidProjectStatus(status) if status == "frozen")); +} + #[test] fn test_stream_detail_json_accepts_empty_files() { let body = br#"{"meta":{"api-version":"1.0"},"name":"flask","files":[]}"#; diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/snapshots/peryx_ecosystem_pypi__tests__simple__parse_tests__detail_json.snap b/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/snapshots/peryx_ecosystem_pypi__tests__simple__parse_tests__detail_json.snap index fcd272997..d5d6310b7 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/snapshots/peryx_ecosystem_pypi__tests__simple__parse_tests__detail_json.snap +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/simple/snapshots/peryx_ecosystem_pypi__tests__simple__parse_tests__detail_json.snap @@ -2,4 +2,4 @@ source: crates/peryx-ecosystem-pypi/tests/unit/tests/simple/parse_tests.rs expression: to_json(&sample_detail()) --- -{"meta":{"api-version":"1.4","project-status":"active","project-status-reason":"available"},"name":"proj&<>","versions":["1.0","2.0"],"files":[{"filename":"proj&<>-2.0-py3-none-any.whl","url":"https://files.example/a?b=1&c=2","hashes":{"sha256":"aaaa"},"requires-python":">=3.8,<4","size":1234,"upload-time":"2024-03-24T00:00:00.000000Z","yanked":false,"core-metadata":{"sha256":"bbbb"},"dist-info-metadata":{"sha256":"bbbb"},"gpg-sig":true,"provenance":"https://files.example/a.provenance"},{"filename":"proj-1.5.tar.gz","url":"https://files.example/q\"uote","hashes":{},"yanked":"broken build","core-metadata":true,"dist-info-metadata":true,"gpg-sig":false},{"filename":"proj-1.0-py3-none-any.whl","url":"https://files.example/c.whl","hashes":{"sha256":"cccc"},"size":9,"yanked":true,"core-metadata":false,"provenance":null}]} +{"meta":{"api-version":"1.4"},"project-status":{"status":"active","reason":"available"},"name":"proj&<>","versions":["1.0","2.0"],"files":[{"filename":"proj&<>-2.0-py3-none-any.whl","url":"https://files.example/a?b=1&c=2","hashes":{"sha256":"aaaa"},"requires-python":">=3.8,<4","size":1234,"upload-time":"2024-03-24T00:00:00.000000Z","yanked":false,"core-metadata":{"sha256":"bbbb"},"dist-info-metadata":{"sha256":"bbbb"},"gpg-sig":true,"provenance":"https://files.example/a.provenance"},{"filename":"proj-1.5.tar.gz","url":"https://files.example/q\"uote","hashes":{},"yanked":"broken build","core-metadata":true,"dist-info-metadata":true,"gpg-sig":false},{"filename":"proj-1.0-py3-none-any.whl","url":"https://files.example/c.whl","hashes":{"sha256":"cccc"},"size":9,"yanked":true,"core-metadata":false,"provenance":null}]} diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/stream/transformer_tests.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/stream/transformer_tests.rs index 95ee21605..0bfc0ee0e 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/stream/transformer_tests.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/stream/transformer_tests.rs @@ -230,8 +230,9 @@ fn test_empty_reason_yank_override_yanks_without_reason() { #[test] fn test_quarantined_project_streams_without_files() { - let page = r#"{"meta":{"api-version":"1.4","project-status":"quarantined", - "project-status-reason":"malware"},"name":"demo","versions":["1.0"],"files":[ + let page = r#"{"meta":{"api-version":"1.4"}, + "project-status":{"status":"quarantined","reason":"malware"}, + "name":"demo","versions":["1.0"],"files":[ {"filename":"demo-1.0-py3-none-any.whl","url":"https://up/demo-1.0-py3-none-any.whl", "hashes":{"sha256":"aa11"}} ]}"#; @@ -243,37 +244,49 @@ fn test_quarantined_project_streams_without_files() { } #[test] -fn test_seeded_quarantine_withholds_files_when_meta_follows_files() { +fn test_seeded_legacy_quarantine_withholds_files_when_meta_follows_files() { let page = r#"{"name":"demo","versions":["1.0"],"files":[ {"filename":"demo-1.0-py3-none-any.whl","url":"https://up/demo-1.0-py3-none-any.whl", "hashes":{"sha256":"aa11"}}], "meta":{"api-version":"1.4","project-status":"quarantined"}}"#; let mut transformer = PageTransformer::new(plain_context()); - transformer.seed_project_status(Some("quarantined".to_owned())); + transformer.seed_project_status(Some("quarantined".to_owned()), Some("malware".to_owned())); let out = transformer.push(page.as_bytes()).unwrap(); let summary = transformer.finish().unwrap(); let detail = parse_detail(&out).unwrap(); - assert_eq!(detail.meta.status(), crate::ProjectStatus::Quarantined); + assert_eq!( + (detail.meta.status(), detail.meta.project_status_reason.as_deref()), + (crate::ProjectStatus::Quarantined, Some("malware")) + ); + let json: serde_json::Value = serde_json::from_slice(&out).unwrap(); + assert_eq!( + json["project-status"], + serde_json::json!({"status": "quarantined", "reason": "malware"}) + ); + assert_eq!(json["meta"], serde_json::json!({"api-version": "1.4"})); assert!(detail.files.is_empty()); assert!(summary.registrations.is_empty()); } -#[test] -fn test_files_before_meta_ends_preflight_for_buffering() { +#[rstest::rstest] +#[case::missing_status(br#"{"meta":{"api-version":"1.4"},"name":"demo","files":["#)] +#[case::missing_meta(br#"{"project-status":{},"name":"demo","files":["#)] +fn test_files_before_headers_ends_preflight_for_buffering(#[case] page: &[u8]) { let mut transformer = PageTransformer::new(plain_context()); - transformer.push(br#"{"name":"demo","files":["#).unwrap(); - assert!(transformer.meta_preflight_done()); - assert!(transformer.files_precede_meta()); + transformer.push(page).unwrap(); + assert!(transformer.header_preflight_done()); + assert!(transformer.files_precede_headers()); } #[test] -fn test_meta_before_files_keeps_streaming() { +fn test_status_before_files_keeps_streaming() { let mut transformer = PageTransformer::new(plain_context()); transformer - .push(br#"{"meta":{"api-version":"1.4"},"name":"demo","files":["#) + .push(br#"{"meta":{"api-version":"1.4"},"project-status":{},"name":"demo","files":["#) .unwrap(); - assert!(transformer.meta_preflight_done()); - assert!(!transformer.files_precede_meta()); + assert!(transformer.header_preflight_done()); + assert!(!transformer.files_precede_headers()); + assert!(transformer.headers_known()); } #[test] @@ -294,7 +307,8 @@ fn test_escapes_and_braces_inside_strings_survive() { fn test_escaped_files_key_withholds_quarantined_files() { // RFC 8259 lets `files` be spelled `files`; the decoded key must still reach quarantine // withholding, or an escaped upstream key would leak a quarantined project's files. - let page = r#"{"m\u0065ta":{"project-status":"quarantined"},"name":"demo","fi\u006ces":[ + let page = r#"{"m\u0065ta":{},"project-\u0073tatus":{"status":"quarantined"}, + "name":"demo","fi\u006ces":[ {"filename":"demo-1.0-py3-none-any.whl","url":"https://up/demo-1.0-py3-none-any.whl", "hashes":{"sha256":"aa11"}} ]}"#; @@ -471,8 +485,9 @@ fn test_nested_array_inside_file_object_is_captured() { #[test] fn test_preserves_simple_api_fields_during_streaming() { - let page = r#"{"meta":{"api-version":"1.4","project-status":"archived", - "project-status-reason":"read only"},"name":"demo","versions":["1.0"],"files":[ + let page = r#"{"meta":{"api-version":"1.4"}, + "project-status":{"status":"archived","reason":"read only"}, + "name":"demo","versions":["1.0"],"files":[ {"filename":"demo-1.0-py3-none-any.whl","url":"https://up/demo-1.0-py3-none-any.whl", "hashes":{"sha256":"aa11"},"size":10,"upload-time":"2024-01-01T00:00:00Z", "core-metadata":{"sha256":"bb22"},"dist-info-metadata":{"sha256":"bb22"}, @@ -552,10 +567,10 @@ fn test_streaming_drops_an_insecure_provenance_url() { } #[test] -fn test_meta_streaming_handles_escaped_and_nested_unknown_values() { - let page = r#"{"meta":{"api-version":"1.4","project-status":"archived", - "project-status-reason":"read \"only\"", - "extra":[{"ignored":"yes"}]},"name":"demo","files":[]}"#; +fn test_project_status_streaming_handles_escaped_and_unknown_values() { + let page = r#"{"meta":{"api-version":"1.4"}, + "project-status":{"status":"archived","reason":"read \"only\"","extra":[{"ignored":"yes"}]}, + "name":"demo","files":[]}"#; let (out, _) = transform(page, plain_context(), 4096); let detail = parse_detail(out.as_bytes()).unwrap(); assert_eq!( @@ -567,6 +582,14 @@ fn test_meta_streaming_handles_escaped_and_nested_unknown_values() { ); } +#[test] +fn test_streaming_rejects_unknown_project_status() { + let mut transformer = PageTransformer::new(plain_context()); + let result = transformer + .push(br#"{"meta":{"api-version":"1.4"},"project-status":{"status":"frozen"},"name":"demo","files":[]}"#); + assert!(matches!(result, Err(crate::stream::TransformError::Simple(_)))); +} + #[test] fn test_streaming_rejects_unsupported_major_api_version() { let mut transformer = PageTransformer::new(plain_context()); diff --git a/crates/peryx-ecosystem-pypi/tests/unit/tests/virtual_tests.rs b/crates/peryx-ecosystem-pypi/tests/unit/tests/virtual_tests.rs index c40b1ef12..7e751eb70 100644 --- a/crates/peryx-ecosystem-pypi/tests/unit/tests/virtual_tests.rs +++ b/crates/peryx-ecosystem-pypi/tests/unit/tests/virtual_tests.rs @@ -141,8 +141,9 @@ async fn test_buffered_virtual_shadows_upstream_when_cached_layer_is_listed_firs async fn mount_status(server: &MockServer, status: &str) { let body = format!( - "{{\"meta\":{{\"api-version\":\"1.1\",\"project-status\":\"{status}\",\ - \"project-status-reason\":\"upstream {status}\"}},\"name\":\"peryxpkg\",\"versions\":[\"1.0\"],\ + "{{\"meta\":{{\"api-version\":\"1.1\"}},\ + \"project-status\":{{\"status\":\"{status}\",\"reason\":\"upstream {status}\"}},\ + \"name\":\"peryxpkg\",\"versions\":[\"1.0\"],\ \"files\":[{{\"filename\":\"peryxpkg-1.0-py3-none-any.whl\",\ \"url\":\"https://upstream.invalid/peryxpkg-1.0-py3-none-any.whl\",\ \"hashes\":{{\"sha256\":\"{UPSTREAM_DIGEST}\"}}}}]}}" @@ -198,7 +199,7 @@ async fn assert_quarantine_dominates(layers: Vec) { assert_eq!(status, StatusCode::OK); let json: serde_json::Value = serde_json::from_str(&detail).unwrap(); assert_eq!( - json["meta"]["project-status"], "quarantined", + json["project-status"]["status"], "quarantined", "a quarantined member must dominate the merged status: {detail}" ); assert_eq!(