//! Streams subsystem. Owns the stream handle and all stream- //! specific types (configs, retention, consumer types). //! //! Entry points live on `Client`: //! - `client.create_stream(stream::StreamConfig { ... })` //! - `client.get_stream("name")` //! - `client.get_or_create_stream(cfg)` //! //! Returned `Stream` carries the per-stream operations: //! - `stream.publish(subject, payload)` //! - `stream.create_consumer(stream::ConsumerConfig { ... })` pub mod consumer; pub use consumer::{Consumer, ConsumerConfig, DeliverPolicy, Message, Messages}; use crate::client::Client; use crate::error::{Error, Result}; use crate::streams_server::{ self as pb, CreateConsumerRequest, DeleteConsumerRequest, GetConsumerInfoRequest, GetStreamInfoRequest, PublishRequest, }; use std::time::Duration; use tonic::Request; /// One row of per-(sourcing, source) status returned by /// `Stream::sources_status` and `Client::get_stream_sources`. #[derive(Debug, Clone)] pub struct SourceStatus { pub sourcing_stream: String, pub source_stream: String, pub last_sourced_seq: u64, pub pulled_total: u64, pub last_error: String, pub last_error_ts_ms: i64, } /// A reference to a stream on the server. Cheap to clone. #[derive(Clone)] pub struct Stream { pub(crate) client: Client, pub(crate) name: String, } impl Stream { pub(crate) fn new(client: Client, name: String) -> Self { Self { client, name } } /// The stream's name. pub fn name(&self) -> &str { &self.name } /// Publish a message into this stream. pub async fn publish( &self, subject: impl Into, payload: impl Into>, ) -> Result { self.publish_with_headers(subject, std::iter::empty::<(String, String)>(), payload) .await } /// Publish with explicit headers. pub async fn publish_with_headers( &self, subject: impl Into, headers: impl IntoIterator, payload: impl Into>, ) -> Result { let mut c = self.client.streams_client(); let req = PublishRequest { stream: self.name.clone(), subject: subject.into(), payload: payload.into(), headers: headers .into_iter() .map(|(k, v)| pb::MessageHeader { key: k, value: v }) .collect(), ts_ms: 0, expected_last_seq: None, }; let r = c.publish(Request::new(req)).await?.into_inner(); if !r.success { return Err(Error::server(r.result_code, r.message)); } Ok(PublishAck { sequence: r.seq }) } /// Slice 3: per-source tail status for this stream. Returns /// one entry per source feeding this stream. Empty when the /// stream has no sources OR the tail tasks are running on a /// different node (this RPC returns the local primary's view). pub async fn sources_status(&self) -> Result> { let mut c = self.client.streams_client(); let r = c .get_stream_info(Request::new(GetStreamInfoRequest { name: self.name.clone(), })) .await? .into_inner(); if !r.success { return Err(Error::server(r.result_code, r.message)); } Ok(r.sources_status .into_iter() .map(|s| SourceStatus { sourcing_stream: self.name.clone(), source_stream: s.source_stream, last_sourced_seq: s.last_sourced_seq, pulled_total: s.pulled_total, last_error: s.last_error, last_error_ts_ms: s.last_error_ts_ms, }) .collect()) } /// Create a new consumer on this stream. Errors if a consumer /// with the same `durable_name` already exists. pub async fn create_consumer(&self, config: ConsumerConfig) -> Result { let name = config .durable_name .clone() .ok_or_else(|| Error::Invalid("ConsumerConfig.durable_name is required".into()))?; let cfg_pb = config.into_pb(&name); let mut c = self.client.streams_client(); let r = c .create_consumer(Request::new(CreateConsumerRequest { stream: self.name.clone(), config: Some(cfg_pb), })) .await? .into_inner(); if !r.success { return Err(Error::server(r.result_code, r.message)); } Ok(Consumer::new(self.client.clone(), self.name.clone(), name)) } /// Idempotent variant — returns an existing consumer if one /// with the same `durable_name` already exists, otherwise /// creates it. pub async fn get_or_create_consumer(&self, config: ConsumerConfig) -> Result { let name = config .durable_name .clone() .ok_or_else(|| Error::Invalid("ConsumerConfig.durable_name is required".into()))?; match self.get_consumer(&name).await { Ok(c) => Ok(c), Err(Error::Server { code, .. }) if code == "no_such_consumer" => { self.create_consumer(config).await } Err(e) => Err(e), } } /// Return a handle to an existing consumer. pub async fn get_consumer(&self, name: impl Into) -> Result { let name: String = name.into(); let mut c = self.client.streams_client(); let r = c .get_consumer_info(Request::new(GetConsumerInfoRequest { stream: self.name.clone(), consumer: name.clone(), })) .await? .into_inner(); if !r.success { return Err(Error::server(r.result_code, r.message)); } Ok(Consumer::new(self.client.clone(), self.name.clone(), name)) } /// Delete a consumer by name. pub async fn delete_consumer(&self, name: impl Into) -> Result<()> { let mut c = self.client.streams_client(); let r = c .delete_consumer(Request::new(DeleteConsumerRequest { stream: self.name.clone(), consumer: name.into(), })) .await? .into_inner(); if !r.success { return Err(Error::server(r.result_code, r.message)); } Ok(()) } } /// Returned by `Stream::publish` — just the assigned stream /// sequence; new fields can be added without breaking callers. #[derive(Debug, Clone, Copy)] pub struct PublishAck { pub sequence: u64, } /// Retention policy. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum RetentionPolicy { #[default] Limits, WorkQueue, Interest, } /// Stream configuration. `Default` produces an unbounded Limits /// stream with no subject filter, suitable for /// `StreamConfig { name: ..., ..Default::default() }`. #[derive(Debug, Clone, Default)] pub struct StreamConfig { pub name: String, pub subjects: Vec, pub retention: RetentionPolicy, /// Maximum age before messages are pruned. `None` = unbounded. pub max_age: Option, /// Maximum number of messages. `None` = unbounded. pub max_messages: Option, /// Maximum total stored bytes. `None` = unbounded. pub max_bytes: Option, /// Max bytes per individual message. `None` = no cap. pub max_message_size: Option, /// Block size override. 0 = server default (100_000). pub block_size: u64, /// Reject publishes that would push over `max_bytes` / /// `max_messages` instead of silently dropping the oldest. pub strict_limits: bool, /// Memory-only stream. Survives node failover via replication /// but a full-cluster restart loses it. pub ephemeral: bool, /// Per-subject revision cap. `0` (default) = unbounded. /// When N > 0, after each publish, older messages at that /// subject beyond the N most recent are dropped. Mirrors /// NATS JetStream's `MaxMsgsPerSubject` (and backs KV's /// `max_revisions` knob). pub max_msgs_per_subject: u64, /// Cross-stream sources. Slice 1: at most one entry, name-only. /// See `crates/streams/specs/SOURCES_DESIGN.md`. pub sources: Vec, } /// Identifies a stream to tail from. #[derive(Debug, Clone, Default)] pub struct StreamSource { pub source_stream: String, /// Optional NATS pattern. Only source messages whose subject /// matches are pulled. Empty = pull all. pub filter_subject: String, /// First source-seq to pull. `0` (default) = pull from /// beginning of source (subject to `max_initial_backfill`). /// Mutually exclusive with `start_time_ms`. pub start_seq: u64, /// Slice 3: start position by wall-clock timestamp (ms since /// epoch). When set, the tail resolves to the first message /// with `ts_ms >= start_time_ms` and seeds there. Future-of- /// stream resolves to "seed at current end". Mutually exclusive /// with `start_seq`. pub start_time_ms: i64, /// Slice 2F: cap on the initial backfill window. `0` = no cap. /// When set, the tail seeds at `max(0, source.last_seq - /// max_initial_backfill)`. Ignored once durable state exists. pub max_initial_backfill: u64, /// Slice 3: NATS-style subject rewrite. When set, the source's /// subject is matched against `subject_transform.source_pattern` /// and the captured wildcards substituted into /// `subject_transform.destination` (1-indexed `{{wildcard(N)}}` /// placeholders). The sourcing stream sees the rewritten /// subject; the original is preserved on the /// `waymaker-source-subject` header. pub subject_transform: Option, /// Slice 3: what to do when source retention drops messages /// past our last_sourced_seq. Default = Halt. pub on_drop: OnDropPolicy, /// Slice 3: dead-letter stream name. When non-empty, tail /// errors (transform mismatch, halt-on-drop firing, etc.) are /// published as JSON records to this stream so operators can /// triage without scraping logs. pub dlq_stream: String, } #[derive(Debug, Clone, Default)] pub struct SubjectTransform { pub source_pattern: String, pub destination: String, } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub enum OnDropPolicy { /// Tail records a persistent error + stops advancing. #[default] Halt, /// Tail jumps watermark to source's first available seq. SkipToFirstAvailable, } impl StreamConfig { pub(crate) fn into_pb(self) -> pb::StreamConfigPb { let retention = match self.retention { RetentionPolicy::Limits => pb::Retention { policy: Some(pb::retention::Policy::Limits(pb::LimitsRetention { max_age_ms: self.max_age.map(|d| d.as_millis() as u64), max_msgs: self.max_messages, max_bytes: self.max_bytes, strict_limits: self.strict_limits, })), }, RetentionPolicy::WorkQueue => pb::Retention { policy: Some(pb::retention::Policy::WorkQueue(pb::WorkQueueRetention {})), }, RetentionPolicy::Interest => pb::Retention { policy: Some(pb::retention::Policy::Interest(pb::InterestRetention {})), }, }; pb::StreamConfigPb { name: self.name, subjects_filter: self.subjects, retention: Some(retention), block_size: self.block_size, max_msg_bytes: self.max_message_size.unwrap_or(0), ephemeral: self.ephemeral, max_msgs_per_subject: self.max_msgs_per_subject, sources: self .sources .into_iter() .map(|s| pb::StreamSourceConfigPb { source_stream: s.source_stream, filter_subject: s.filter_subject, start_seq: s.start_seq, start_time_ms: s.start_time_ms, subject_transform: s.subject_transform.map(|t| { pb::SubjectTransformPb { source_pattern: t.source_pattern, destination: t.destination, } }), max_initial_backfill: s.max_initial_backfill, on_drop: match s.on_drop { OnDropPolicy::Halt => 0, OnDropPolicy::SkipToFirstAvailable => 1, }, dlq_stream: s.dlq_stream, }) .collect(), } } } /// Partial-update payload for `Client::update_stream`. Any field /// left at `None` is not touched server-side. #[derive(Debug, Clone, Default)] pub struct StreamUpdate { pub max_age_ms: Option, pub max_msgs: Option, pub max_bytes: Option, pub max_msg_bytes: Option, pub strict_limits: Option, }