Rustrustasyncstreamstokioproduction-ready
Async Streams with Tokio
Producing and consuming async data streams using async iterators, stream combinators, and bounded channel-based producers.
use tokio::sync::mpsc;
use tokio_stream::{Stream, StreamExt, wrappers::ReceiverStream};
use std::pin::Pin;
use std::time::Duration;
/// A paginated API response
#[derive(Debug, Clone)]
pub struct Page<T> {
pub items: Vec<T>,
pub next_cursor: Option<String>,
pub total: usize,
}
/// Stream paginated API results automatically
pub fn paginated_stream<T, F, Fut>(
fetch_page: F,
page_size: usize,
) -> impl Stream<Item = Result<T, Box<dyn std::error::Error + Send>>>
where
T: Send + 'static,
F: Fn(Option<String>, usize) -> Fut + Send + 'static,
Fut: std::future::Future<Output = Result<Page<T>, Box<dyn std::error::Error + Send>>> + Send,
{
let (tx, rx) = mpsc::channel(page_size);
tokio::spawn(async move {
let mut cursor: Option<String> = None;
loop {
match fetch_page(cursor.clone(), page_size).await {
Ok(page) => {
let next = page.next_cursor.clone();
for item in page.items {
if tx.send(Ok(item)).await.is_err() {
return; // Receiver dropped
}
}
cursor = next;
if cursor.is_none() {
break; // No more pages
}
}
Err(e) => {
let _ = tx.send(Err(e)).await;
break;
}
}
}
});
ReceiverStream::new(rx)
}
/// Chunk a stream into fixed-size batches
pub fn chunk_stream<T>(
mut input: Pin<Box<dyn Stream<Item = T> + Send>>,
chunk_size: usize,
) -> impl Stream<Item = Vec<T>>
where
T: Send + 'static,
{
let (tx, rx) = mpsc::channel(4);
tokio::spawn(async move {
let mut buffer = Vec::with_capacity(chunk_size);
while let Some(item) = input.next().await {
buffer.push(item);
if buffer.len() >= chunk_size {
if tx.send(std::mem::take(&mut buffer)).await.is_err() {
return;
}
buffer = Vec::with_capacity(chunk_size);
}
}
if !buffer.is_empty() {
let _ = tx.send(buffer).await;
}
});
ReceiverStream::new(rx)
}
/// Rate-limited stream consumer
pub async fn rate_limited_consume<T, F, Fut>(
mut stream: Pin<Box<dyn Stream<Item = T> + Send>>,
rate: usize, // items per second
handler: F,
) -> usize
where
F: Fn(T) -> Fut,
Fut: std::future::Future<Output = ()>,
{
let interval = Duration::from_secs(1) / rate as u32;
let mut processed = 0;
while let Some(item) = stream.next().await {
handler(item).await;
processed += 1;
tokio::time::sleep(interval).await;
}
processed
}
/// Stream that produces items with exponential backoff on errors
pub fn resilient_stream<T, F, Fut>(
producer: F,
max_retries: usize,
) -> impl Stream<Item = T>
where
T: Send + 'static,
F: Fn() -> Fut + Send + 'static + Clone,
Fut: std::future::Future<Output = Option<T>> + Send,
{
let (tx, rx) = mpsc::channel(32);
tokio::spawn(async move {
let mut consecutive_failures = 0;
loop {
match (producer.clone())().await {
Some(item) => {
consecutive_failures = 0;
if tx.send(item).await.is_err() {
break;
}
}
None => {
consecutive_failures += 1;
if consecutive_failures > max_retries {
break;
}
let backoff = Duration::from_millis(100 * 2u64.pow(consecutive_failures as u32));
tokio::time::sleep(backoff.min(Duration::from_secs(30))).await;
}
}
}
});
ReceiverStream::new(rx)
}title: "Async Streams with Tokio" description: "Producing and consuming async data streams using async iterators, stream combinators, and bounded channel-based producers." category: "Rust" language: "rust" tags: ["rust", "async", "streams", "tokio", "production-ready"] createdAt: "2025-03-15" updatedAt: "2025-04-01"
Overview
Async stream patterns for processing data that arrives over time or in pages. Streams provide lazy evaluation with backpressure, enabling memory-efficient processing of large datasets through composable operations like map, filter, and chunk.
Code
let stream = paginated_stream(fetch_page, 100);
tokio::pin!(stream);
while let Some(Ok(item)) = stream.next().await {
process(item).await;
}Notes
- Streams are lazy; items are only produced when the consumer requests them (backpressure)
mpscchannels provide the backpressure mechanism between producer and consumer taskstokio::pin!is required for stack-pinning streams before calling.next()StreamExtfrom tokio-stream provides combinator methods (filter, map, take, collect)- The
resilient_streampattern handles intermittent failures with exponential backoff