Rust Programming2026-03-03 01:01
How to do lazy scan and sink properly with the new Polars streaming engine in Rust?
rustfan
Author
I’m using Polars 0.53.0 in Rust. As far as I can tell, the way to do lazy scan and sink (no collect() / materializing the data) with the new streaming engine is as follows, but I get no output. I couldn’t find an example of code that actually does this with 0.53.0. Anyone know how to do it? Cargo.toml
[dependencies]
polars = {version = "0.53.0", features = ["lazy", "new_streaming", "polars-io", "csv"]}
main.rs
use polars::prelude::*;
fn process_data(intsv: &str, outtsv: &str) -> std::io::Result<()> {
let schema = Schema::from_iter(vec![
Field::new("idx".into(), DataType::Int64),
Field::new("f1".into(), DataType::Int64),
Field::new("f2".into(), DataType::Int64),
]);
let lf1: LazyFrame =
LazyCsvReader::new(PlRefPath::new(&*intsv))
.with_has_header(true)
.with_separator(b'\t')
.with_schema(Some(Arc::new(schema.clone())))
.finish().unwrap();
let outpath = PlRefPath::new(&*outtsv);
let mut seropts = SerializeOptions::default();
seropts.separator = b'\t';
let aseropts = Arc::new(seropts);
let mut csvwo = CsvWriterOptions::default();
csvwo.include_header = true;
csvwo.serialize_options = aseropts;
let frame = lf1
.with_new_streaming(true)
.filter(col("idx").eq(lit(1)))
//.collect()
.sink(
SinkDestination::File {
target: SinkTarget::Path(outpath)
},
FileWriteFormat::Csv(csvwo),
UnifiedSinkArgs::default())
.expect("problem writing CSV output");
//println!("{}", frame);
Ok(())
}
fn main() {
let input_tsv = "mydata.tsv";
let output_tsv = "mydata-filt.tsv";
std::process::exit(match process_data(input_tsv, output_tsv) {
Ok(_) => 0,
Err(err) => {
eprintln!("error: {err:?}");
1
}
});
}
mydata.tsv
”` idx f1 f2 1 4 5 2 6 7 3 7 8 1
50