1use std::fs::{self, File, OpenOptions};
19use std::io::{self, BufWriter, Write};
20use std::path::{Path, PathBuf};
21use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
22use std::sync::{Arc, Mutex};
23
24use sha2::{Digest, Sha256};
25
26use crate::error::{DimaError, IoAction, IoTarget, Result};
27
28const BUFFER: usize = 1 << 20;
30const INPUT_BUFFER: usize = 1 << 16;
33const TEMP_NAME_CHARS: usize = 64;
36
37#[derive(Clone, Debug, PartialEq, Eq)]
39pub enum Target {
40 Stdout,
42 Path(PathBuf),
44}
45
46impl Target {
47 #[must_use]
49 pub fn from_arg(arg: &Path) -> Self {
50 if arg.as_os_str() == "-" {
51 Self::Stdout
52 } else {
53 Self::Path(arg.to_path_buf())
54 }
55 }
56
57 fn io_target(&self) -> IoTarget {
58 match self {
59 Self::Stdout => IoTarget::Stdout,
60 Self::Path(p) => IoTarget::Path(p.clone()),
61 }
62 }
63}
64
65#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
67pub enum OutputCompression {
68 None,
70 Gzip,
72 Zstd,
74}
75
76enum Raw {
77 Stdout(io::Stdout),
78 File(File),
79}
80
81impl Write for Raw {
82 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
83 match self {
84 Self::Stdout(s) => s.write(buf),
85 Self::File(f) => f.write(buf),
86 }
87 }
88
89 fn flush(&mut self) -> io::Result<()> {
90 match self {
91 Self::Stdout(s) => s.flush(),
92 Self::File(f) => f.flush(),
93 }
94 }
95}
96
97#[derive(Debug)]
99struct WriteCancelled;
100
101impl std::fmt::Display for WriteCancelled {
102 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103 f.write_str("cancelled")
104 }
105}
106
107impl std::error::Error for WriteCancelled {}
108
109struct Counted {
112 raw: Raw,
113 hasher: Sha256,
114 bytes: u64,
115 cancel: Option<Arc<AtomicBool>>,
116 discard: Arc<AtomicBool>,
117}
118
119impl Write for Counted {
120 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
121 if self.discard.load(Ordering::Relaxed) {
122 return Ok(buf.len());
123 }
124 if self
125 .cancel
126 .as_ref()
127 .is_some_and(|c| c.load(Ordering::Relaxed))
128 {
129 return Err(io::Error::other(WriteCancelled));
130 }
131 let n = self.raw.write(buf)?;
132 self.hasher.update(&buf[..n]);
133 self.bytes += n as u64;
134 Ok(n)
135 }
136
137 fn flush(&mut self) -> io::Result<()> {
138 if self.discard.load(Ordering::Relaxed) {
139 return Ok(());
140 }
141 self.raw.flush()
142 }
143}
144
145#[cfg(unix)]
148fn keep_permissions(dest: &Path, file: &File) {
149 use std::os::unix::fs::PermissionsExt;
150 if let Ok(meta) = fs::metadata(dest) {
151 if meta.is_file() {
152 let mode = meta.permissions().mode() & 0o777;
153 let _ = file.set_permissions(fs::Permissions::from_mode(mode));
154 }
155 }
156}
157
158#[cfg(not(unix))]
159fn keep_permissions(_dest: &Path, _file: &File) {}
160
161fn write_error(target: &Target, action: IoAction, e: io::Error) -> DimaError {
163 if e.get_ref().is_some_and(|i| i.is::<WriteCancelled>()) {
164 DimaError::Cancelled
165 } else {
166 DimaError::io(action, target.io_target(), e)
167 }
168}
169
170#[derive(Clone, Debug, PartialEq, Eq)]
172pub struct Written {
173 pub bytes: u64,
175 pub sha256: String,
177}
178
179enum Encoder {
180 Plain(BufWriter<Counted>),
181 Gzip(BufWriter<flate2::write::GzEncoder<BufWriter<Counted>>>),
182 Zstd(BufWriter<zstd::stream::write::Encoder<'static, BufWriter<Counted>>>),
183}
184
185impl Encoder {
186 fn finish(self) -> io::Result<Counted> {
187 let buffered = match self {
188 Self::Plain(w) => w,
189 Self::Gzip(w) => w
190 .into_inner()
191 .map_err(io::IntoInnerError::into_error)?
192 .finish()?,
193 Self::Zstd(w) => w
194 .into_inner()
195 .map_err(io::IntoInnerError::into_error)?
196 .finish()?,
197 };
198 buffered
199 .into_inner()
200 .map_err(io::IntoInnerError::into_error)
201 }
202}
203
204pub struct Sink {
206 target: Target,
207 atomic: Option<(PathBuf, PathBuf)>,
209 encoder: Option<Encoder>,
210 discard: Arc<AtomicBool>,
212}
213
214static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0);
215
216static LIVE_TEMPORARIES: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
218
219fn live() -> std::sync::MutexGuard<'static, Vec<PathBuf>> {
220 LIVE_TEMPORARIES
221 .lock()
222 .unwrap_or_else(std::sync::PoisonError::into_inner)
223}
224
225pub fn remove_temporaries() {
229 for p in live().drain(..) {
230 let _ = fs::remove_file(p);
231 }
232}
233
234#[derive(Debug, PartialEq, Eq)]
236enum Destination {
237 Atomic(PathBuf),
240 Direct { path: PathBuf, append: bool },
244}
245
246fn resolve_destination(path: &Path) -> Destination {
249 let mut current = path.to_path_buf();
250 for _ in 0..40 {
251 if crate::input::is_descriptor_alias(¤t) {
252 return Destination::Direct {
253 path: path.to_path_buf(),
254 append: true,
255 };
256 }
257 match fs::symlink_metadata(¤t) {
258 Ok(m) if m.file_type().is_symlink() => match fs::read_link(¤t) {
259 Ok(target) if target.is_absolute() => current = target,
260 Ok(target) => {
261 let dir = current.parent().map(Path::to_path_buf).unwrap_or_default();
262 current = dir.join(target);
263 }
264 Err(_) => break,
265 },
266 Ok(m) if !m.is_file() => {
267 return Destination::Direct {
268 path: current,
269 append: false,
270 };
271 }
272 _ => break,
273 }
274 }
275 Destination::Atomic(current)
276}
277
278#[must_use]
282pub fn destination_of(path: &Path) -> PathBuf {
283 match resolve_destination(path) {
284 Destination::Atomic(p) | Destination::Direct { path: p, .. } => p,
285 }
286}
287
288impl Sink {
289 pub fn create(target: &Target, compression: OutputCompression) -> Result<Self> {
294 Self::create_cancellable(target, compression, None)
295 }
296
297 pub fn create_cancellable(
303 target: &Target,
304 compression: OutputCompression,
305 cancel: Option<Arc<AtomicBool>>,
306 ) -> Result<Self> {
307 let io_err = |action, e| DimaError::io(action, target.io_target(), e);
308 let (raw, atomic) = match target {
309 Target::Stdout => (Raw::Stdout(io::stdout()), None),
310 Target::Path(path) => match resolve_destination(path) {
311 Destination::Direct { path, append } => {
312 let f = OpenOptions::new()
313 .write(true)
314 .append(append)
315 .open(&path)
316 .map_err(|e| io_err(IoAction::Open, e))?;
317 (Raw::File(f), None)
318 }
319 Destination::Atomic(dest) => {
320 let dir = match dest.parent() {
321 Some(d) if !d.as_os_str().is_empty() => d.to_path_buf(),
322 _ => PathBuf::from("."),
323 };
324 let name: String = dest
325 .file_name()
326 .map_or_else(|| "output".into(), |n| n.to_string_lossy().into_owned())
327 .chars()
328 .take(TEMP_NAME_CHARS)
329 .collect();
330 let (file, temp) = loop {
331 let n = TEMP_COUNTER.fetch_add(1, Ordering::Relaxed);
332 let temp = dir.join(format!(".{name}.dima-{}-{n}.tmp", std::process::id()));
333 match OpenOptions::new().write(true).create_new(true).open(&temp) {
334 Ok(f) => {
335 live().push(temp.clone());
336 keep_permissions(&dest, &f);
337 break (f, temp);
338 }
339 Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {}
340 Err(e) => return Err(io_err(IoAction::Create, e)),
341 }
342 };
343 (Raw::File(file), Some((dest, temp)))
344 }
345 },
346 };
347 let discard = Arc::new(AtomicBool::new(false));
348 let buffered = BufWriter::with_capacity(
349 BUFFER,
350 Counted {
351 raw,
352 hasher: Sha256::new(),
353 bytes: 0,
354 cancel,
355 discard: Arc::clone(&discard),
356 },
357 );
358 let encoder = match compression {
359 OutputCompression::None => Encoder::Plain(buffered),
360 OutputCompression::Gzip => Encoder::Gzip(BufWriter::with_capacity(
361 INPUT_BUFFER,
362 flate2::write::GzEncoder::new(buffered, flate2::Compression::default()),
363 )),
364 OutputCompression::Zstd => Encoder::Zstd(BufWriter::with_capacity(
365 INPUT_BUFFER,
366 zstd::stream::write::Encoder::new(buffered, 3)
367 .map_err(|e| io_err(IoAction::Create, e))?,
368 )),
369 };
370 Ok(Self {
371 target: target.clone(),
372 atomic,
373 encoder: Some(encoder),
374 discard,
375 })
376 }
377
378 #[must_use]
380 pub fn target(&self) -> &Target {
381 &self.target
382 }
383
384 #[must_use]
387 pub fn error(&self, e: io::Error) -> DimaError {
388 write_error(&self.target, IoAction::Write, e)
389 }
390
391 pub fn commit(mut self) -> Result<Written> {
396 let target = self.target.io_target();
397 let encoder = self.encoder.take().expect("committed once");
398 let counted = encoder
399 .finish()
400 .map_err(|e| write_error(&self.target, IoAction::Write, e))?;
401 let written = Written {
402 bytes: counted.bytes,
403 sha256: counted
404 .hasher
405 .finalize()
406 .iter()
407 .fold(String::with_capacity(64), |mut s, b| {
408 use std::fmt::Write as _;
409 let _ = write!(s, "{b:02x}");
410 s
411 }),
412 };
413 match counted.raw {
414 Raw::Stdout(mut s) => s
415 .flush()
416 .map(|()| written)
417 .map_err(|e| DimaError::io(IoAction::Write, target, e)),
418 Raw::File(f) => {
419 if let Some((dest, temp)) = self.atomic.take() {
422 let unregister = || live().retain(|p| *p != temp);
425 if let Err(e) = f.sync_all() {
426 drop(f);
427 let _ = fs::remove_file(&temp);
428 unregister();
429 return Err(DimaError::io(IoAction::Sync, target, e));
430 }
431 drop(f);
432 if let Err(e) = fs::rename(&temp, &dest) {
433 let _ = fs::remove_file(&temp);
434 unregister();
435 return Err(DimaError::io(IoAction::Replace, target, e));
436 }
437 unregister();
438 sync_parent(&dest);
439 }
440 tracing::debug!(target = %target, bytes = written.bytes, "output committed");
441 Ok(written)
442 }
443 }
444 }
445}
446
447fn sync_parent(path: &Path) {
449 #[cfg(unix)]
450 if let Some(dir) = path.parent() {
451 let dir = if dir.as_os_str().is_empty() {
452 Path::new(".")
453 } else {
454 dir
455 };
456 if let Ok(d) = File::open(dir) {
457 let _ = d.sync_all();
458 }
459 }
460 #[cfg(not(unix))]
461 let _ = path;
462}
463
464impl Write for Sink {
465 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
466 match self.encoder.as_mut().expect("not committed") {
467 Encoder::Plain(w) => w.write(buf),
468 Encoder::Gzip(w) => w.write(buf),
469 Encoder::Zstd(w) => w.write(buf),
470 }
471 }
472
473 fn flush(&mut self) -> io::Result<()> {
474 match self.encoder.as_mut().expect("not committed") {
475 Encoder::Plain(w) => w.flush(),
476 Encoder::Gzip(w) => w.flush(),
477 Encoder::Zstd(w) => w.flush(),
478 }
479 }
480}
481
482impl Drop for Sink {
483 fn drop(&mut self) {
484 if self.encoder.is_some() {
485 self.discard.store(true, Ordering::Relaxed);
488 }
489 drop(self.encoder.take());
491 if let Some((_, temp)) = self.atomic.take() {
492 live().retain(|p| *p != temp);
493 let _ = fs::remove_file(temp);
494 }
495 }
496}
497
498#[cfg(test)]
499mod tests {
500 use super::*;
501 use std::io::Read;
502
503 fn entries(dir: &Path) -> Vec<String> {
504 let mut v: Vec<String> = fs::read_dir(dir)
505 .unwrap()
506 .map(|e| e.unwrap().file_name().to_string_lossy().into_owned())
507 .collect();
508 v.sort();
509 v
510 }
511
512 #[test]
513 fn a_cancelled_sink_stops_and_leaves_nothing() {
514 let dir = tempfile::tempdir().unwrap();
515 let dest = dir.path().join("big.json");
516 let cancel = Arc::new(AtomicBool::new(false));
517 let mut sink = Sink::create_cancellable(
518 &Target::Path(dest.clone()),
519 OutputCompression::None,
520 Some(Arc::clone(&cancel)),
521 )
522 .unwrap();
523 sink.write_all(b"[1,").unwrap();
524 cancel.store(true, Ordering::Relaxed);
525 let e = (0..10_000)
527 .find_map(|i| sink.write_all(format!("{i},").repeat(100).as_bytes()).err())
528 .expect("a write fails once cancelled");
529 assert!(matches!(sink.error(e), DimaError::Cancelled));
530 drop(sink);
531 assert!(entries(dir.path()).is_empty(), "{:?}", entries(dir.path()));
532 }
533
534 #[test]
535 fn compressed_outputs_round_trip_from_small_writes() {
536 let dir = tempfile::tempdir().unwrap();
537 let text: Vec<u8> = (0..200_000u32)
538 .flat_map(|i| format!("{i},").into_bytes())
539 .collect();
540 for (name, compression) in [
541 ("a.gz", OutputCompression::Gzip),
542 ("a.zst", OutputCompression::Zstd),
543 ] {
544 let dest = dir.path().join(name);
545 let mut sink = Sink::create(&Target::Path(dest.clone()), compression).unwrap();
546 for chunk in text.chunks(3) {
548 sink.write_all(chunk).unwrap();
549 }
550 let written = sink.commit().unwrap();
551 let stored = fs::read(&dest).unwrap();
552 assert_eq!(written.bytes, stored.len() as u64);
553 let mut decoded = Vec::new();
554 match compression {
555 OutputCompression::Gzip => {
556 flate2::read::GzDecoder::new(&stored[..])
557 .read_to_end(&mut decoded)
558 .unwrap();
559 }
560 _ => decoded = zstd::decode_all(&stored[..]).unwrap(),
561 }
562 assert_eq!(decoded, text, "{name}");
563 }
564 }
565
566 #[test]
567 fn long_names_get_short_temporary_files() {
568 let dir = tempfile::tempdir().unwrap();
569 let dest = dir.path().join(format!("{}.json", "r".repeat(245)));
570 let mut sink = Sink::create(&Target::Path(dest.clone()), OutputCompression::None).unwrap();
571 sink.write_all(b"{}").unwrap();
572 sink.commit().unwrap();
573 assert_eq!(fs::read(&dest).unwrap(), b"{}");
574 }
575
576 #[cfg(unix)]
577 #[test]
578 fn destinations() {
579 let dir = tempfile::tempdir().unwrap();
580 let file = dir.path().join("data.json");
581 fs::write(&file, "x").unwrap();
582 let link = dir.path().join("link.json");
583 std::os::unix::fs::symlink("data.json", &link).unwrap();
584 assert_eq!(
586 resolve_destination(&link),
587 Destination::Atomic(file.clone())
588 );
589 let missing = dir.path().join("new.json");
590 assert_eq!(
591 resolve_destination(&missing),
592 Destination::Atomic(missing.clone())
593 );
594 for alias in ["/dev/stdout", "/dev/stderr", "/dev/fd/1", "/proc/self/fd/1"] {
596 assert_eq!(
597 resolve_destination(Path::new(alias)),
598 Destination::Direct {
599 path: PathBuf::from(alias),
600 append: true
601 },
602 "{alias}"
603 );
604 }
605 let to_stdout = dir.path().join("to_stdout");
606 std::os::unix::fs::symlink("/dev/stdout", &to_stdout).unwrap();
607 assert!(matches!(
608 resolve_destination(&to_stdout),
609 Destination::Direct { append: true, .. }
610 ));
611 assert_eq!(
613 resolve_destination(Path::new("/dev/null")),
614 Destination::Direct {
615 path: PathBuf::from("/dev/null"),
616 append: false
617 }
618 );
619 }
620
621 #[test]
622 fn commit_replaces_atomically() {
623 let dir = tempfile::tempdir().unwrap();
624 let p = dir.path().join("out.json");
625 fs::write(&p, "old").unwrap();
626 let mut s = Sink::create(&Target::Path(p.clone()), OutputCompression::None).unwrap();
627 s.write_all(b"new").unwrap();
628 assert_eq!(
629 fs::read_to_string(&p).unwrap(),
630 "old",
631 "unchanged until commit"
632 );
633 assert_eq!(
634 entries(dir.path()).len(),
635 2,
636 "temporary file beside the target"
637 );
638 s.commit().unwrap();
639 assert_eq!(fs::read_to_string(&p).unwrap(), "new");
640 assert_eq!(entries(dir.path()), ["out.json"]);
641 }
642
643 #[test]
644 fn uncommitted_sink_leaves_nothing() {
645 let dir = tempfile::tempdir().unwrap();
646 let p = dir.path().join("never.json");
647 {
648 let mut s = Sink::create(&Target::Path(p.clone()), OutputCompression::Gzip).unwrap();
649 s.write_all(b"partial").unwrap();
650 }
651 assert!(entries(dir.path()).is_empty());
652 }
653
654 #[test]
655 fn compressed_outputs_round_trip() {
656 let dir = tempfile::tempdir().unwrap();
657 for (c, name) in [
658 (OutputCompression::Gzip, "a.gz"),
659 (OutputCompression::Zstd, "a.zst"),
660 ] {
661 let p = dir.path().join(name);
662 let mut s = Sink::create(&Target::Path(p.clone()), c).unwrap();
663 s.write_all(b"hello\n").unwrap();
664 s.commit().unwrap();
665 let bytes = fs::read(&p).unwrap();
666 let mut out = String::new();
667 match c {
668 OutputCompression::Gzip => {
669 flate2::read::MultiGzDecoder::new(&bytes[..])
670 .read_to_string(&mut out)
671 .unwrap();
672 }
673 _ => {
674 out = String::from_utf8(zstd::decode_all(&bytes[..]).unwrap()).unwrap();
675 }
676 }
677 assert_eq!(out, "hello\n");
678 }
679 }
680
681 #[test]
682 fn missing_directory_is_an_io_error() {
683 let e = Sink::create(
684 &Target::Path("/nonexistent-dir/x.json".into()),
685 OutputCompression::None,
686 )
687 .err()
688 .unwrap();
689 assert_eq!(e.class(), crate::error::ExitClass::Io);
690 assert!(e.to_string().contains("/nonexistent-dir/x.json"));
691 }
692
693 #[cfg(unix)]
694 #[test]
695 fn symlinks_are_followed_and_permissions_respect_umask() {
696 use std::os::unix::fs::{PermissionsExt, symlink};
697 let dir = tempfile::tempdir().unwrap();
698 let real = dir.path().join("real.json");
699 fs::write(&real, "old").unwrap();
700 let link = dir.path().join("link.json");
701 symlink(&real, &link).unwrap();
702 let mut s = Sink::create(&Target::Path(link.clone()), OutputCompression::None).unwrap();
703 s.write_all(b"new").unwrap();
704 s.commit().unwrap();
705 assert!(
706 fs::symlink_metadata(&link)
707 .unwrap()
708 .file_type()
709 .is_symlink()
710 );
711 assert_eq!(fs::read_to_string(&real).unwrap(), "new");
712 let mode = fs::metadata(&real).unwrap().permissions().mode() & 0o777;
713 assert_ne!(mode, 0o600, "not tempfile's private mode");
714 assert_eq!(mode & 0o200, 0o200);
715 fs::set_permissions(&real, fs::Permissions::from_mode(0o640)).unwrap();
717 let mut s = Sink::create(&Target::Path(link), OutputCompression::None).unwrap();
718 s.write_all(b"newer").unwrap();
719 s.commit().unwrap();
720 assert_eq!(fs::read_to_string(&real).unwrap(), "newer");
721 assert_eq!(
722 fs::metadata(&real).unwrap().permissions().mode() & 0o777,
723 0o640
724 );
725 }
726
727 #[cfg(unix)]
728 #[test]
729 fn fifos_are_written_directly() {
730 let dir = tempfile::tempdir().unwrap();
731 let p = dir.path().join("fifo");
732 assert!(
733 std::process::Command::new("mkfifo")
734 .arg(&p)
735 .status()
736 .unwrap()
737 .success()
738 );
739 let reader_path = p.clone();
740 let t = std::thread::spawn(move || fs::read_to_string(reader_path).unwrap());
741 let mut s = Sink::create(&Target::Path(p), OutputCompression::None).unwrap();
742 s.write_all(b"through the pipe").unwrap();
743 s.commit().unwrap();
744 assert_eq!(t.join().unwrap(), "through the pipe");
745 assert_eq!(entries(dir.path()), ["fifo"]);
746 }
747}