1mod sink;
15pub mod tsv;
16
17use std::fs::File;
18use std::io::{self, BufReader, Read, Write};
19use std::path::Path;
20use std::sync::atomic::{AtomicBool, Ordering};
21
22pub use sink::{OutputCompression, Sink, Target, Written, destination_of, remove_temporaries};
23pub use tsv::{NA, Table, write_table};
24
25use crate::error::{DimaError, IoAction, IoTarget, Location, Result};
26use crate::input::StoredReadError;
27use crate::model::{HcsResults, Results, SCHEMA_VERSION};
28
29#[must_use]
33pub fn file_stem(name: &str) -> String {
34 let s: String = name
35 .chars()
36 .map(|c| {
37 if c.is_alphanumeric() || matches!(c, '.' | '-' | '_') {
38 c
39 } else {
40 '_'
41 }
42 })
43 .collect();
44 let s = s.trim_matches(|c| c == '.' || c == '_');
45 if s.is_empty() {
46 "dima".to_owned()
47 } else {
48 s.to_owned()
49 }
50}
51
52#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
54pub enum OutputFormat {
55 Json,
57 Jsonl,
59 Tsv,
61}
62
63impl OutputFormat {
64 #[must_use]
66 pub const fn name(self) -> &'static str {
67 match self {
68 Self::Json => "json",
69 Self::Jsonl => "jsonl",
70 Self::Tsv => "tsv",
71 }
72 }
73}
74
75#[must_use]
79pub fn detect(path: &Path) -> (Option<OutputFormat>, OutputCompression) {
80 let name = path
81 .file_name()
82 .map(|n| n.to_string_lossy().to_ascii_lowercase())
83 .unwrap_or_default();
84 let (stem, compression) = if let Some(s) = name.strip_suffix(".gz") {
85 (s, OutputCompression::Gzip)
86 } else if let Some(s) = name.strip_suffix(".zst") {
87 (s, OutputCompression::Zstd)
88 } else {
89 (name.as_str(), OutputCompression::None)
90 };
91 let format = if stem.ends_with(".jsonl") || stem.ends_with(".ndjson") {
92 Some(OutputFormat::Jsonl)
93 } else if stem.ends_with(".json") {
94 Some(OutputFormat::Json)
95 } else if stem.ends_with(".tsv") || stem.ends_with(".tab") {
96 Some(OutputFormat::Tsv)
97 } else {
98 None
99 };
100 (format, compression)
101}
102
103pub fn check_finite(r: &Results) -> Result<()> {
108 let bad = |what: String| Err(DimaError::internal(format!("non-finite {what} in results")));
109 let ok = |v: Option<f64>| v.is_none_or(f64::is_finite);
110 let summaries = [
111 r.average_entropy,
112 r.lowest_entropy.map(|e| e.entropy),
113 r.highest_entropy.map(|e| e.entropy),
114 r.all_positions_summary.average_entropy,
115 r.all_positions_summary.lowest_entropy.map(|e| e.entropy),
116 r.all_positions_summary.highest_entropy.map(|e| e.entropy),
117 r.coverage_summary.min,
118 r.coverage_summary.mean,
119 r.coverage_summary.min_coverage,
120 ];
121 if !summaries.into_iter().all(ok) {
122 return bad("summary value".into());
123 }
124 for p in &r.results {
125 let values = [
126 p.entropy,
127 p.observed_entropy,
128 p.distinct_variants_incidence,
129 p.total_variants_incidence,
130 p.sample_coverage,
131 ];
132 if !values.into_iter().all(ok)
133 || !p.diversity_motifs.iter().all(|v| v.incidence.is_finite())
134 {
135 return bad(format!("value at position {}", p.position));
136 }
137 }
138 Ok(())
139}
140
141pub fn write_json(results: &Results, pretty: bool, out: &mut Sink) -> Result<()> {
146 check_finite(results)?;
147 if results.schema_version == SCHEMA_VERSION {
148 write_json_value(results, pretty, out)
149 } else {
150 let mut current = results.clone();
153 current.schema_version = SCHEMA_VERSION.to_owned();
154 write_json_value(¤t, pretty, out)
155 }
156}
157
158pub fn write_hcs_json(hcs: &HcsResults, pretty: bool, out: &mut Sink) -> Result<()> {
163 check_finite_hcs(hcs)?;
164 if hcs.schema_version == SCHEMA_VERSION {
165 write_json_value(hcs, pretty, out)
166 } else {
167 let mut current = hcs.clone();
168 current.schema_version = SCHEMA_VERSION.to_owned();
169 write_json_value(¤t, pretty, out)
170 }
171}
172
173fn write_json_value(value: &impl serde::Serialize, pretty: bool, out: &mut Sink) -> Result<()> {
174 let r = if pretty {
175 serde_json::to_writer_pretty(&mut *out, value)
176 } else {
177 serde_json::to_writer(&mut *out, value)
178 };
179 r.map_err(|e| out.error(io::Error::from(e)))?;
180 out.write_all(b"\n").map_err(|e| out.error(e))
181}
182
183pub fn write_jsonl(results: &Results, out: &mut Sink) -> Result<()> {
188 check_finite(results)?;
189 for p in &results.results {
190 serde_json::to_writer(&mut *out, p).map_err(|e| out.error(io::Error::from(e)))?;
191 out.write_all(b"\n").map_err(|e| out.error(e))?;
192 }
193 Ok(())
194}
195
196pub fn write_tsv(
201 results: &Results,
202 hcs: Option<&HcsResults>,
203 table: Table,
204 out: &mut Sink,
205) -> Result<()> {
206 check_finite(results)?;
207 match (table, hcs) {
208 (Table::Hcs, None) => return Err(DimaError::internal("the HCS table needs HCS results")),
209 (Table::Hcs, Some(h)) => check_finite_hcs(h)?,
210 _ => {}
211 }
212 write_table(results, hcs, table, out).map_err(|e| out.error(e))
213}
214
215pub fn read_results(path: &Path) -> Result<Results> {
222 read_results_from(open_json(path, None)?)
223}
224
225pub fn read_results_cancellable(path: &Path, cancel: &AtomicBool) -> Result<Results> {
231 read_results_from(open_json(path, Some(cancel))?)
232}
233
234pub fn read_hcs(path: &Path) -> Result<HcsResults> {
239 let h: HcsResults = open_json(path, None)?.parse(Kind::Hcs)?;
240 check_finite_hcs(&h)?;
241 Ok(h)
242}
243
244#[derive(Clone, Debug, PartialEq)]
246pub enum Document {
247 Results(Box<Results>),
249 Hcs(Box<HcsResults>),
251}
252
253pub fn read_document(path: &Path) -> Result<Document> {
259 let file = open_json(path, None)?;
260 if file.kind() == Kind::Hcs {
261 let h: HcsResults = file.parse(Kind::Hcs)?;
262 check_finite_hcs(&h)?;
263 Ok(Document::Hcs(Box::new(h)))
264 } else {
265 read_results_from(file).map(|r| Document::Results(Box::new(r)))
266 }
267}
268
269fn read_results_from(file: JsonFile<'_>) -> Result<Results> {
270 let name = file.name.clone();
271 let mut results: Results = file.parse(Kind::Results)?;
272 if !results.results.is_sorted_by_key(|p| p.position) {
275 results.results.sort_by_key(|p| p.position);
276 }
277 check_consistent(&results, &name)?;
278 Ok(results)
279}
280
281#[derive(Clone, Copy, Debug, PartialEq, Eq)]
283enum Kind {
284 Results,
285 Hcs,
286}
287
288impl Kind {
289 const fn what(self) -> &'static str {
290 match self {
291 Self::Results => "results file",
292 Self::Hcs => "conserved-sequences file",
293 }
294 }
295
296 const fn hint(self) -> &'static str {
297 match self {
298 Self::Results => "give the JSON written by `dima analyze` (optionally .gz or .zst)",
299 Self::Hcs => {
300 "give the JSON written by `dima hcs` or `--hcs-output` (optionally .gz or .zst)"
301 }
302 }
303 }
304}
305
306trait Versioned {
308 fn schema_version(&self) -> &str;
309}
310
311impl Versioned for Results {
312 fn schema_version(&self) -> &str {
313 &self.schema_version
314 }
315}
316
317impl Versioned for HcsResults {
318 fn schema_version(&self) -> &str {
319 &self.schema_version
320 }
321}
322
323struct Stored<'a> {
326 raw: Box<dyn Read + 'a>,
327 cancel: Option<&'a AtomicBool>,
328}
329
330#[derive(Debug)]
332struct ReadCancelled;
333
334impl std::fmt::Display for ReadCancelled {
335 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
336 f.write_str("cancelled")
337 }
338}
339
340impl std::error::Error for ReadCancelled {}
341
342impl Read for Stored<'_> {
343 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
344 if self.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
345 return Err(io::Error::other(ReadCancelled));
346 }
347 loop {
348 match self.raw.read(buf) {
349 Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
350 Err(e) => return Err(io::Error::new(e.kind(), StoredReadError(e))),
351 ok => return ok,
352 }
353 }
354 }
355}
356
357#[derive(Clone, Copy)]
359enum Packed {
360 Plain,
361 Gzip,
362 Zstd,
363}
364
365type Rejoined<'a> = io::Chain<io::Cursor<Vec<u8>>, Box<dyn Read + 'a>>;
367
368struct JsonFile<'a> {
370 reader: BufReader<Rejoined<'a>>,
371 prefix: String,
373 name: String,
374 target: IoTarget,
375 packed: Packed,
376}
377
378fn open_json<'a>(path: &Path, cancel: Option<&'a AtomicBool>) -> Result<JsonFile<'a>> {
379 let (target, raw): (IoTarget, Box<dyn Read>) = if path.as_os_str() == "-" {
380 (IoTarget::Stdin, Box::new(io::stdin()))
381 } else {
382 let t = IoTarget::Path(path.to_path_buf());
383 if path.is_dir() {
384 return Err(
385 DimaError::usage(format!("'{}' is a directory", path.display()))
386 .with_hint("give a JSON file written by DiMA"),
387 );
388 }
389 let f = File::open(path).map_err(|e| DimaError::io(IoAction::Open, t.clone(), e))?;
390 (t, Box::new(f))
391 };
392 let name = match &target {
393 IoTarget::Path(p) => p.display().to_string(),
394 _ => "standard input".to_owned(),
395 };
396 let mut packed = Packed::Plain;
397 let read_err = |e: io::Error, packed: Packed| stream_error(e, &target, &name, packed);
398 let mut raw = Stored { raw, cancel };
401 let head = read_prefix(&mut raw, 4).map_err(|e| read_err(e, packed))?;
402 let stored = BufReader::with_capacity(1 << 20, io::Cursor::new(head.clone()).chain(raw));
403 let mut decoded: Box<dyn Read + 'a> = match head.as_slice() {
404 [0x1F, 0x8B, ..] => {
405 packed = Packed::Gzip;
406 Box::new(flate2::bufread::MultiGzDecoder::new(stored))
407 }
408 [0x28, 0xB5, 0x2F, 0xFD, ..] => {
409 packed = Packed::Zstd;
410 Box::new(
411 zstd::stream::read::Decoder::with_buffer(stored)
412 .map_err(|e| read_err(e, packed))?,
413 )
414 }
415 _ => Box::new(stored),
416 };
417 let mut first = read_prefix(&mut decoded, 256).map_err(|e| read_err(e, packed))?;
421 if first.starts_with(b"\xEF\xBB\xBF") {
422 first.drain(..3);
423 }
424 let prefix = String::from_utf8_lossy(&first).into_owned();
425 Ok(JsonFile {
426 reader: BufReader::with_capacity(1 << 20, io::Cursor::new(first).chain(decoded)),
427 prefix,
428 name,
429 target,
430 packed,
431 })
432}
433
434impl JsonFile<'_> {
435 fn kind(&self) -> Kind {
439 let at = |key: &str| self.prefix.find(&format!("\"{key}\""));
440 match (at("source"), at("query_name")) {
441 (Some(s), Some(q)) if s < q => Kind::Hcs,
442 (Some(_), None) => Kind::Hcs,
443 _ => Kind::Results,
444 }
445 }
446
447 fn parse<T: serde::de::DeserializeOwned + Versioned>(self, kind: Kind) -> Result<T> {
448 let name = self.name.clone();
449 let not_it = || {
450 DimaError::input_data(
451 Some(Location::new(name.clone())),
452 format!("not a DiMA {}", kind.what()),
453 )
454 .with_hint(kind.hint())
455 };
456 if !self.prefix.trim_start().starts_with('{') {
457 return Err(not_it());
458 }
459 let found = self.kind();
460 if found != kind {
461 return Err(DimaError::input_data(
462 Some(Location::new(name.clone())),
463 format!("this is a DiMA {}, not a {}", found.what(), kind.what()),
464 )
465 .with_hint(kind.hint()));
466 }
467 let early_version = self
468 .prefix
469 .split("\"schema_version\"")
470 .nth(1)
471 .and_then(|rest| rest.split('"').nth(1));
472 if let Some(version) = early_version {
473 check_schema_major(version, &name)?;
474 }
475 let newer = early_version.filter(|v| is_newer(v)).map(str::to_owned);
477 let (target, packed) = (self.target, self.packed);
478 let value: T = serde_json::from_reader(self.reader).map_err(|e| {
479 if e.is_io() {
480 stream_error(io::Error::from(e), &target, &name, packed)
481 } else if e.to_string().starts_with("missing field `schema_version`") {
482 not_it()
484 } else if let Some(version) = &newer {
485 DimaError::input_data(
486 Some(Location::new(name.clone())),
487 format!(
488 "written by a newer DiMA (results schema {version}) with values this \
489 version (schema {SCHEMA_VERSION}) does not know: {e}"
490 ),
491 )
492 .with_hint(
493 "open it with the DiMA version that wrote it (its run.tool.version) or a \
494 newer one",
495 )
496 } else {
497 DimaError::input_data(
498 Some(Location::new(name.clone())),
499 format!("not a valid DiMA {}: {e}", kind.what()),
500 )
501 .with_hint(kind.hint())
502 }
503 })?;
504 check_schema_major(value.schema_version(), &name)?;
505 Ok(value)
506 }
507}
508
509fn stream_error(e: io::Error, target: &IoTarget, name: &str, packed: Packed) -> DimaError {
512 let kind = e.kind();
513 let detail = e.to_string();
514 if e.get_ref().is_some_and(|i| i.is::<ReadCancelled>()) {
515 return DimaError::Cancelled;
516 }
517 if e.get_ref().is_some_and(|i| i.is::<StoredReadError>()) {
518 let stored = e
519 .into_inner()
520 .and_then(|i| i.downcast::<StoredReadError>().ok())
521 .map_or_else(|| io::Error::from(kind), |s| s.0);
522 return DimaError::io(IoAction::Read, target.clone(), stored);
523 }
524 let (format, check) = match packed {
526 Packed::Plain => return DimaError::io(IoAction::Read, target.clone(), e),
527 Packed::Gzip => ("gzip", "gzip -t"),
528 Packed::Zstd => ("Zstandard", "zstd -t"),
529 };
530 let what = if kind == io::ErrorKind::UnexpectedEof {
531 "truncated"
532 } else {
533 "corrupt"
534 };
535 DimaError::input_data(
536 Some(Location::new(name.to_owned())),
537 format!("{what} {format} data: {detail}"),
538 )
539 .with_hint(format!(
540 "the file may be truncated or damaged; test it with `{check}`, or write it again with \
541 `dima analyze` or `dima view`"
542 ))
543}
544
545fn check_schema_major(version: &str, name: &str) -> Result<()> {
547 let major = |v: &str| v.split('.').next().unwrap_or("").to_owned();
548 if major(version) == major(SCHEMA_VERSION) {
549 return Ok(());
550 }
551 Err(DimaError::input_data(
552 Some(Location::new(name.to_owned())),
553 format!(
554 "written with results schema {version}, but this version of DiMA reads schema \
555 {SCHEMA_VERSION} (major version {})",
556 major(SCHEMA_VERSION)
557 ),
558 )
559 .with_hint(
560 "re-run the analysis with this version, or use the DiMA version that wrote the file",
561 ))
562}
563
564fn is_newer(version: &str) -> bool {
567 let parse = |v: &str| -> Option<Vec<u64>> { v.split('.').map(|p| p.parse().ok()).collect() };
568 matches!((parse(version), parse(SCHEMA_VERSION)), (Some(a), Some(b)) if a > b)
569}
570
571fn read_prefix(r: &mut impl Read, n: u64) -> io::Result<Vec<u8>> {
573 let mut buf = Vec::new();
574 r.by_ref().take(n).read_to_end(&mut buf)?;
575 Ok(buf)
576}
577
578fn check_consistent(r: &Results, name: &str) -> Result<()> {
583 let invalid = |message: String| {
584 DimaError::input_data(Some(Location::new(name.to_owned())), message)
585 .with_hint("the file was edited or damaged; run `dima analyze` again to recreate it")
586 };
587 if r.kmer_length == 0 || r.support_threshold == 0 {
588 return Err(invalid(
589 "the k-mer length and the support threshold must be at least 1".to_owned(),
590 ));
591 }
592 if r.results.len() as u64 != r.position_count {
593 return Err(invalid(format!(
594 "position_count is {} but there are {} positions",
595 r.position_count,
596 r.results.len()
597 )));
598 }
599 let k = usize::try_from(r.kmer_length).unwrap_or(usize::MAX);
600 for (i, p) in r.results.iter().enumerate() {
601 let expected = i as u64 + 1;
602 if p.position != expected {
603 return Err(invalid(if p.position == 0 {
604 "positions start at 1, but position 0 is listed".to_owned()
605 } else if p.position < expected {
606 format!("position {} is listed more than once", p.position)
607 } else {
608 format!("position {expected} is missing")
609 }));
610 }
611 let mut total = 0u64;
612 for v in &p.diversity_motifs {
613 if v.sequence.len() != k || !v.sequence.bytes().all(|b| b.is_ascii_uppercase()) {
614 return Err(invalid(format!(
615 "position {}: variant {:?} is not {k} upper-case letters",
616 p.position, v.sequence
617 )));
618 }
619 if v.count == 0 {
620 return Err(invalid(format!(
621 "position {}: variant {} has a count of 0",
622 p.position, v.sequence
623 )));
624 }
625 total = total.saturating_add(v.count);
626 }
627 if total != p.support {
628 return Err(invalid(format!(
629 "position {}: the variant counts sum to {total}, but the support is {}",
630 p.position, p.support
631 )));
632 }
633 let values = [
636 ("entropy", p.entropy, 0.0, f64::INFINITY),
637 ("observed_entropy", p.observed_entropy, 0.0, f64::INFINITY),
638 (
639 "distinct_variants_incidence",
640 p.distinct_variants_incidence,
641 0.0,
642 100.0,
643 ),
644 (
645 "total_variants_incidence",
646 p.total_variants_incidence,
647 0.0,
648 100.0,
649 ),
650 ("sample_coverage", p.sample_coverage, 0.0, 1.0),
651 ];
652 for (field, value, low, high) in values {
653 if let Some(v) = value {
654 if !(low..=high).contains(&v) {
655 return Err(invalid(format!(
656 "position {}: {field} is {v}, outside {low} to {high}",
657 p.position
658 )));
659 }
660 }
661 }
662 if let Some(v) = p
663 .diversity_motifs
664 .iter()
665 .find(|v| !(v.incidence > 0.0 && v.incidence <= 100.0))
666 {
667 return Err(invalid(format!(
668 "position {}: variant {} has an incidence of {}, outside (0, 100]",
669 p.position, v.sequence, v.incidence
670 )));
671 }
672 }
673 Ok(())
674}
675
676pub fn check_finite_hcs(h: &HcsResults) -> Result<()> {
681 let finite = h.regions.iter().all(|r| {
682 r.min_incidence.is_finite()
683 && r.conservation_lower_bound_95.is_finite()
684 && r.members
685 .iter()
686 .all(|m| m.incidence.is_finite() && m.conservation_lower_bound_95.is_finite())
687 });
688 if finite {
689 Ok(())
690 } else {
691 Err(DimaError::internal(
692 "non-finite value in conserved-sequence results",
693 ))
694 }
695}
696
697#[cfg(test)]
698mod tests {
699 use super::*;
700 use std::path::PathBuf;
701
702 #[test]
703 fn file_stems_are_safe_names() {
704 assert_eq!(file_stem("MERS spike/aa"), "MERS_spike_aa");
705 assert_eq!(file_stem("..."), "dima");
706 assert_eq!(file_stem(""), "dima");
707 assert_eq!(file_stem("HIV-1.pol"), "HIV-1.pol");
708 assert_eq!(file_stem(".hidden_"), "hidden");
709 assert_eq!(file_stem("Zürich"), "Zürich");
710 }
711
712 #[test]
713 fn formats_from_extensions() {
714 let d = |s: &str| detect(&PathBuf::from(s));
715 assert_eq!(
716 d("out.json"),
717 (Some(OutputFormat::Json), OutputCompression::None)
718 );
719 assert_eq!(
720 d("dir.v1/OUT.JSON.GZ"),
721 (Some(OutputFormat::Json), OutputCompression::Gzip)
722 );
723 assert_eq!(
724 d("x.jsonl.zst"),
725 (Some(OutputFormat::Jsonl), OutputCompression::Zstd)
726 );
727 assert_eq!(
728 d("x.ndjson"),
729 (Some(OutputFormat::Jsonl), OutputCompression::None)
730 );
731 assert_eq!(
732 d("x.tab"),
733 (Some(OutputFormat::Tsv), OutputCompression::None)
734 );
735 assert_eq!(
736 d("x.tsv.gz"),
737 (Some(OutputFormat::Tsv), OutputCompression::Gzip)
738 );
739 assert_eq!(d("x.txt"), (None, OutputCompression::None));
740 assert_eq!(d("x.gz"), (None, OutputCompression::Gzip));
741 assert_eq!(d("-"), (None, OutputCompression::None));
742 }
743}