Skip to main content

dima_core/output/
mod.rs

1//! Writing and reading results (D10, D22, D28, D31, D50, D61; findings E2, E11, F4, P4).
2//!
3//! * **JSON** (canonical, complete): compact by default or indented with `pretty`; always ends
4//!   with a newline. Every number is finite (checked before writing: JSON has no NaN).
5//! * **JSONL**: one position per line (Nextclade's NDJSON convention); the provenance and
6//!   summaries live only in the JSON.
7//! * **TSV**: the tidy tables of [`tsv::Table`].
8//! * `.gz` and `.zst` outputs are compressed; the format comes from the extension (compound and
9//!   case-insensitive: `x.JSONL.ZST`) unless given explicitly.
10//! * [`read_results`] reads plain, gzip or Zstandard results JSON (detected by content) and
11//!   rejects other schema major versions with a clear message. Errors reading the file are I/O
12//!   errors (exit 3); corrupt or truncated compressed data is invalid input (exit 4).
13
14mod sink;
15pub mod tsv;
16
17use std::fs::File;
18use std::io::{self, BufReader, Read, Write};
19use std::path::Path;
20use std::sync::atomic::{AtomicBool, Ordering};
21
22pub use sink::{OutputCompression, Sink, Target, Written, destination_of, remove_temporaries};
23pub use tsv::{NA, Table, write_table};
24
25use crate::error::{DimaError, IoAction, IoTarget, Location, Result};
26use crate::input::StoredReadError;
27use crate::model::{HcsResults, Results, SCHEMA_VERSION};
28
29/// A file-name-safe version of a query name, for output file names (`--tsv-dir`, the desktop
30/// app's exports): characters other than letters, digits, `.`, `-` and `_` become `_`; leading
31/// and trailing dots and underscores are removed (no hidden files); `dima` if nothing is left.
32#[must_use]
33pub fn file_stem(name: &str) -> String {
34    let s: String = name
35        .chars()
36        .map(|c| {
37            if c.is_alphanumeric() || matches!(c, '.' | '-' | '_') {
38                c
39            } else {
40                '_'
41            }
42        })
43        .collect();
44    let s = s.trim_matches(|c| c == '.' || c == '_');
45    if s.is_empty() {
46        "dima".to_owned()
47    } else {
48        s.to_owned()
49    }
50}
51
52/// Serialisation format of an output.
53#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
54pub enum OutputFormat {
55    /// The complete results as one JSON document.
56    Json,
57    /// One JSON object per position per line.
58    Jsonl,
59    /// A tab-separated table.
60    Tsv,
61}
62
63impl OutputFormat {
64    /// Name on the command line.
65    #[must_use]
66    pub const fn name(self) -> &'static str {
67        match self {
68            Self::Json => "json",
69            Self::Jsonl => "jsonl",
70            Self::Tsv => "tsv",
71        }
72    }
73}
74
75/// Format and compression implied by a file name's extensions (case-insensitive): an optional
76/// `.gz`/`.zst` after `.json`, `.jsonl`/`.ndjson` or `.tsv`/`.tab`. `None` when the name has no
77/// recognised format extension.
78#[must_use]
79pub fn detect(path: &Path) -> (Option<OutputFormat>, OutputCompression) {
80    let name = path
81        .file_name()
82        .map(|n| n.to_string_lossy().to_ascii_lowercase())
83        .unwrap_or_default();
84    let (stem, compression) = if let Some(s) = name.strip_suffix(".gz") {
85        (s, OutputCompression::Gzip)
86    } else if let Some(s) = name.strip_suffix(".zst") {
87        (s, OutputCompression::Zstd)
88    } else {
89        (name.as_str(), OutputCompression::None)
90    };
91    let format = if stem.ends_with(".jsonl") || stem.ends_with(".ndjson") {
92        Some(OutputFormat::Jsonl)
93    } else if stem.ends_with(".json") {
94        Some(OutputFormat::Json)
95    } else if stem.ends_with(".tsv") || stem.ends_with(".tab") {
96        Some(OutputFormat::Tsv)
97    } else {
98        None
99    };
100    (format, compression)
101}
102
103/// Checks that every floating-point value of the results is finite.
104///
105/// # Errors
106/// [`DimaError::Internal`] naming the first non-finite value.
107pub fn check_finite(r: &Results) -> Result<()> {
108    let bad = |what: String| Err(DimaError::internal(format!("non-finite {what} in results")));
109    let ok = |v: Option<f64>| v.is_none_or(f64::is_finite);
110    let summaries = [
111        r.average_entropy,
112        r.lowest_entropy.map(|e| e.entropy),
113        r.highest_entropy.map(|e| e.entropy),
114        r.all_positions_summary.average_entropy,
115        r.all_positions_summary.lowest_entropy.map(|e| e.entropy),
116        r.all_positions_summary.highest_entropy.map(|e| e.entropy),
117        r.coverage_summary.min,
118        r.coverage_summary.mean,
119        r.coverage_summary.min_coverage,
120    ];
121    if !summaries.into_iter().all(ok) {
122        return bad("summary value".into());
123    }
124    for p in &r.results {
125        let values = [
126            p.entropy,
127            p.observed_entropy,
128            p.distinct_variants_incidence,
129            p.total_variants_incidence,
130            p.sample_coverage,
131        ];
132        if !values.into_iter().all(ok)
133            || !p.diversity_motifs.iter().all(|v| v.incidence.is_finite())
134        {
135            return bad(format!("value at position {}", p.position));
136        }
137    }
138    Ok(())
139}
140
141/// Writes the results as JSON (compact or indented), followed by a newline.
142///
143/// # Errors
144/// [`DimaError::Internal`] for non-finite values; write errors of `out`.
145pub fn write_json(results: &Results, pretty: bool, out: &mut Sink) -> Result<()> {
146    check_finite(results)?;
147    if results.schema_version == SCHEMA_VERSION {
148        write_json_value(results, pretty, out)
149    } else {
150        // Read from a file of another minor version: written as this version holds it (keys it
151        // does not know were not read), so with this version's schema number.
152        let mut current = results.clone();
153        current.schema_version = SCHEMA_VERSION.to_owned();
154        write_json_value(&current, pretty, out)
155    }
156}
157
158/// Writes HCS results as JSON, followed by a newline.
159///
160/// # Errors
161/// Write errors of `out`.
162pub fn write_hcs_json(hcs: &HcsResults, pretty: bool, out: &mut Sink) -> Result<()> {
163    check_finite_hcs(hcs)?;
164    if hcs.schema_version == SCHEMA_VERSION {
165        write_json_value(hcs, pretty, out)
166    } else {
167        let mut current = hcs.clone();
168        current.schema_version = SCHEMA_VERSION.to_owned();
169        write_json_value(&current, pretty, out)
170    }
171}
172
173fn write_json_value(value: &impl serde::Serialize, pretty: bool, out: &mut Sink) -> Result<()> {
174    let r = if pretty {
175        serde_json::to_writer_pretty(&mut *out, value)
176    } else {
177        serde_json::to_writer(&mut *out, value)
178    };
179    r.map_err(|e| out.error(io::Error::from(e)))?;
180    out.write_all(b"\n").map_err(|e| out.error(e))
181}
182
183/// Writes one JSON object per position, one per line.
184///
185/// # Errors
186/// [`DimaError::Internal`] for non-finite values; write errors of `out`.
187pub fn write_jsonl(results: &Results, out: &mut Sink) -> Result<()> {
188    check_finite(results)?;
189    for p in &results.results {
190        serde_json::to_writer(&mut *out, p).map_err(|e| out.error(io::Error::from(e)))?;
191        out.write_all(b"\n").map_err(|e| out.error(e))?;
192    }
193    Ok(())
194}
195
196/// Writes a TSV table.
197///
198/// # Errors
199/// [`DimaError::Internal`] for non-finite values; write errors of `out`.
200pub fn write_tsv(
201    results: &Results,
202    hcs: Option<&HcsResults>,
203    table: Table,
204    out: &mut Sink,
205) -> Result<()> {
206    check_finite(results)?;
207    match (table, hcs) {
208        (Table::Hcs, None) => return Err(DimaError::internal("the HCS table needs HCS results")),
209        (Table::Hcs, Some(h)) => check_finite_hcs(h)?,
210        _ => {}
211    }
212    write_table(results, hcs, table, out).map_err(|e| out.error(e))
213}
214
215/// Reads a results JSON file (plain, gzip or Zstandard, detected by content); `-` reads standard
216/// input.
217///
218/// # Errors
219/// [`DimaError::Io`] if it cannot be read; [`DimaError::InputData`] if it is not a DiMA results
220/// file of this schema major version, or its compressed data is corrupt or truncated.
221pub fn read_results(path: &Path) -> Result<Results> {
222    read_results_from(open_json(path, None)?)
223}
224
225/// [`read_results`] that stops with [`DimaError::Cancelled`] soon after `cancel` is set (a
226/// front end no longer wants the file, which can be gigabytes).
227///
228/// # Errors
229/// As [`read_results`], and [`DimaError::Cancelled`].
230pub fn read_results_cancellable(path: &Path, cancel: &AtomicBool) -> Result<Results> {
231    read_results_from(open_json(path, Some(cancel))?)
232}
233
234/// Reads a conserved-sequences JSON file (written by `dima hcs` or `--hcs-output`).
235///
236/// # Errors
237/// As [`read_results`].
238pub fn read_hcs(path: &Path) -> Result<HcsResults> {
239    let h: HcsResults = open_json(path, None)?.parse(Kind::Hcs)?;
240    check_finite_hcs(&h)?;
241    Ok(h)
242}
243
244/// A DiMA JSON document of either kind.
245#[derive(Clone, Debug, PartialEq)]
246pub enum Document {
247    /// The results of an analysis.
248    Results(Box<Results>),
249    /// Conserved sequences.
250    Hcs(Box<HcsResults>),
251}
252
253/// Reads a results or conserved-sequences JSON file, whichever it is (told apart by the keys
254/// DiMA writes first).
255///
256/// # Errors
257/// As [`read_results`].
258pub fn read_document(path: &Path) -> Result<Document> {
259    let file = open_json(path, None)?;
260    if file.kind() == Kind::Hcs {
261        let h: HcsResults = file.parse(Kind::Hcs)?;
262        check_finite_hcs(&h)?;
263        Ok(Document::Hcs(Box::new(h)))
264    } else {
265        read_results_from(file).map(|r| Document::Results(Box::new(r)))
266    }
267}
268
269fn read_results_from(file: JsonFile<'_>) -> Result<Results> {
270    let name = file.name.clone();
271    let mut results: Results = file.parse(Kind::Results)?;
272    // Positions listed out of order (for example by a tool that edited the file) are sorted
273    // (G25); repeated or missing positions are then reported by the check.
274    if !results.results.is_sorted_by_key(|p| p.position) {
275        results.results.sort_by_key(|p| p.position);
276    }
277    check_consistent(&results, &name)?;
278    Ok(results)
279}
280
281/// The two kinds of DiMA JSON file.
282#[derive(Clone, Copy, Debug, PartialEq, Eq)]
283enum Kind {
284    Results,
285    Hcs,
286}
287
288impl Kind {
289    const fn what(self) -> &'static str {
290        match self {
291            Self::Results => "results file",
292            Self::Hcs => "conserved-sequences file",
293        }
294    }
295
296    const fn hint(self) -> &'static str {
297        match self {
298            Self::Results => "give the JSON written by `dima analyze` (optionally .gz or .zst)",
299            Self::Hcs => {
300                "give the JSON written by `dima hcs` or `--hcs-output` (optionally .gz or .zst)"
301            }
302        }
303    }
304}
305
306/// A document's schema version.
307trait Versioned {
308    fn schema_version(&self) -> &str;
309}
310
311impl Versioned for Results {
312    fn schema_version(&self) -> &str {
313        &self.schema_version
314    }
315}
316
317impl Versioned for HcsResults {
318    fn schema_version(&self) -> &str {
319        &self.schema_version
320    }
321}
322
323/// The stored bytes of a JSON file: marks read errors (I/O, exit 3) apart from decoding errors
324/// (corrupt data, exit 4), and stops when cancelled.
325struct Stored<'a> {
326    raw: Box<dyn Read + 'a>,
327    cancel: Option<&'a AtomicBool>,
328}
329
330/// Marks the error that stops a cancelled read.
331#[derive(Debug)]
332struct ReadCancelled;
333
334impl std::fmt::Display for ReadCancelled {
335    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
336        f.write_str("cancelled")
337    }
338}
339
340impl std::error::Error for ReadCancelled {}
341
342impl Read for Stored<'_> {
343    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
344        if self.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
345            return Err(io::Error::other(ReadCancelled));
346        }
347        loop {
348            match self.raw.read(buf) {
349                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
350                Err(e) => return Err(io::Error::new(e.kind(), StoredReadError(e))),
351                ok => return ok,
352            }
353        }
354    }
355}
356
357/// The compression of a JSON file, for messages and the command that tests it.
358#[derive(Clone, Copy)]
359enum Packed {
360    Plain,
361    Gzip,
362    Zstd,
363}
364
365/// The decompressed text of a JSON file: the bytes already read for recognition, then the rest.
366type Rejoined<'a> = io::Chain<io::Cursor<Vec<u8>>, Box<dyn Read + 'a>>;
367
368/// A JSON file opened for parsing: decompressed, with its first bytes read.
369struct JsonFile<'a> {
370    reader: BufReader<Rejoined<'a>>,
371    /// The first bytes (lossily decoded), where DiMA writes its leading keys.
372    prefix: String,
373    name: String,
374    target: IoTarget,
375    packed: Packed,
376}
377
378fn open_json<'a>(path: &Path, cancel: Option<&'a AtomicBool>) -> Result<JsonFile<'a>> {
379    let (target, raw): (IoTarget, Box<dyn Read>) = if path.as_os_str() == "-" {
380        (IoTarget::Stdin, Box::new(io::stdin()))
381    } else {
382        let t = IoTarget::Path(path.to_path_buf());
383        if path.is_dir() {
384            return Err(
385                DimaError::usage(format!("'{}' is a directory", path.display()))
386                    .with_hint("give a JSON file written by DiMA"),
387            );
388        }
389        let f = File::open(path).map_err(|e| DimaError::io(IoAction::Open, t.clone(), e))?;
390        (t, Box::new(f))
391    };
392    let name = match &target {
393        IoTarget::Path(p) => p.display().to_string(),
394        _ => "standard input".to_owned(),
395    };
396    let mut packed = Packed::Plain;
397    let read_err = |e: io::Error, packed: Packed| stream_error(e, &target, &name, packed);
398    // Sniffing reads fixed prefixes however the input is split into reads (pipes, slow
399    // decompression), then puts them back in front of the stream.
400    let mut raw = Stored { raw, cancel };
401    let head = read_prefix(&mut raw, 4).map_err(|e| read_err(e, packed))?;
402    let stored = BufReader::with_capacity(1 << 20, io::Cursor::new(head.clone()).chain(raw));
403    let mut decoded: Box<dyn Read + 'a> = match head.as_slice() {
404        [0x1F, 0x8B, ..] => {
405            packed = Packed::Gzip;
406            Box::new(flate2::bufread::MultiGzDecoder::new(stored))
407        }
408        [0x28, 0xB5, 0x2F, 0xFD, ..] => {
409            packed = Packed::Zstd;
410            Box::new(
411                zstd::stream::read::Decoder::with_buffer(stored)
412                    .map_err(|e| read_err(e, packed))?,
413            )
414        }
415        _ => Box::new(stored),
416    };
417    // DiMA writes the schema version first, then the keys that tell the kinds apart, so a file
418    // of another major version or kind is refused before the whole file is parsed. A
419    // byte-order mark (added by some editors) is skipped.
420    let mut first = read_prefix(&mut decoded, 256).map_err(|e| read_err(e, packed))?;
421    if first.starts_with(b"\xEF\xBB\xBF") {
422        first.drain(..3);
423    }
424    let prefix = String::from_utf8_lossy(&first).into_owned();
425    Ok(JsonFile {
426        reader: BufReader::with_capacity(1 << 20, io::Cursor::new(first).chain(decoded)),
427        prefix,
428        name,
429        target,
430        packed,
431    })
432}
433
434impl JsonFile<'_> {
435    /// The kind the leading keys show: results open with `query_name`, conserved sequences
436    /// with `tool` and `source` (whose object holds the results' `query_name`). Results when
437    /// neither shows.
438    fn kind(&self) -> Kind {
439        let at = |key: &str| self.prefix.find(&format!("\"{key}\""));
440        match (at("source"), at("query_name")) {
441            (Some(s), Some(q)) if s < q => Kind::Hcs,
442            (Some(_), None) => Kind::Hcs,
443            _ => Kind::Results,
444        }
445    }
446
447    fn parse<T: serde::de::DeserializeOwned + Versioned>(self, kind: Kind) -> Result<T> {
448        let name = self.name.clone();
449        let not_it = || {
450            DimaError::input_data(
451                Some(Location::new(name.clone())),
452                format!("not a DiMA {}", kind.what()),
453            )
454            .with_hint(kind.hint())
455        };
456        if !self.prefix.trim_start().starts_with('{') {
457            return Err(not_it());
458        }
459        let found = self.kind();
460        if found != kind {
461            return Err(DimaError::input_data(
462                Some(Location::new(name.clone())),
463                format!("this is a DiMA {}, not a {}", found.what(), kind.what()),
464            )
465            .with_hint(kind.hint()));
466        }
467        let early_version = self
468            .prefix
469            .split("\"schema_version\"")
470            .nth(1)
471            .and_then(|rest| rest.split('"').nth(1));
472        if let Some(version) = early_version {
473            check_schema_major(version, &name)?;
474        }
475        // A newer 1.x may use values (an estimator, a method) this version does not know.
476        let newer = early_version.filter(|v| is_newer(v)).map(str::to_owned);
477        let (target, packed) = (self.target, self.packed);
478        let value: T = serde_json::from_reader(self.reader).map_err(|e| {
479            if e.is_io() {
480                stream_error(io::Error::from(e), &target, &name, packed)
481            } else if e.to_string().starts_with("missing field `schema_version`") {
482                // A JSON object, but not one written by DiMA.
483                not_it()
484            } else if let Some(version) = &newer {
485                DimaError::input_data(
486                    Some(Location::new(name.clone())),
487                    format!(
488                        "written by a newer DiMA (results schema {version}) with values this \
489                         version (schema {SCHEMA_VERSION}) does not know: {e}"
490                    ),
491                )
492                .with_hint(
493                    "open it with the DiMA version that wrote it (its run.tool.version) or a \
494                     newer one",
495                )
496            } else {
497                DimaError::input_data(
498                    Some(Location::new(name.clone())),
499                    format!("not a valid DiMA {}: {e}", kind.what()),
500                )
501                .with_hint(kind.hint())
502            }
503        })?;
504        check_schema_major(value.schema_version(), &name)?;
505        Ok(value)
506    }
507}
508
509/// An error while reading a JSON file: a cancellation, an I/O error on the stored bytes,
510/// or corrupt or truncated compressed data.
511fn stream_error(e: io::Error, target: &IoTarget, name: &str, packed: Packed) -> DimaError {
512    let kind = e.kind();
513    let detail = e.to_string();
514    if e.get_ref().is_some_and(|i| i.is::<ReadCancelled>()) {
515        return DimaError::Cancelled;
516    }
517    if e.get_ref().is_some_and(|i| i.is::<StoredReadError>()) {
518        let stored = e
519            .into_inner()
520            .and_then(|i| i.downcast::<StoredReadError>().ok())
521            .map_or_else(|| io::Error::from(kind), |s| s.0);
522        return DimaError::io(IoAction::Read, target.clone(), stored);
523    }
524    // Every error reading the stored bytes is marked, so the others come from the decoder.
525    let (format, check) = match packed {
526        Packed::Plain => return DimaError::io(IoAction::Read, target.clone(), e),
527        Packed::Gzip => ("gzip", "gzip -t"),
528        Packed::Zstd => ("Zstandard", "zstd -t"),
529    };
530    let what = if kind == io::ErrorKind::UnexpectedEof {
531        "truncated"
532    } else {
533        "corrupt"
534    };
535    DimaError::input_data(
536        Some(Location::new(name.to_owned())),
537        format!("{what} {format} data: {detail}"),
538    )
539    .with_hint(format!(
540        "the file may be truncated or damaged; test it with `{check}`, or write it again with \
541         `dima analyze` or `dima view`"
542    ))
543}
544
545/// Refuses results of another schema major version.
546fn check_schema_major(version: &str, name: &str) -> Result<()> {
547    let major = |v: &str| v.split('.').next().unwrap_or("").to_owned();
548    if major(version) == major(SCHEMA_VERSION) {
549        return Ok(());
550    }
551    Err(DimaError::input_data(
552        Some(Location::new(name.to_owned())),
553        format!(
554            "written with results schema {version}, but this version of DiMA reads schema \
555             {SCHEMA_VERSION} (major version {})",
556            major(SCHEMA_VERSION)
557        ),
558    )
559    .with_hint(
560        "re-run the analysis with this version, or use the DiMA version that wrote the file",
561    ))
562}
563
564/// `true` when `version` is a later schema version than this one (`1.2.0` after `1.0.0`);
565/// unparsable versions are not.
566fn is_newer(version: &str) -> bool {
567    let parse = |v: &str| -> Option<Vec<u64>> { v.split('.').map(|p| p.parse().ok()).collect() };
568    matches!((parse(version), parse(SCHEMA_VERSION)), (Some(a), Some(b)) if a > b)
569}
570
571/// Up to `n` bytes from the start of `r` (fewer only at the end of the input).
572fn read_prefix(r: &mut impl Read, n: u64) -> io::Result<Vec<u8>> {
573    let mut buf = Vec::new();
574    r.by_ref().take(n).read_to_end(&mut buf)?;
575    Ok(buf)
576}
577
578/// Checks what downstream computations (conserved sequences, tables) rely on, so that an
579/// edited or damaged results file is reported as invalid input instead of failing later:
580/// k and T at least 1, positions 1..P (sorted) each listed once, every variant k upper-case
581/// letters, counts at least 1 and summing to the position's support.
582fn check_consistent(r: &Results, name: &str) -> Result<()> {
583    let invalid = |message: String| {
584        DimaError::input_data(Some(Location::new(name.to_owned())), message)
585            .with_hint("the file was edited or damaged; run `dima analyze` again to recreate it")
586    };
587    if r.kmer_length == 0 || r.support_threshold == 0 {
588        return Err(invalid(
589            "the k-mer length and the support threshold must be at least 1".to_owned(),
590        ));
591    }
592    if r.results.len() as u64 != r.position_count {
593        return Err(invalid(format!(
594            "position_count is {} but there are {} positions",
595            r.position_count,
596            r.results.len()
597        )));
598    }
599    let k = usize::try_from(r.kmer_length).unwrap_or(usize::MAX);
600    for (i, p) in r.results.iter().enumerate() {
601        let expected = i as u64 + 1;
602        if p.position != expected {
603            return Err(invalid(if p.position == 0 {
604                "positions start at 1, but position 0 is listed".to_owned()
605            } else if p.position < expected {
606                format!("position {} is listed more than once", p.position)
607            } else {
608                format!("position {expected} is missing")
609            }));
610        }
611        let mut total = 0u64;
612        for v in &p.diversity_motifs {
613            if v.sequence.len() != k || !v.sequence.bytes().all(|b| b.is_ascii_uppercase()) {
614                return Err(invalid(format!(
615                    "position {}: variant {:?} is not {k} upper-case letters",
616                    p.position, v.sequence
617                )));
618            }
619            if v.count == 0 {
620                return Err(invalid(format!(
621                    "position {}: variant {} has a count of 0",
622                    p.position, v.sequence
623                )));
624            }
625            total = total.saturating_add(v.count);
626        }
627        if total != p.support {
628            return Err(invalid(format!(
629                "position {}: the variant counts sum to {total}, but the support is {}",
630                p.position, p.support
631            )));
632        }
633        // The ranges of `schemas/results.schema.json`, so an edited value is refused here
634        // instead of being written on.
635        let values = [
636            ("entropy", p.entropy, 0.0, f64::INFINITY),
637            ("observed_entropy", p.observed_entropy, 0.0, f64::INFINITY),
638            (
639                "distinct_variants_incidence",
640                p.distinct_variants_incidence,
641                0.0,
642                100.0,
643            ),
644            (
645                "total_variants_incidence",
646                p.total_variants_incidence,
647                0.0,
648                100.0,
649            ),
650            ("sample_coverage", p.sample_coverage, 0.0, 1.0),
651        ];
652        for (field, value, low, high) in values {
653            if let Some(v) = value {
654                if !(low..=high).contains(&v) {
655                    return Err(invalid(format!(
656                        "position {}: {field} is {v}, outside {low} to {high}",
657                        p.position
658                    )));
659                }
660            }
661        }
662        if let Some(v) = p
663            .diversity_motifs
664            .iter()
665            .find(|v| !(v.incidence > 0.0 && v.incidence <= 100.0))
666        {
667            return Err(invalid(format!(
668                "position {}: variant {} has an incidence of {}, outside (0, 100]",
669                p.position, v.sequence, v.incidence
670            )));
671        }
672    }
673    Ok(())
674}
675
676/// Rejects non-finite numbers in conserved-sequence results.
677///
678/// # Errors
679/// [`DimaError::Internal`]: a non-finite value is a bug.
680pub fn check_finite_hcs(h: &HcsResults) -> Result<()> {
681    let finite = h.regions.iter().all(|r| {
682        r.min_incidence.is_finite()
683            && r.conservation_lower_bound_95.is_finite()
684            && r.members
685                .iter()
686                .all(|m| m.incidence.is_finite() && m.conservation_lower_bound_95.is_finite())
687    });
688    if finite {
689        Ok(())
690    } else {
691        Err(DimaError::internal(
692            "non-finite value in conserved-sequence results",
693        ))
694    }
695}
696
697#[cfg(test)]
698mod tests {
699    use super::*;
700    use std::path::PathBuf;
701
702    #[test]
703    fn file_stems_are_safe_names() {
704        assert_eq!(file_stem("MERS spike/aa"), "MERS_spike_aa");
705        assert_eq!(file_stem("..."), "dima");
706        assert_eq!(file_stem(""), "dima");
707        assert_eq!(file_stem("HIV-1.pol"), "HIV-1.pol");
708        assert_eq!(file_stem(".hidden_"), "hidden");
709        assert_eq!(file_stem("Zürich"), "Zürich");
710    }
711
712    #[test]
713    fn formats_from_extensions() {
714        let d = |s: &str| detect(&PathBuf::from(s));
715        assert_eq!(
716            d("out.json"),
717            (Some(OutputFormat::Json), OutputCompression::None)
718        );
719        assert_eq!(
720            d("dir.v1/OUT.JSON.GZ"),
721            (Some(OutputFormat::Json), OutputCompression::Gzip)
722        );
723        assert_eq!(
724            d("x.jsonl.zst"),
725            (Some(OutputFormat::Jsonl), OutputCompression::Zstd)
726        );
727        assert_eq!(
728            d("x.ndjson"),
729            (Some(OutputFormat::Jsonl), OutputCompression::None)
730        );
731        assert_eq!(
732            d("x.tab"),
733            (Some(OutputFormat::Tsv), OutputCompression::None)
734        );
735        assert_eq!(
736            d("x.tsv.gz"),
737            (Some(OutputFormat::Tsv), OutputCompression::Gzip)
738        );
739        assert_eq!(d("x.txt"), (None, OutputCompression::None));
740        assert_eq!(d("x.gz"), (None, OutputCompression::Gzip));
741        assert_eq!(d("-"), (None, OutputCompression::None));
742    }
743}