Skip to main content

dima_core/
provenance.rs

1//! The deterministic provenance of a run (`run` in the results) and its diagnostics.
2//!
3//! Everything recorded here can change the results or is needed to interpret them: the tool
4//! version, every parameter with its origin, the alphabet decision, the input's identity, what
5//! validation found, and the references of the methods that ran. Nothing here depends on the
6//! machine, the time or the thread count, so identical inputs give identical bytes.
7
8use crate::alphabet::{ByteClassTable, Resolution};
9use crate::citations;
10use crate::config::{Alphabet, Estimator, ValidationMode};
11use crate::diagnostics::{Code, Diagnostic, Tally, agree, plural};
12use crate::header::{self, HeaderSpec, HeaderStats};
13use crate::model::{
14    AlphabetInfo, CharacterCount, EntropyInfo, MetadataInfo, Mismatches, Rarefaction, Reference,
15    ValidationInfo, WhitespaceCounts,
16};
17use crate::normalise::SequenceStats;
18
19/// Key of the reference to this software in `run.references`.
20pub const SOFTWARE_REFERENCE_KEY: &str = "dima_software";
21
22/// The references of a run: the DiMA paper, this software, Shannon entropy, the estimator, the
23/// coverage estimate, the motif definitions and, where used, alphabet detection and nucleotide
24/// ambiguity codes (D89).
25#[must_use]
26pub fn references(estimator: Estimator, alphabet: Alphabet, detected: bool) -> Vec<Reference> {
27    let mut keys: Vec<&str> = vec!["tharanga2025"];
28    let software = Reference {
29        key: SOFTWARE_REFERENCE_KEY.to_owned(),
30        citation: citations::software_text(citations::SOFTWARE_VERSION),
31        doi: None,
32        url: Some(citations::SOFTWARE_REPOSITORY.to_owned()),
33    };
34    keys.push("shannon1948");
35    keys.extend_from_slice(estimator_references(estimator));
36    keys.push("chao2012");
37    keys.push("hu2013");
38    if detected {
39        keys.push("eddy2011");
40    }
41    if alphabet.is_nucleotide() {
42        keys.push("cornishbowden1985");
43    }
44    let mut out: Vec<Reference> = Vec::with_capacity(keys.len() + 1);
45    out.push(Reference::of(keys[0]));
46    out.push(software);
47    out.extend(keys[1..].iter().map(|k| Reference::of(k)));
48    out
49}
50
51/// Citation keys of an estimator.
52#[must_use]
53pub const fn estimator_references(estimator: Estimator) -> &'static [&'static str] {
54    match estimator {
55        Estimator::Dima => &["rao1945", "blackwell1947"],
56        Estimator::ChaoWangJost => &["chao2013"],
57        Estimator::MillerMadow => &["miller1955"],
58        Estimator::Raw => &[],
59    }
60}
61
62/// Entropy settings of the run.
63#[must_use]
64pub fn entropy_info(estimator: Estimator) -> EntropyInfo {
65    EntropyInfo {
66        estimator,
67        log_base: 2,
68        rarefaction: (estimator == Estimator::Dima).then(|| Rarefaction {
69            grid: "paper_fig2".to_owned(),
70            size_rounding: "ceil".to_owned(),
71            start_rounding: "ceil_to_5".to_owned(),
72            sampling: "expected_with_replacement".to_owned(),
73            min_points: crate::entropy::RAREFACTION_MIN_POINTS as u64,
74        }),
75    }
76}
77
78/// Alphabet provenance.
79#[must_use]
80pub fn alphabet_info(
81    resolution: &Resolution,
82    records_examined: u64,
83    t_u_folded: u64,
84) -> AlphabetInfo {
85    AlphabetInfo {
86        resolved: resolution.alphabet,
87        method: resolution.method,
88        records_examined,
89        residues_examined: resolution.residues_examined,
90        t_u_folded,
91        contradiction: resolution.contradiction.clone(),
92    }
93}
94
95/// Validation provenance.
96#[must_use]
97pub fn validation_info(
98    mode: ValidationMode,
99    records: u64,
100    table: &ByteClassTable,
101    seq: &SequenceStats,
102    headers: Option<&HeaderStats>,
103) -> ValidationInfo {
104    let characters = seq
105        .characters
106        .iter()
107        .enumerate()
108        .filter(|&(_, &n)| n > 0)
109        .map(|(b, &count)| {
110            #[allow(clippy::cast_possible_truncation)] // b < 256.
111            let byte = b as u8;
112            CharacterCount {
113                character: byte
114                    .is_ascii_graphic()
115                    .then(|| char::from(byte).to_string()),
116                byte,
117                class: table.class(byte),
118                count,
119            }
120        })
121        .collect();
122    ValidationInfo {
123        mode,
124        records,
125        lowercase_folded: seq.lowercase_folded,
126        whitespace: WhitespaceCounts {
127            trailing: seq.trailing_whitespace_lines,
128            interior: seq.interior_whitespace.count,
129        },
130        characters,
131        unknown_total: seq.unknown.count,
132        header_utf8_replacements: headers.map_or(0, |h| h.utf8_replacements),
133        header_cleaning: headers.map(|h| header::cleaning_record(&h.cleaning)),
134    }
135}
136
137/// Header-metadata provenance.
138#[must_use]
139pub fn metadata_info(spec: &HeaderSpec, stats: &HeaderStats, probe_records: u64) -> MetadataInfo {
140    MetadataInfo {
141        fields: spec.fields().to_vec(),
142        stratified: spec.stratified_names(),
143        probe_records,
144        mismatches: Mismatches {
145            count: stats.mismatch_count,
146            first: stats.mismatches.clone(),
147        },
148        fill_uses: stats.cleaning.fill_uses,
149    }
150}
151
152/// Inputs of [`diagnostics`].
153#[derive(Debug)]
154pub struct DiagnosticInputs<'a> {
155    /// Resolved alphabet decision.
156    pub resolution: &'a Resolution,
157    /// Sequence statistics.
158    pub sequences: &'a SequenceStats,
159    /// Header statistics and format, with metadata.
160    pub headers: Option<(&'a HeaderSpec, &'a HeaderStats)>,
161    /// Header-value diagnostics computed from the value dictionary.
162    pub value_diagnostics: Vec<Diagnostic>,
163    /// Identifier-like field diagnostic from the probe.
164    pub identifier: Option<Diagnostic>,
165    /// Number of sequences.
166    pub sequence_count: u64,
167    /// Support threshold.
168    pub threshold: u64,
169    /// Eligible positions.
170    pub eligible_positions: u64,
171    /// `--min-coverage` and the positions below it.
172    pub below_min_coverage: Option<(f64, Tally)>,
173    /// A header fill value was given without a header format.
174    pub fill_without_format: bool,
175}
176
177/// The diagnostics of a run, in report order.
178#[must_use]
179pub fn diagnostics(d: DiagnosticInputs<'_>) -> Vec<Diagnostic> {
180    let mut out = Vec::new();
181    let alphabet = d.resolution.alphabet;
182    let s = d.sequences;
183    if s.lowercase_folded > 0 {
184        let t = Tally::count_only(s.lowercase_folded);
185        out.push(Diagnostic::new(
186            Code::LowercaseFolded,
187            format!(
188                "{} {} read as upper case",
189                plural(
190                    s.lowercase_folded,
191                    "lower-case letter",
192                    "lower-case letters"
193                ),
194                agree(s.lowercase_folded, "was", "were")
195            ),
196            t,
197        ));
198    }
199    if s.t_u_folded > 0 {
200        let (from, to) = if alphabet == Alphabet::Dna {
201            ("U", "T")
202        } else {
203            ("T", "U")
204        };
205        let t = Tally::count_only(s.t_u_folded);
206        out.push(Diagnostic::new(
207            Code::TUFolded,
208            format!(
209                "{} {from} {} read as {to} ({alphabet} alphabet)",
210                s.t_u_folded,
211                agree(s.t_u_folded, "letter was", "letters were")
212            ),
213            t,
214        ));
215    }
216    if let Some(c) = &d.resolution.contradiction {
217        let mut t = Tally::default();
218        t.add(1, || c.clone());
219        out.push(
220            Diagnostic::new(Code::AlphabetContradiction, c.clone(), t)
221                .with_hint("check --alphabet; characters outside it are excluded or rejected"),
222        );
223    }
224    if !s.interior_whitespace.is_empty() {
225        out.push(
226            Diagnostic::new(
227                Code::InteriorWhitespace,
228                format!(
229                    "{} whitespace between residues; it was ignored",
230                    plural(
231                        s.interior_whitespace.count,
232                        "sequence line contains",
233                        "sequence lines contain"
234                    )
235                ),
236                s.interior_whitespace.clone(),
237            )
238            .with_hint(
239                "check that the alignment file was not reformatted (e.g. by a word processor)",
240            ),
241        );
242    }
243    if !s.unknown.is_empty() {
244        out.push(
245            Diagnostic::new(
246                Code::UnknownCharactersExcluded,
247                format!(
248                    "{} not valid in a {alphabet} alignment; the k-mers that contain {} were \
249                     excluded",
250                    plural(s.unknown.count, "character is", "characters are"),
251                    agree(s.unknown.count, "it", "them")
252                ),
253                s.unknown.clone(),
254            )
255            .with_hint("run.validation.characters lists every character; check --alphabet"),
256        );
257    }
258    if let Some((spec, h)) = d.headers {
259        if h.mismatch_count > 0 {
260            let mut t = Tally::default();
261            for m in &h.mismatches {
262                t.add(1, || {
263                    format!(
264                        "record {} (line {}): {}",
265                        m.record,
266                        m.line,
267                        plural(m.found, "field", "fields")
268                    )
269                });
270            }
271            t.count = h.mismatch_count;
272            out.push(
273                Diagnostic::new(
274                    Code::HeaderFieldCountMismatch,
275                    format!(
276                        "{} not have the {} of the header format; {} metadata were set to {:?}",
277                        plural(h.mismatch_count, "header does", "headers do"),
278                        plural(spec.fields().len() as u64, "field", "fields"),
279                        agree(h.mismatch_count, "its", "their"),
280                        spec.fill()
281                    ),
282                    t,
283                )
284                .with_hint("run.metadata.mismatches lists the first headers in full"),
285            );
286        }
287        if h.utf8_replacements > 0 {
288            let t = Tally::count_only(h.utf8_replacements);
289            out.push(
290                Diagnostic::new(
291                    Code::HeaderUtf8Replaced,
292                    format!(
293                        "{} in headers {} replaced by U+FFFD",
294                        plural(
295                            h.utf8_replacements,
296                            "invalid UTF-8 sequence",
297                            "invalid UTF-8 sequences"
298                        ),
299                        agree(h.utf8_replacements, "was", "were")
300                    ),
301                    t,
302                )
303                .with_hint("convert the file to UTF-8, e.g. with iconv"),
304            );
305        }
306        if let Some(c) = header::cleaning_diagnostic(&h.cleaning) {
307            out.push(c);
308        }
309    }
310    out.extend(d.value_diagnostics);
311    out.extend(d.identifier);
312    if d.sequence_count <= d.threshold {
313        let mut t = Tally::default();
314        t.add(1, || {
315            format!(
316                "{}, threshold {}",
317                plural(d.sequence_count, "sequence", "sequences"),
318                d.threshold
319            )
320        });
321        out.push(
322            Diagnostic::new(
323                Code::ThresholdExceedsSequenceCount,
324                format!(
325                    "the support threshold ({}) is not below the number of sequences ({}), so no \
326                     position can have support above it; every entropy is the observed value",
327                    d.threshold, d.sequence_count
328                ),
329                t,
330            )
331            .with_hint("lower --support-threshold, or add sequences"),
332        );
333    } else if d.eligible_positions == 0 {
334        let t = Tally::count_only(1);
335        out.push(
336            Diagnostic::new(
337                Code::NoEligiblePositions,
338                format!("no position has support above the threshold ({})", d.threshold),
339                t,
340            )
341            .with_hint("gappy or ambiguous columns reduce support; consider a lower --support-threshold or a shorter k"),
342        );
343    }
344    if let Some((min, t)) = d.below_min_coverage {
345        if !t.is_empty() {
346            out.push(
347                Diagnostic::new(
348                    Code::BelowMinCoverage,
349                    format!(
350                        "{} sample coverage below {min}",
351                        plural(t.count, "position has", "positions have")
352                    ),
353                    t,
354                )
355                .with_hint("their k-mer diversity is incompletely sampled; see sample_coverage"),
356            );
357        }
358    }
359    if d.fill_without_format {
360        let mut t = Tally::default();
361        t.add(1, || "--header-fillna".to_owned());
362        out.push(
363            Diagnostic::new(
364                Code::FlagHasNoEffect,
365                "a header fill value was given without a header format; it has no effect",
366                t,
367            )
368            .with_hint("name the header fields with --header-format to use metadata"),
369        );
370    }
371    crate::diagnostics::sort(&mut out);
372    out
373}
374
375#[cfg(test)]
376mod tests {
377    use super::*;
378
379    #[test]
380    fn references_follow_the_methods() {
381        let keys = |e, a, d| {
382            references(e, a, d)
383                .into_iter()
384                .map(|r| r.key)
385                .collect::<Vec<_>>()
386        };
387        assert_eq!(
388            keys(Estimator::Dima, Alphabet::Protein, true),
389            [
390                "tharanga2025",
391                "dima_software",
392                "shannon1948",
393                "rao1945",
394                "blackwell1947",
395                "chao2012",
396                "hu2013",
397                "eddy2011"
398            ]
399        );
400        assert_eq!(
401            keys(Estimator::ChaoWangJost, Alphabet::Dna, false),
402            [
403                "tharanga2025",
404                "dima_software",
405                "shannon1948",
406                "chao2013",
407                "chao2012",
408                "hu2013",
409                "cornishbowden1985"
410            ]
411        );
412        assert!(
413            keys(Estimator::MillerMadow, Alphabet::Rna, false).contains(&"miller1955".to_owned())
414        );
415        let r = references(Estimator::Raw, Alphabet::Protein, false);
416        assert_eq!(r[0].doi.as_deref(), Some("10.1093/bib/bbae607"));
417        assert!(r[1].citation.contains("dima"));
418    }
419
420    #[test]
421    fn entropy_settings() {
422        assert!(entropy_info(Estimator::Dima).rarefaction.is_some());
423        assert!(entropy_info(Estimator::Raw).rarefaction.is_none());
424        assert_eq!(
425            entropy_info(Estimator::Dima)
426                .rarefaction
427                .unwrap()
428                .min_points,
429            5
430        );
431    }
432}