Skip to main content

dima_core/
validation.rs

1//! The first-batch probe shared by the analysis and the GUI's pre-check (D40, D85).
2//!
3//! [`inspect`] reads only the first batch (at least min(1,000, N) records, at most 10,000
4//! records or 64 MiB) and reports what a user needs to configure a run — alphabet, alignment
5//! length, default k, header fields with sample values, compression, size and an estimated
6//! record count — in well under a second. It validates nothing beyond what it reads; the run
7//! itself is the validator (fail-fast, with the exact location of the first problem).
8
9use std::sync::Arc;
10use std::sync::atomic::AtomicBool;
11use std::time::SystemTime;
12
13use crate::alphabet::{LetterCounts, guess};
14use crate::config::Alphabet;
15use crate::control::{ProbeFacts, Progress};
16use crate::header::{CleanStats, clean_value};
17use crate::input::{self, RawBatch, Source};
18use crate::model::Compression;
19use crate::normalise::count_columns;
20
21/// Letter counts of the sequences in a batch (the alphabet probe).
22///
23/// Sequential on purpose: the pre-check runs outside any analysis thread pool, where rayon
24/// would fall back to its global pool, which ignores the chosen thread count and, once the
25/// system refuses one of its threads, stays unusable for the rest of the process. One batch
26/// (at most 64 MiB) takes tens of milliseconds as a byte histogram.
27#[must_use]
28pub fn probe_letters(batch: &RawBatch) -> LetterCounts {
29    let mut histogram = [0u64; 256];
30    for i in 0..batch.len() {
31        for &b in batch.sequence(i).0 {
32            histogram[usize::from(b)] += 1;
33        }
34    }
35    LetterCounts::from_histogram(&histogram)
36}
37
38/// What the first batch shows (D108): the same facts for the pre-check and the run, from this
39/// one function, so that a difference can only mean the input changed in between.
40#[must_use]
41pub fn probe_facts(first: &RawBatch) -> ProbeFacts {
42    ProbeFacts {
43        records: first.len() as u64,
44        letters: probe_letters(first),
45        alignment_length: if first.is_empty() {
46            0
47        } else {
48            count_columns(first.sequence(0).0)
49        },
50        header_fields: if first.is_empty() {
51            0
52        } else {
53            memchr::memchr_iter(b'|', first.header(0)).count() + 1
54        },
55    }
56}
57
58/// One `|`-separated header field seen in the probe.
59#[derive(Clone, Debug, PartialEq, Eq)]
60pub struct HeaderField {
61    /// Up to three distinct cleaned sample values, in input order.
62    pub samples: Vec<String>,
63    /// `true` when every probed record had a different value (likely an identifier).
64    pub identifier_like: bool,
65}
66
67/// What the pre-check found.
68#[derive(Clone, Debug, PartialEq)]
69pub struct Inspection {
70    /// Input name for display.
71    pub source_name: String,
72    /// Compression detected from the content.
73    pub compression: Compression,
74    /// Stored size (regular files).
75    pub size_bytes: Option<u64>,
76    /// Modification time (regular files), to detect changes before the run (D85, G65).
77    pub modified: Option<SystemTime>,
78    /// Records examined.
79    pub records_examined: u64,
80    /// `true` when the probe read the whole input (the record count is exact).
81    pub complete: bool,
82    /// Estimated number of records (exact when `complete`), extrapolated from the stored bytes
83    /// consumed per record; `None` for streams of unknown size.
84    pub estimated_records: Option<u64>,
85    /// Alignment columns of the first record.
86    pub alignment_length: usize,
87    /// Alphabet detected from the probe, if it can be determined.
88    pub alphabet: Option<Alphabet>,
89    /// Residue letters examined.
90    pub residues_examined: u64,
91    /// The query name the file name implies.
92    pub suggested_query_name: Option<String>,
93    /// Fields of the first header (split on `|`).
94    pub header_fields: Vec<HeaderField>,
95    /// Probed records whose headers have a different number of fields than the first.
96    pub header_field_count_mismatches: u64,
97    /// The facts a run checks before counting, to refuse a file that changed since (D108).
98    pub probe: ProbeFacts,
99}
100
101/// Reads the first batch of `source` and describes it (D85).
102///
103/// # Errors
104/// The errors of [`input::open`] and of reading the first batch (I/O, unsupported or corrupt
105/// compression, structural FASTA errors).
106pub fn inspect(source: &Source) -> crate::Result<Inspection> {
107    let progress = Arc::new(Progress::new());
108    let mut opened = input::open(source, &progress)?;
109    let mut batch = RawBatch::default();
110    let cancel = AtomicBool::new(false);
111    opened.batches.read_batch(&mut batch, &cancel)?;
112    let complete = opened.batches.is_finished();
113    let probe = probe_facts(&batch);
114    let records = probe.records;
115    let estimated_records = if complete {
116        Some(records)
117    } else {
118        estimate_records(&batch, opened.compression, opened.stored_size, &progress)
119    };
120    let (header_fields, mismatches) = header_fields(&batch);
121    Ok(Inspection {
122        source_name: source.display_name(),
123        compression: opened.compression,
124        size_bytes: opened.stored_size,
125        modified: opened.modified,
126        records_examined: records,
127        complete,
128        estimated_records,
129        alignment_length: probe.alignment_length,
130        alphabet: guess(&probe.letters),
131        residues_examined: probe.letters.total(),
132        suggested_query_name: input::query_name_from_source(source),
133        header_fields,
134        header_field_count_mismatches: mismatches,
135        probe,
136    })
137}
138
139/// Extrapolates the record count from the first batch: from the text consumed per record for
140/// plain files (exact offsets), from the stored bytes read per record for compressed files
141/// (approximate: decoders read ahead).
142fn estimate_records(
143    batch: &RawBatch,
144    compression: Compression,
145    stored_size: Option<u64>,
146    progress: &Progress,
147) -> Option<u64> {
148    let size = stored_size?;
149    let records = batch.len() as u64;
150    let last = batch.records.last()?;
151    let consumed = if compression == Compression::None {
152        last.byte + last.range.len() as u64
153    } else {
154        progress.snapshot().bytes_read
155    };
156    if consumed == 0 {
157        return None;
158    }
159    #[allow(
160        clippy::cast_possible_truncation,
161        clippy::cast_sign_loss,
162        clippy::cast_precision_loss
163    )]
164    let estimate = (size as f64 * records as f64 / consumed as f64).round() as u64;
165    Some(estimate.max(records))
166}
167
168fn header_fields(batch: &RawBatch) -> (Vec<HeaderField>, u64) {
169    let headers: Vec<String> = (0..batch.len())
170        .map(|i| String::from_utf8_lossy(batch.header(i)).into_owned())
171        .collect();
172    let first: Vec<&str> = headers[0].split('|').collect();
173    let mut columns: Vec<Vec<String>> = vec![Vec::new(); first.len()];
174    let mut mismatches = 0u64;
175    let mut stats = CleanStats::default();
176    for h in &headers {
177        let parts: Vec<&str> = h.split('|').collect();
178        if parts.len() != first.len() {
179            mismatches += 1;
180            continue;
181        }
182        for (c, p) in columns.iter_mut().zip(parts) {
183            c.push(clean_value(p, "", &mut stats));
184        }
185    }
186    let fields = columns
187        .into_iter()
188        .map(|values| {
189            let mut samples: Vec<String> = Vec::with_capacity(3);
190            for v in &values {
191                if samples.len() == 3 {
192                    break;
193                }
194                if !samples.contains(v) {
195                    samples.push(v.clone());
196                }
197            }
198            let mut distinct: Vec<&String> = values.iter().collect();
199            distinct.sort_unstable();
200            distinct.dedup();
201            HeaderField {
202                samples,
203                identifier_like: values.len() >= crate::config::ID_DIAG_MIN_RECORDS
204                    && distinct.len() == values.len(),
205            }
206        })
207        .collect();
208    (fields, mismatches)
209}
210
211#[cfg(test)]
212mod tests {
213    use super::*;
214    use std::io::Write;
215
216    #[test]
217    fn inspect_mers_sample() {
218        let p = std::path::PathBuf::from(concat!(
219            env!("CARGO_MANIFEST_DIR"),
220            "/../../samples/mers_spike_aa.fasta"
221        ));
222        let i = inspect(&Source::Path(p)).unwrap();
223        assert_eq!(i.alphabet, Some(Alphabet::Protein));
224        assert_eq!(i.alignment_length, 1353);
225        assert!(i.complete);
226        assert_eq!(i.records_examined, 191);
227        assert_eq!(i.estimated_records, Some(191));
228        assert_eq!(i.compression, Compression::None);
229        assert_eq!(i.suggested_query_name.as_deref(), Some("mers_spike_aa"));
230        assert_eq!(i.header_fields.len(), 4);
231        assert!((1..=3).contains(&i.header_fields[1].samples.len()));
232        assert!(
233            i.header_fields[0].identifier_like,
234            "accessions are all distinct"
235        );
236        assert!(!i.header_fields[1].identifier_like);
237    }
238
239    #[test]
240    fn inspect_estimates_large_inputs() {
241        let dir = tempfile::tempdir().unwrap();
242        let p = dir.path().join("big.fa");
243        let mut f = std::fs::File::create(&p).unwrap();
244        for i in 0..25_000 {
245            writeln!(f, ">s{i}|x\nACGTACGTAC").unwrap();
246        }
247        drop(f);
248        let i = inspect(&Source::Path(p)).unwrap();
249        assert!(!i.complete);
250        assert_eq!(i.records_examined, 10_000);
251        let est = i.estimated_records.unwrap();
252        assert!((20_000..=30_000).contains(&est), "{est}");
253        assert_eq!(i.alphabet, Some(Alphabet::Dna));
254        assert_eq!(i.alignment_length, 10);
255    }
256
257    #[test]
258    fn inspect_reports_mismatched_headers_and_undetermined_alphabet() {
259        let dir = tempfile::tempdir().unwrap();
260        let p = dir.path().join("h.fa");
261        std::fs::write(&p, ">a|b|c\nAC\n>d|e\nAC\n").unwrap();
262        let i = inspect(&Source::Path(p)).unwrap();
263        assert_eq!(i.header_fields.len(), 3);
264        assert_eq!(i.header_field_count_mismatches, 1);
265        assert_eq!(i.alphabet, None, "4 letters are not enough");
266        assert_eq!(i.residues_examined, 4);
267    }
268}