Skip to main content

dima_core/
analysis.rs

1//! Orchestration of an analysis: input → probe → normalisation → counting → statistics →
2//! summaries and provenance. Each step lives in its own module; this one only wires them.
3//!
4//! Reading runs on its own thread, one batch ahead of normalisation and counting, which run in
5//! parallel on a local rayon pool sized by [`RunControl::thread_count`], or smaller when the
6//! system refuses some of its threads (D110). At most two batches are in memory at a time.
7
8use std::sync::Arc;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::mpsc;
11use std::thread;
12use std::time::{Duration, Instant};
13
14use rayon::prelude::*;
15
16use crate::alphabet::{self, ByteClassTable};
17use crate::config::{AnalysisParams, Origin, Param, ValidationMode};
18use crate::control::{Phase, RunControl};
19use crate::diagnostics::Tally;
20use crate::engine::Engine;
21use crate::error::{DimaError, Result};
22use crate::header::{self, HeaderSpec, HeaderStats, ValueDictionary};
23use crate::input::{self, RawBatch, Source};
24use crate::model::{AlphabetMethod, Parameters, Results, RunInfo, SCHEMA_VERSION, ToolInfo};
25use crate::normalise::{NormalisedBatch, Normaliser, SequenceStats};
26use crate::pool;
27use crate::position::Finaliser;
28use crate::provenance;
29use crate::summary;
30use crate::validation::probe_facts;
31
32/// How often a wait for input checks for cancellation.
33const CANCEL_POLL: Duration = Duration::from_millis(50);
34
35/// Execution facts of a run (not part of the deterministic results).
36#[derive(Clone, Debug, Default, PartialEq, Eq)]
37pub struct Execution {
38    /// Worker threads used: those requested, or fewer when the system refused some (D110).
39    pub threads: usize,
40    /// Worker threads requested ([`RunControl::thread_count`]).
41    pub threads_requested: usize,
42    /// Wall time per phase, in order.
43    pub phases: Vec<(Phase, Duration)>,
44    /// Time the counting loop waited for the reader (high values mean reading or
45    /// decompression is the bottleneck).
46    pub waited_for_input: Duration,
47    /// Records read.
48    pub records: u64,
49    /// Positions analysed.
50    pub positions: u64,
51}
52
53/// The outcome of [`analyze`].
54#[derive(Clone, Debug)]
55pub struct Analysis {
56    /// The deterministic results.
57    pub results: Results,
58    /// How the run went.
59    pub execution: Execution,
60}
61
62/// Analyses `source` with `params`.
63///
64/// # Errors
65/// Usage, I/O, input-format and input-data errors with their locations; [`DimaError::Cancelled`]
66/// when `control.cancel` is set.
67pub fn analyze(source: &Source, params: &AnalysisParams, control: &RunControl) -> Result<Analysis> {
68    params.validate()?;
69    let spec = match &params.header_format {
70        Some(format) => Some(HeaderSpec::new(
71            format,
72            params.metadata_fields.as_deref(),
73            &params.header_fill().value,
74        )?),
75        None => None,
76    };
77    let requested = control.thread_count().get();
78    Run::new(source, params, spec.as_ref(), control, requested).execute()
79}
80
81struct Run<'a> {
82    source: &'a Source,
83    params: &'a AnalysisParams,
84    spec: Option<&'a HeaderSpec>,
85    control: &'a RunControl,
86    /// Worker threads requested.
87    requested: usize,
88    /// Worker threads started (D110); `requested` until the pool is built.
89    threads: usize,
90    phases: Vec<(Phase, Duration)>,
91    phase_start: Instant,
92}
93
94/// Everything accumulated while reading.
95struct Counting {
96    engine: Engine,
97    table: ByteClassTable,
98    resolution: alphabet::Resolution,
99    width: usize,
100    k: usize,
101    probe_records: u64,
102    records: u64,
103    sequence_stats: SequenceStats,
104    header_stats: HeaderStats,
105    dictionary: ValueDictionary,
106    identifier: Option<crate::diagnostics::Diagnostic>,
107    waited: Duration,
108}
109
110impl<'a> Run<'a> {
111    fn new(
112        source: &'a Source,
113        params: &'a AnalysisParams,
114        spec: Option<&'a HeaderSpec>,
115        control: &'a RunControl,
116        requested: usize,
117    ) -> Self {
118        Self {
119            source,
120            params,
121            spec,
122            control,
123            requested,
124            threads: requested,
125            phases: Vec::new(),
126            phase_start: Instant::now(),
127        }
128    }
129
130    fn enter(&mut self, phase: Phase) {
131        let now = Instant::now();
132        if let Some(last) = self.phases.last_mut() {
133            last.1 = now - self.phase_start;
134        }
135        self.control.progress.set_phase(phase);
136        self.phases.push((phase, Duration::ZERO));
137        self.phase_start = now;
138    }
139
140    fn execute(mut self) -> Result<Analysis> {
141        self.enter(Phase::Opening);
142        let thread_start = self.control.thread_start.clone();
143        let opened = input::open_with(self.source, &self.control.progress, thread_start.as_ref())?;
144        let input::Opened {
145            compression,
146            batches,
147            checksum,
148            ..
149        } = opened;
150        let stop = Arc::new(AtomicBool::new(false));
151        let (batch_tx, batch_rx) = mpsc::sync_channel::<Result<RawBatch>>(0);
152        let (recycle_tx, recycle_rx) = mpsc::channel::<RawBatch>();
153        let reader_stop = Arc::clone(&stop);
154        let limits = self.control.batch_limits;
155        let cancel = Arc::clone(&self.control.cancel);
156        // Detached on purpose: a reader blocked on a slow pipe must never delay an error or a
157        // cancellation (D96). It exits at the next record once `stop` is set.
158        let reader_start = thread_start.clone();
159        let reader = thread::Builder::new()
160            .name("dima-reader".into())
161            .spawn(move || {
162                if let Some(h) = &reader_start {
163                    h.call();
164                }
165                let mut batches = batches.with_limits(limits);
166                loop {
167                    if reader_stop.load(Ordering::Relaxed) || cancel.load(Ordering::Relaxed) {
168                        break;
169                    }
170                    let mut b = recycle_rx.try_recv().unwrap_or_default();
171                    match batches.read_batch(&mut b, &reader_stop) {
172                        Ok(true) => {
173                            if batch_tx.send(Ok(b)).is_err() {
174                                break;
175                            }
176                        }
177                        Ok(false) => break,
178                        Err(e) => {
179                            let _ = batch_tx.send(Err(e));
180                            break;
181                        }
182                    }
183                }
184            })
185            .map_err(|e| DimaError::system("the reader thread", e))?;
186        // The worker pool comes after the threads the run cannot do without (checksum, reader),
187        // so a system short of threads runs fewer workers instead of refusing those (D110).
188        let pool = match pool::build(self.requested, thread_start) {
189            Ok(fitted) => {
190                self.threads = fitted.started;
191                self.control.progress.set_threads_started(fitted.started);
192                fitted.pool
193            }
194            Err(e) => {
195                stop.store(true, Ordering::Relaxed);
196                return Err(e);
197            }
198        };
199        if let Some(h) = &self.control.threads_ready {
200            h.call();
201        }
202        pool.install(move || {
203            let counting = match self.count(&batch_rx, &recycle_tx) {
204                Ok(c) => c,
205                Err(e) => {
206                    stop.store(true, Ordering::Relaxed);
207                    return Err(e);
208                }
209            };
210            drop(batch_rx);
211            reader
212                .join()
213                .map_err(|_| DimaError::internal("the reader thread panicked"))?;
214            let (checksum, size_bytes) = checksum.finish()?;
215            let input_info = crate::model::InputInfo {
216                name: self.source.provenance_name(),
217                size_bytes,
218                checksum,
219                compression,
220            };
221            self.finish(counting, input_info)
222        })
223    }
224
225    /// The next batch, `None` at the end of the input. Waiting for a slow pipe still notices a
226    /// cancellation within [`CANCEL_POLL`].
227    fn next_batch(
228        rx: &mpsc::Receiver<Result<RawBatch>>,
229        waited: &mut Duration,
230        control: &RunControl,
231    ) -> Result<Option<RawBatch>> {
232        let t = Instant::now();
233        let r = loop {
234            match rx.recv_timeout(CANCEL_POLL) {
235                Ok(r) => break Some(r),
236                Err(mpsc::RecvTimeoutError::Timeout) => {
237                    if let Err(e) = control.check() {
238                        *waited += t.elapsed();
239                        return Err(e);
240                    }
241                }
242                Err(mpsc::RecvTimeoutError::Disconnected) => break None,
243            }
244        };
245        *waited += t.elapsed();
246        match r {
247            Some(Ok(b)) => Ok(Some(b)),
248            Some(Err(e)) => Err(e),
249            None => Ok(None),
250        }
251    }
252
253    fn count(
254        &mut self,
255        rx: &mpsc::Receiver<Result<RawBatch>>,
256        recycle: &mpsc::Sender<RawBatch>,
257    ) -> Result<Counting> {
258        let control = self.control;
259        let mut waited = Duration::ZERO;
260        let first = Self::next_batch(rx, &mut waited, control)?.ok_or_else(|| {
261            control
262                .check()
263                .err()
264                .unwrap_or_else(|| DimaError::internal("the reader stopped before the first batch"))
265        })?;
266        control.check()?;
267        // Probe (D40): alphabet and alignment width from the first batch, computed like the
268        // pre-check's (D108), which a front end may ask to match.
269        let probe = probe_facts(&first);
270        if control.expected_probe.as_ref().is_some_and(|e| *e != probe) {
271            return Err(DimaError::InputChanged {
272                source_name: self.source.display_name(),
273            });
274        }
275        let probe_records = probe.records;
276        let source_name = self.source.display_name();
277        // An empty first record leaves nothing to detect the alphabet from: report the record.
278        if let (0, Some(r)) = (probe.alignment_length, first.records.first()) {
279            return Err(crate::normalise::empty_sequence(
280                &source_name,
281                first.first_record,
282                r.line,
283            ));
284        }
285        let resolution = alphabet::resolve(self.params.alphabet, &probe.letters, probe_records)
286            .map_err(|e| e.in_source(&source_name))?;
287        let table = ByteClassTable::new(resolution.alphabet);
288        let width = probe.alignment_length;
289        let mode = self.params.validation().value;
290        let strict = mode == ValidationMode::Strict;
291        let normaliser = Normaliser {
292            table: &table,
293            mode,
294            width,
295            header: self.spec,
296            source: &source_name,
297        };
298        let mut norm = NormalisedBatch::default();
299        normaliser.normalise(&first, &mut norm)?;
300        let k = self.params.kmer_length(resolution.alphabet).value;
301        let strata = self.spec.map_or(0, |s| s.stratified().len());
302        let mut engine = Engine::new(width, k, strata, self.threads)?;
303        tracing::debug!(
304            alphabet = %resolution.alphabet,
305            method = ?resolution.method,
306            probe_records,
307            alignment_length = width,
308            k,
309            positions = engine.positions(),
310            shards = engine.shard_count(),
311            threads = self.threads,
312            strata,
313            "probe done; counting"
314        );
315        control
316            .progress
317            .set_positions_total(engine.positions() as u64);
318        self.enter(Phase::Counting);
319        let identifier = self.spec.and_then(|spec| {
320            let mut columns: Vec<Vec<String>> = vec![Vec::with_capacity(norm.records); strata];
321            for values in &norm.header_values {
322                for (c, v) in columns.iter_mut().zip(values) {
323                    c.push(v.clone());
324                }
325            }
326            header::identifier_diagnostic(spec, &columns, probe_records)
327        });
328        let mut records = 0u64;
329        let mut sequence_stats = SequenceStats::default();
330        let mut header_stats = HeaderStats::default();
331        let mut dictionary = ValueDictionary::new(strata);
332        let mut values: Vec<u32> = Vec::new();
333        let mut current = Some(first);
334        let mut normalised = true;
335        while let Some(batch) = current.take() {
336            control.check()?;
337            if !normalised {
338                normaliser.normalise(&batch, &mut norm)?;
339            }
340            normalised = false;
341            values.clear();
342            for record in norm.header_values.drain(..) {
343                for (f, v) in record.into_iter().enumerate() {
344                    values.push(dictionary.intern(f, v)?);
345                }
346            }
347            header_stats.merge(std::mem::take(&mut norm.header_stats));
348            // In strict mode a header mismatch ends the run once all headers are read (D18);
349            // counting the rest would be wasted.
350            if !(strict && header_stats.mismatch_count > 0) {
351                engine.add_batch(&norm.rows, norm.records, &values, &control.cancel)?;
352            }
353            records = records
354                .checked_add(norm.records as u64)
355                .ok_or(DimaError::Overflow { what: "records" })?;
356            control.progress.add_records(norm.records as u64);
357            sequence_stats.merge(std::mem::take(&mut norm.sequence_stats));
358            let _ = recycle.send(batch);
359            current = Self::next_batch(rx, &mut waited, control)?;
360        }
361        control.check()?;
362        tracing::debug!(
363            records,
364            waited_for_input_ms = waited.as_millis(),
365            distinct_header_values = dictionary.len(),
366            "input read"
367        );
368        if let Some(spec) = self.spec {
369            if strict && header_stats.mismatch_count > 0 {
370                return Err(header::mismatch_error(&source_name, spec, &header_stats));
371            }
372        }
373        Ok(Counting {
374            engine,
375            table,
376            resolution,
377            width,
378            k,
379            probe_records,
380            records,
381            sequence_stats,
382            header_stats,
383            dictionary,
384            identifier,
385            waited,
386        })
387    }
388
389    fn finish(mut self, c: Counting, input_info: crate::model::InputInfo) -> Result<Analysis> {
390        self.enter(Phase::Finalising);
391        let params = self.params;
392        let threshold = params.support_threshold();
393        let estimator = params.estimator();
394        let validation = params.validation();
395        let alphabet = c.resolution.alphabet;
396        let strata_names = self
397            .spec
398            .map(HeaderSpec::stratified_names)
399            .unwrap_or_default();
400        let values: Vec<(usize, String)> = (0..c.dictionary.len())
401            .map(|id| {
402                #[allow(clippy::cast_possible_truncation)] // ids are u32.
403                let (f, v, _) = c.dictionary.entry(id as u32);
404                (f, v.to_owned())
405            })
406            .collect();
407        let finaliser = Finaliser {
408            threshold: threshold.value,
409            estimator: estimator.value,
410            min_coverage: params.min_coverage,
411            fields: &strata_names,
412            values: &values,
413        };
414        let progress = &self.control.progress;
415        let cancel = &self.control.cancel;
416        let counts = c.engine.finish();
417        let positions: Vec<crate::model::PositionResult> = counts
418            .into_par_iter()
419            .enumerate()
420            .map(|(i, pc)| {
421                if cancel.load(Ordering::Relaxed) {
422                    return Err(DimaError::Cancelled);
423                }
424                let r = finaliser.finalise(i as u64 + 1, &pc);
425                progress.add_positions_done(1);
426                r
427            })
428            .collect::<Result<_>>()?;
429        self.control.check()?;
430
431        let label_counts = summary::label_counts(&positions);
432        let (average_entropy, lowest_entropy, highest_entropy) = summary::eligible(&positions);
433        let mut below = Tally::default();
434        for p in positions
435            .iter()
436            .filter(|p| p.below_min_coverage == Some(true))
437        {
438            let coverage = p.sample_coverage.unwrap_or_default();
439            below.add(1, || {
440                format!("position {} (coverage {coverage:.4})", p.position)
441            });
442        }
443        let value_diagnostics = self
444            .spec
445            .map(|spec| header::value_diagnostics(spec, &c.dictionary))
446            .unwrap_or_default();
447        let diagnostics = provenance::diagnostics(provenance::DiagnosticInputs {
448            resolution: &c.resolution,
449            sequences: &c.sequence_stats,
450            headers: self.spec.map(|s| (s, &c.header_stats)),
451            value_diagnostics,
452            identifier: c.identifier.clone(),
453            sequence_count: c.records,
454            threshold: threshold.value,
455            eligible_positions: label_counts.eligible,
456            below_min_coverage: params.min_coverage.map(|m| (m, below)),
457            fill_without_format: params.header_fill.is_some() && self.spec.is_none(),
458        });
459
460        let query_name = match &params.query_name {
461            Some(q) => Param {
462                value: q.trim().to_owned(),
463                origin: Origin::User,
464            },
465            None => match input::query_name_from_source(self.source) {
466                Some(stem) => Param {
467                    value: stem,
468                    origin: Origin::Derived,
469                },
470                None => Param {
471                    value: format!("Unknown {}", alphabet.display_name()),
472                    origin: Origin::Default,
473                },
474            },
475        };
476        let user_or_default = |given: bool| if given { Origin::User } else { Origin::Default };
477        let parameters = Parameters {
478            alphabet: Param {
479                value: alphabet,
480                origin: if c.resolution.method == AlphabetMethod::User {
481                    Origin::User
482                } else {
483                    Origin::Derived
484                },
485            },
486            kmer_length: {
487                let k = params.kmer_length(alphabet);
488                Param {
489                    value: k.value as u64,
490                    origin: k.origin,
491                }
492            },
493            support_threshold: threshold.clone(),
494            estimator: estimator.clone(),
495            validation: validation.clone(),
496            min_coverage: Param {
497                value: params.min_coverage,
498                origin: user_or_default(params.min_coverage.is_some()),
499            },
500            query_name: query_name.clone(),
501            header_format: Param {
502                value: self.spec.map(|s| s.fields().to_vec()),
503                origin: user_or_default(self.spec.is_some()),
504            },
505            metadata_fields: Param {
506                value: self.spec.map(HeaderSpec::stratified_names),
507                origin: user_or_default(params.metadata_fields.is_some()),
508            },
509            // The value used: trimmed and NFC-normalised like the field names.
510            header_fill: {
511                let fill = params.header_fill();
512                Param {
513                    value: self
514                        .spec
515                        .map_or_else(|| header::clean_name(&fill.value), |s| s.fill().to_owned()),
516                    origin: fill.origin,
517                }
518            },
519        };
520        let run = RunInfo {
521            tool: ToolInfo {
522                name: "dima".to_owned(),
523                version: crate::citations::SOFTWARE_VERSION.to_owned(),
524            },
525            parameters,
526            alphabet: provenance::alphabet_info(
527                &c.resolution,
528                c.probe_records,
529                c.sequence_stats.t_u_folded,
530            ),
531            alignment_length: c.width as u64,
532            input: input_info,
533            validation: provenance::validation_info(
534                validation.value,
535                c.records,
536                &c.table,
537                &c.sequence_stats,
538                self.spec.map(|_| &c.header_stats),
539            ),
540            metadata: self
541                .spec
542                .map(|s| provenance::metadata_info(s, &c.header_stats, c.probe_records)),
543            entropy: provenance::entropy_info(estimator.value),
544            diagnostics,
545            references: provenance::references(
546                estimator.value,
547                alphabet,
548                c.resolution.method == AlphabetMethod::Detected,
549            ),
550        };
551        let results = Results {
552            schema_version: SCHEMA_VERSION.to_owned(),
553            query_name: query_name.value,
554            sequence_count: c.records,
555            support_threshold: threshold.value,
556            kmer_length: c.k as u64,
557            low_support_count: label_counts.no_support
558                + label_counts.low
559                + label_counts.equal_to_threshold,
560            average_entropy,
561            lowest_entropy,
562            highest_entropy,
563            position_count: positions.len() as u64,
564            eligible_position_count: label_counts.eligible,
565            support_label_counts: label_counts,
566            all_positions_summary: summary::all_positions(&positions),
567            coverage_summary: summary::coverage(&positions, params.min_coverage),
568            run,
569            results: positions,
570        };
571        self.enter(Phase::Done);
572        let execution = Execution {
573            threads: self.threads,
574            threads_requested: self.requested,
575            phases: self.phases[..self.phases.len() - 1].to_vec(),
576            waited_for_input: c.waited,
577            records: c.records,
578            positions: results.position_count,
579        };
580        Ok(Analysis { results, execution })
581    }
582}