1use std::sync::Arc;
9use std::sync::atomic::{AtomicBool, Ordering};
10use std::sync::mpsc;
11use std::thread;
12use std::time::{Duration, Instant};
13
14use rayon::prelude::*;
15
16use crate::alphabet::{self, ByteClassTable};
17use crate::config::{AnalysisParams, Origin, Param, ValidationMode};
18use crate::control::{Phase, RunControl};
19use crate::diagnostics::Tally;
20use crate::engine::Engine;
21use crate::error::{DimaError, Result};
22use crate::header::{self, HeaderSpec, HeaderStats, ValueDictionary};
23use crate::input::{self, RawBatch, Source};
24use crate::model::{AlphabetMethod, Parameters, Results, RunInfo, SCHEMA_VERSION, ToolInfo};
25use crate::normalise::{NormalisedBatch, Normaliser, SequenceStats};
26use crate::pool;
27use crate::position::Finaliser;
28use crate::provenance;
29use crate::summary;
30use crate::validation::probe_facts;
31
32const CANCEL_POLL: Duration = Duration::from_millis(50);
34
35#[derive(Clone, Debug, Default, PartialEq, Eq)]
37pub struct Execution {
38 pub threads: usize,
40 pub threads_requested: usize,
42 pub phases: Vec<(Phase, Duration)>,
44 pub waited_for_input: Duration,
47 pub records: u64,
49 pub positions: u64,
51}
52
53#[derive(Clone, Debug)]
55pub struct Analysis {
56 pub results: Results,
58 pub execution: Execution,
60}
61
62pub fn analyze(source: &Source, params: &AnalysisParams, control: &RunControl) -> Result<Analysis> {
68 params.validate()?;
69 let spec = match ¶ms.header_format {
70 Some(format) => Some(HeaderSpec::new(
71 format,
72 params.metadata_fields.as_deref(),
73 ¶ms.header_fill().value,
74 )?),
75 None => None,
76 };
77 let requested = control.thread_count().get();
78 Run::new(source, params, spec.as_ref(), control, requested).execute()
79}
80
81struct Run<'a> {
82 source: &'a Source,
83 params: &'a AnalysisParams,
84 spec: Option<&'a HeaderSpec>,
85 control: &'a RunControl,
86 requested: usize,
88 threads: usize,
90 phases: Vec<(Phase, Duration)>,
91 phase_start: Instant,
92}
93
94struct Counting {
96 engine: Engine,
97 table: ByteClassTable,
98 resolution: alphabet::Resolution,
99 width: usize,
100 k: usize,
101 probe_records: u64,
102 records: u64,
103 sequence_stats: SequenceStats,
104 header_stats: HeaderStats,
105 dictionary: ValueDictionary,
106 identifier: Option<crate::diagnostics::Diagnostic>,
107 waited: Duration,
108}
109
110impl<'a> Run<'a> {
111 fn new(
112 source: &'a Source,
113 params: &'a AnalysisParams,
114 spec: Option<&'a HeaderSpec>,
115 control: &'a RunControl,
116 requested: usize,
117 ) -> Self {
118 Self {
119 source,
120 params,
121 spec,
122 control,
123 requested,
124 threads: requested,
125 phases: Vec::new(),
126 phase_start: Instant::now(),
127 }
128 }
129
130 fn enter(&mut self, phase: Phase) {
131 let now = Instant::now();
132 if let Some(last) = self.phases.last_mut() {
133 last.1 = now - self.phase_start;
134 }
135 self.control.progress.set_phase(phase);
136 self.phases.push((phase, Duration::ZERO));
137 self.phase_start = now;
138 }
139
140 fn execute(mut self) -> Result<Analysis> {
141 self.enter(Phase::Opening);
142 let thread_start = self.control.thread_start.clone();
143 let opened = input::open_with(self.source, &self.control.progress, thread_start.as_ref())?;
144 let input::Opened {
145 compression,
146 batches,
147 checksum,
148 ..
149 } = opened;
150 let stop = Arc::new(AtomicBool::new(false));
151 let (batch_tx, batch_rx) = mpsc::sync_channel::<Result<RawBatch>>(0);
152 let (recycle_tx, recycle_rx) = mpsc::channel::<RawBatch>();
153 let reader_stop = Arc::clone(&stop);
154 let limits = self.control.batch_limits;
155 let cancel = Arc::clone(&self.control.cancel);
156 let reader_start = thread_start.clone();
159 let reader = thread::Builder::new()
160 .name("dima-reader".into())
161 .spawn(move || {
162 if let Some(h) = &reader_start {
163 h.call();
164 }
165 let mut batches = batches.with_limits(limits);
166 loop {
167 if reader_stop.load(Ordering::Relaxed) || cancel.load(Ordering::Relaxed) {
168 break;
169 }
170 let mut b = recycle_rx.try_recv().unwrap_or_default();
171 match batches.read_batch(&mut b, &reader_stop) {
172 Ok(true) => {
173 if batch_tx.send(Ok(b)).is_err() {
174 break;
175 }
176 }
177 Ok(false) => break,
178 Err(e) => {
179 let _ = batch_tx.send(Err(e));
180 break;
181 }
182 }
183 }
184 })
185 .map_err(|e| DimaError::system("the reader thread", e))?;
186 let pool = match pool::build(self.requested, thread_start) {
189 Ok(fitted) => {
190 self.threads = fitted.started;
191 self.control.progress.set_threads_started(fitted.started);
192 fitted.pool
193 }
194 Err(e) => {
195 stop.store(true, Ordering::Relaxed);
196 return Err(e);
197 }
198 };
199 if let Some(h) = &self.control.threads_ready {
200 h.call();
201 }
202 pool.install(move || {
203 let counting = match self.count(&batch_rx, &recycle_tx) {
204 Ok(c) => c,
205 Err(e) => {
206 stop.store(true, Ordering::Relaxed);
207 return Err(e);
208 }
209 };
210 drop(batch_rx);
211 reader
212 .join()
213 .map_err(|_| DimaError::internal("the reader thread panicked"))?;
214 let (checksum, size_bytes) = checksum.finish()?;
215 let input_info = crate::model::InputInfo {
216 name: self.source.provenance_name(),
217 size_bytes,
218 checksum,
219 compression,
220 };
221 self.finish(counting, input_info)
222 })
223 }
224
225 fn next_batch(
228 rx: &mpsc::Receiver<Result<RawBatch>>,
229 waited: &mut Duration,
230 control: &RunControl,
231 ) -> Result<Option<RawBatch>> {
232 let t = Instant::now();
233 let r = loop {
234 match rx.recv_timeout(CANCEL_POLL) {
235 Ok(r) => break Some(r),
236 Err(mpsc::RecvTimeoutError::Timeout) => {
237 if let Err(e) = control.check() {
238 *waited += t.elapsed();
239 return Err(e);
240 }
241 }
242 Err(mpsc::RecvTimeoutError::Disconnected) => break None,
243 }
244 };
245 *waited += t.elapsed();
246 match r {
247 Some(Ok(b)) => Ok(Some(b)),
248 Some(Err(e)) => Err(e),
249 None => Ok(None),
250 }
251 }
252
253 fn count(
254 &mut self,
255 rx: &mpsc::Receiver<Result<RawBatch>>,
256 recycle: &mpsc::Sender<RawBatch>,
257 ) -> Result<Counting> {
258 let control = self.control;
259 let mut waited = Duration::ZERO;
260 let first = Self::next_batch(rx, &mut waited, control)?.ok_or_else(|| {
261 control
262 .check()
263 .err()
264 .unwrap_or_else(|| DimaError::internal("the reader stopped before the first batch"))
265 })?;
266 control.check()?;
267 let probe = probe_facts(&first);
270 if control.expected_probe.as_ref().is_some_and(|e| *e != probe) {
271 return Err(DimaError::InputChanged {
272 source_name: self.source.display_name(),
273 });
274 }
275 let probe_records = probe.records;
276 let source_name = self.source.display_name();
277 if let (0, Some(r)) = (probe.alignment_length, first.records.first()) {
279 return Err(crate::normalise::empty_sequence(
280 &source_name,
281 first.first_record,
282 r.line,
283 ));
284 }
285 let resolution = alphabet::resolve(self.params.alphabet, &probe.letters, probe_records)
286 .map_err(|e| e.in_source(&source_name))?;
287 let table = ByteClassTable::new(resolution.alphabet);
288 let width = probe.alignment_length;
289 let mode = self.params.validation().value;
290 let strict = mode == ValidationMode::Strict;
291 let normaliser = Normaliser {
292 table: &table,
293 mode,
294 width,
295 header: self.spec,
296 source: &source_name,
297 };
298 let mut norm = NormalisedBatch::default();
299 normaliser.normalise(&first, &mut norm)?;
300 let k = self.params.kmer_length(resolution.alphabet).value;
301 let strata = self.spec.map_or(0, |s| s.stratified().len());
302 let mut engine = Engine::new(width, k, strata, self.threads)?;
303 tracing::debug!(
304 alphabet = %resolution.alphabet,
305 method = ?resolution.method,
306 probe_records,
307 alignment_length = width,
308 k,
309 positions = engine.positions(),
310 shards = engine.shard_count(),
311 threads = self.threads,
312 strata,
313 "probe done; counting"
314 );
315 control
316 .progress
317 .set_positions_total(engine.positions() as u64);
318 self.enter(Phase::Counting);
319 let identifier = self.spec.and_then(|spec| {
320 let mut columns: Vec<Vec<String>> = vec![Vec::with_capacity(norm.records); strata];
321 for values in &norm.header_values {
322 for (c, v) in columns.iter_mut().zip(values) {
323 c.push(v.clone());
324 }
325 }
326 header::identifier_diagnostic(spec, &columns, probe_records)
327 });
328 let mut records = 0u64;
329 let mut sequence_stats = SequenceStats::default();
330 let mut header_stats = HeaderStats::default();
331 let mut dictionary = ValueDictionary::new(strata);
332 let mut values: Vec<u32> = Vec::new();
333 let mut current = Some(first);
334 let mut normalised = true;
335 while let Some(batch) = current.take() {
336 control.check()?;
337 if !normalised {
338 normaliser.normalise(&batch, &mut norm)?;
339 }
340 normalised = false;
341 values.clear();
342 for record in norm.header_values.drain(..) {
343 for (f, v) in record.into_iter().enumerate() {
344 values.push(dictionary.intern(f, v)?);
345 }
346 }
347 header_stats.merge(std::mem::take(&mut norm.header_stats));
348 if !(strict && header_stats.mismatch_count > 0) {
351 engine.add_batch(&norm.rows, norm.records, &values, &control.cancel)?;
352 }
353 records = records
354 .checked_add(norm.records as u64)
355 .ok_or(DimaError::Overflow { what: "records" })?;
356 control.progress.add_records(norm.records as u64);
357 sequence_stats.merge(std::mem::take(&mut norm.sequence_stats));
358 let _ = recycle.send(batch);
359 current = Self::next_batch(rx, &mut waited, control)?;
360 }
361 control.check()?;
362 tracing::debug!(
363 records,
364 waited_for_input_ms = waited.as_millis(),
365 distinct_header_values = dictionary.len(),
366 "input read"
367 );
368 if let Some(spec) = self.spec {
369 if strict && header_stats.mismatch_count > 0 {
370 return Err(header::mismatch_error(&source_name, spec, &header_stats));
371 }
372 }
373 Ok(Counting {
374 engine,
375 table,
376 resolution,
377 width,
378 k,
379 probe_records,
380 records,
381 sequence_stats,
382 header_stats,
383 dictionary,
384 identifier,
385 waited,
386 })
387 }
388
389 fn finish(mut self, c: Counting, input_info: crate::model::InputInfo) -> Result<Analysis> {
390 self.enter(Phase::Finalising);
391 let params = self.params;
392 let threshold = params.support_threshold();
393 let estimator = params.estimator();
394 let validation = params.validation();
395 let alphabet = c.resolution.alphabet;
396 let strata_names = self
397 .spec
398 .map(HeaderSpec::stratified_names)
399 .unwrap_or_default();
400 let values: Vec<(usize, String)> = (0..c.dictionary.len())
401 .map(|id| {
402 #[allow(clippy::cast_possible_truncation)] let (f, v, _) = c.dictionary.entry(id as u32);
404 (f, v.to_owned())
405 })
406 .collect();
407 let finaliser = Finaliser {
408 threshold: threshold.value,
409 estimator: estimator.value,
410 min_coverage: params.min_coverage,
411 fields: &strata_names,
412 values: &values,
413 };
414 let progress = &self.control.progress;
415 let cancel = &self.control.cancel;
416 let counts = c.engine.finish();
417 let positions: Vec<crate::model::PositionResult> = counts
418 .into_par_iter()
419 .enumerate()
420 .map(|(i, pc)| {
421 if cancel.load(Ordering::Relaxed) {
422 return Err(DimaError::Cancelled);
423 }
424 let r = finaliser.finalise(i as u64 + 1, &pc);
425 progress.add_positions_done(1);
426 r
427 })
428 .collect::<Result<_>>()?;
429 self.control.check()?;
430
431 let label_counts = summary::label_counts(&positions);
432 let (average_entropy, lowest_entropy, highest_entropy) = summary::eligible(&positions);
433 let mut below = Tally::default();
434 for p in positions
435 .iter()
436 .filter(|p| p.below_min_coverage == Some(true))
437 {
438 let coverage = p.sample_coverage.unwrap_or_default();
439 below.add(1, || {
440 format!("position {} (coverage {coverage:.4})", p.position)
441 });
442 }
443 let value_diagnostics = self
444 .spec
445 .map(|spec| header::value_diagnostics(spec, &c.dictionary))
446 .unwrap_or_default();
447 let diagnostics = provenance::diagnostics(provenance::DiagnosticInputs {
448 resolution: &c.resolution,
449 sequences: &c.sequence_stats,
450 headers: self.spec.map(|s| (s, &c.header_stats)),
451 value_diagnostics,
452 identifier: c.identifier.clone(),
453 sequence_count: c.records,
454 threshold: threshold.value,
455 eligible_positions: label_counts.eligible,
456 below_min_coverage: params.min_coverage.map(|m| (m, below)),
457 fill_without_format: params.header_fill.is_some() && self.spec.is_none(),
458 });
459
460 let query_name = match ¶ms.query_name {
461 Some(q) => Param {
462 value: q.trim().to_owned(),
463 origin: Origin::User,
464 },
465 None => match input::query_name_from_source(self.source) {
466 Some(stem) => Param {
467 value: stem,
468 origin: Origin::Derived,
469 },
470 None => Param {
471 value: format!("Unknown {}", alphabet.display_name()),
472 origin: Origin::Default,
473 },
474 },
475 };
476 let user_or_default = |given: bool| if given { Origin::User } else { Origin::Default };
477 let parameters = Parameters {
478 alphabet: Param {
479 value: alphabet,
480 origin: if c.resolution.method == AlphabetMethod::User {
481 Origin::User
482 } else {
483 Origin::Derived
484 },
485 },
486 kmer_length: {
487 let k = params.kmer_length(alphabet);
488 Param {
489 value: k.value as u64,
490 origin: k.origin,
491 }
492 },
493 support_threshold: threshold.clone(),
494 estimator: estimator.clone(),
495 validation: validation.clone(),
496 min_coverage: Param {
497 value: params.min_coverage,
498 origin: user_or_default(params.min_coverage.is_some()),
499 },
500 query_name: query_name.clone(),
501 header_format: Param {
502 value: self.spec.map(|s| s.fields().to_vec()),
503 origin: user_or_default(self.spec.is_some()),
504 },
505 metadata_fields: Param {
506 value: self.spec.map(HeaderSpec::stratified_names),
507 origin: user_or_default(params.metadata_fields.is_some()),
508 },
509 header_fill: {
511 let fill = params.header_fill();
512 Param {
513 value: self
514 .spec
515 .map_or_else(|| header::clean_name(&fill.value), |s| s.fill().to_owned()),
516 origin: fill.origin,
517 }
518 },
519 };
520 let run = RunInfo {
521 tool: ToolInfo {
522 name: "dima".to_owned(),
523 version: crate::citations::SOFTWARE_VERSION.to_owned(),
524 },
525 parameters,
526 alphabet: provenance::alphabet_info(
527 &c.resolution,
528 c.probe_records,
529 c.sequence_stats.t_u_folded,
530 ),
531 alignment_length: c.width as u64,
532 input: input_info,
533 validation: provenance::validation_info(
534 validation.value,
535 c.records,
536 &c.table,
537 &c.sequence_stats,
538 self.spec.map(|_| &c.header_stats),
539 ),
540 metadata: self
541 .spec
542 .map(|s| provenance::metadata_info(s, &c.header_stats, c.probe_records)),
543 entropy: provenance::entropy_info(estimator.value),
544 diagnostics,
545 references: provenance::references(
546 estimator.value,
547 alphabet,
548 c.resolution.method == AlphabetMethod::Detected,
549 ),
550 };
551 let results = Results {
552 schema_version: SCHEMA_VERSION.to_owned(),
553 query_name: query_name.value,
554 sequence_count: c.records,
555 support_threshold: threshold.value,
556 kmer_length: c.k as u64,
557 low_support_count: label_counts.no_support
558 + label_counts.low
559 + label_counts.equal_to_threshold,
560 average_entropy,
561 lowest_entropy,
562 highest_entropy,
563 position_count: positions.len() as u64,
564 eligible_position_count: label_counts.eligible,
565 support_label_counts: label_counts,
566 all_positions_summary: summary::all_positions(&positions),
567 coverage_summary: summary::coverage(&positions, params.min_coverage),
568 run,
569 results: positions,
570 };
571 self.enter(Phase::Done);
572 let execution = Execution {
573 threads: self.threads,
574 threads_requested: self.requested,
575 phases: self.phases[..self.phases.len() - 1].to_vec(),
576 waited_for_input: c.waited,
577 records: c.records,
578 positions: results.position_count,
579 };
580 Ok(Analysis { results, execution })
581 }
582}