1use std::fs::File;
13use std::io::{self, BufReader, Cursor, Read};
14use std::ops::Range;
15use std::path::{Path, PathBuf};
16use std::sync::Arc;
17use std::sync::atomic::{AtomicBool, Ordering};
18use std::sync::mpsc::{self, SyncSender};
19use std::thread;
20use std::time::SystemTime;
21
22use sha2::{Digest, Sha256};
23
24use crate::alphabet;
25use crate::config::{BATCH_MAX_BYTES, BATCH_MAX_RECORDS, PROBE_MIN_RECORDS};
26use crate::control::{Hook, Progress};
27use crate::error::{DimaError, IoAction, IoTarget, Location, Result};
28use crate::model::{Checksum, Compression, InputInfo};
29
30mod members;
31
32use members::{Members, TrailingData};
33
34const RAW_BUFFER: usize = 1 << 20;
36const HASH_QUEUE: usize = 16;
38
39#[derive(Clone, Debug, PartialEq, Eq)]
41pub enum Source {
42 Path(PathBuf),
44 Stdin,
46}
47
48#[must_use]
53pub fn is_descriptor_alias(p: &Path) -> bool {
54 if !cfg!(unix) {
55 return false;
56 }
57 let components: Vec<_> = p.components().map(|c| c.as_os_str().to_owned()).collect();
58 let is = |i: usize, s: &str| components.get(i).is_some_and(|c| c == s);
59 (is(1, "dev") && (is(2, "stdout") || is(2, "stderr") || is(2, "stdin") || is(2, "fd")))
60 || (is(1, "proc") && is(3, "fd"))
61}
62
63impl Source {
64 #[must_use]
66 pub fn display_name(&self) -> String {
67 match self {
68 Self::Path(p) => p.display().to_string(),
69 Self::Stdin => "standard input".to_owned(),
70 }
71 }
72
73 #[must_use]
76 pub fn provenance_name(&self) -> String {
77 match self {
78 Self::Path(p) if is_descriptor_alias(p) => "-".to_owned(),
79 Self::Path(p) => p.file_name().map_or_else(
80 || p.display().to_string(),
81 |n| n.to_string_lossy().into_owned(),
82 ),
83 Self::Stdin => "-".to_owned(),
84 }
85 }
86
87 fn target(&self) -> IoTarget {
88 match self {
89 Self::Path(p) => IoTarget::Path(p.clone()),
90 Self::Stdin => IoTarget::Stdin,
91 }
92 }
93}
94
95pub const COMPRESSION_EXTENSIONS: &[&str] = &[".gz", ".bgz", ".bz2", ".zst"];
97pub const FASTA_EXTENSIONS: &[&str] = &[
99 ".fasta", ".fas", ".fa", ".fna", ".ffn", ".faa", ".frn", ".afa", ".aln", ".mfa",
100];
101const UNREAD_COMPRESSION_EXTENSIONS: &[&str] = &[".xz"];
104
105#[must_use]
108pub fn query_name_from_source(source: &Source) -> Option<String> {
109 let Source::Path(p) = source else {
110 return None;
111 };
112 if is_descriptor_alias(p) {
113 return None;
114 }
115 let mut name = p.file_name()?.to_string_lossy().into_owned();
116 let strip = |name: &mut String, list: &[&str]| {
117 let lower = name.to_ascii_lowercase();
118 if let Some(ext) = list.iter().find(|e| lower.ends_with(*e)) {
119 name.truncate(name.len() - ext.len());
120 true
121 } else {
122 false
123 }
124 };
125 while strip(&mut name, COMPRESSION_EXTENSIONS)
128 || strip(&mut name, UNREAD_COMPRESSION_EXTENSIONS)
129 {}
130 while strip(&mut name, FASTA_EXTENSIONS) {}
131 let name = name.trim().to_owned();
132 (!name.is_empty()).then_some(name)
133}
134
135#[derive(Clone)]
137enum RawHandle {
138 File(Arc<File>),
139 Stdin,
140}
141
142impl Read for RawHandle {
143 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
144 match self {
145 Self::File(f) => (&**f).read(buf),
146 Self::Stdin => io::stdin().read(buf),
147 }
148 }
149}
150
151#[derive(Debug)]
154pub(crate) struct StoredReadError(pub(crate) io::Error);
155
156impl std::fmt::Display for StoredReadError {
157 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158 self.0.fmt(f)
159 }
160}
161
162impl std::error::Error for StoredReadError {
163 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
164 Some(&self.0)
165 }
166}
167
168struct Tee {
171 raw: RawHandle,
172 tx: SyncSender<Vec<u8>>,
173 progress: Arc<Progress>,
174}
175
176impl Tee {
177 fn duplicate(&self) -> Self {
178 Self {
179 raw: self.raw.clone(),
180 tx: self.tx.clone(),
181 progress: Arc::clone(&self.progress),
182 }
183 }
184}
185
186impl Read for Tee {
187 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
188 loop {
189 match self.raw.read(buf) {
190 Ok(n) => {
191 if n > 0 {
192 let _ = self.tx.send(buf[..n].to_vec());
194 self.progress.add_bytes_read(n as u64);
195 }
196 return Ok(n);
197 }
198 Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
199 Err(e) => return Err(io::Error::new(e.kind(), StoredReadError(e))),
200 }
201 }
202 }
203}
204
205pub struct ChecksumJob {
207 handle: Option<thread::JoinHandle<([u8; 32], u64)>>,
209}
210
211impl ChecksumJob {
212 fn spawn(thread_start: Option<Hook>) -> Result<(Self, SyncSender<Vec<u8>>)> {
213 let (tx, rx) = mpsc::sync_channel::<Vec<u8>>(HASH_QUEUE);
214 let handle = thread::Builder::new()
215 .name("dima-sha256".into())
216 .spawn(move || {
217 if let Some(h) = &thread_start {
218 h.call();
219 }
220 let mut hasher = Sha256::new();
221 let mut size = 0u64;
222 for chunk in rx {
223 hasher.update(&chunk);
224 size += chunk.len() as u64;
225 }
226 (hasher.finalize().into(), size)
227 })
228 .map_err(|e| DimaError::system("the checksum thread", e))?;
229 Ok((
230 Self {
231 handle: Some(handle),
232 },
233 tx,
234 ))
235 }
236
237 pub fn finish(mut self) -> Result<(Checksum, u64)> {
243 let (digest, size) = self
244 .handle
245 .take()
246 .ok_or_else(|| DimaError::internal("the checksum was finished twice"))?
247 .join()
248 .map_err(|_| DimaError::internal("the checksum thread panicked"))?;
249 let value = digest.iter().fold(String::with_capacity(64), |mut s, b| {
250 use std::fmt::Write as _;
251 let _ = write!(s, "{b:02x}");
252 s
253 });
254 Ok((
255 Checksum {
256 kind: "sha-256".to_owned(),
257 value,
258 },
259 size,
260 ))
261 }
262}
263
264fn read_up_to(r: &mut impl Read, buf: &mut [u8]) -> io::Result<usize> {
266 let mut filled = 0;
267 while filled < buf.len() {
268 match r.read(&mut buf[filled..]) {
269 Ok(0) => break,
270 Ok(n) => filled += n,
271 Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
272 Err(e) => return Err(e),
273 }
274 }
275 Ok(filled)
276}
277
278#[derive(Clone, Copy, Debug, PartialEq, Eq)]
280enum Sniffed {
281 Supported(Compression),
282 Unsupported(&'static str, &'static str),
283}
284
285fn sniff(prefix: &[u8]) -> Sniffed {
286 use Compression as C;
287 match prefix {
288 [0x1F, 0x8B, ..] => Sniffed::Supported(C::Gzip),
289 [b'B', b'Z', b'h', b'1'..=b'9', ..] => Sniffed::Supported(C::Bzip2),
290 [0xFD, b'7', b'z', b'X', b'Z', 0x00, ..] => Sniffed::Unsupported(
291 "xz (.xz)",
292 "decompress it first (`xz -dk NAME.xz`, or 7-Zip on Windows) or recompress it with \
293 Zstandard (`xz -dc NAME.xz | zstd -19 -o NAME.zst`)",
294 ),
295 [0x28, 0xB5, 0x2F, 0xFD, ..] => Sniffed::Supported(C::Zstd),
296 [b, 0x2A, 0x4D, 0x18, ..] if b & 0xF0 == 0x50 => Sniffed::Supported(C::Zstd),
298 [0x1F, 0x9D, ..] => Sniffed::Unsupported(
299 "Unix compress (.Z)",
300 "decompress it first (`gzip -d` or `uncompress`) or recompress with gzip or zstd",
301 ),
302 [0x04, 0x22, 0x4D, 0x18, ..] => Sniffed::Unsupported(
303 "LZ4",
304 "decompress it first (`lz4 -d`) or recompress with gzip or zstd",
305 ),
306 [0x5D, 0x00, 0x00, ..] => Sniffed::Unsupported(
307 "legacy LZMA (.lzma)",
308 "decompress it first (`unlzma`) or recompress with gzip or zstd",
309 ),
310 [b'P', b'K', 0x03, 0x04, ..] => Sniffed::Unsupported(
311 "a zip archive",
312 "extract the FASTA file from the archive first",
313 ),
314 [b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C, ..] => Sniffed::Unsupported(
315 "a 7z archive",
316 "extract the FASTA file from the archive first",
317 ),
318 _ => Sniffed::Supported(C::None),
319 }
320}
321
322fn handle_bom(
324 mut stream: Box<dyn Read + Send>,
325 source: &Source,
326 compression: Compression,
327) -> Result<(Box<dyn Read + Send>, u64)> {
328 let mut head = [0u8; 4];
329 let n = read_up_to(&mut stream, &mut head)
331 .map_err(|e| map_stream_error(e, source, Some(compression)))?;
332 let head = &head[..n];
333 let encoding = match head {
335 [0xFF, 0xFE, 0x00, 0x00] | [0x00, 0x00, 0xFE, 0xFF] => Some("UTF-32"),
336 [0xFF, 0xFE, ..] | [0xFE, 0xFF, ..] => Some("UTF-16"),
337 _ => None,
338 };
339 if let Some(encoding) = encoding {
340 return Err(DimaError::input_format(
341 Location::new(source.display_name()).byte(0),
342 format!("the text is {encoding} encoded"),
343 )
344 .with_hint(format!(
345 "convert it to UTF-8 or ASCII, e.g. `iconv -f {encoding} -t UTF-8 in.fasta > out.fasta`"
346 )));
347 }
348 let skip = if head.starts_with(&[0xEF, 0xBB, 0xBF]) {
349 3
350 } else {
351 0
352 };
353 let rest = head[skip..].to_vec();
354 Ok((Box::new(Cursor::new(rest).chain(stream)), skip as u64))
355}
356
357#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
359struct TextStart {
360 bytes: u64,
362 lines: u64,
364}
365
366fn skip_blank_lines(
380 mut stream: Box<dyn Read + Send>,
381 source: &Source,
382 compression: Compression,
383 bom: u64,
384) -> Result<(Box<dyn Read + Send>, TextStart)> {
385 let mut buf: Vec<u8> = Vec::new();
386 let mut i = 0;
387 let mut start = TextStart {
388 bytes: bom,
389 lines: 0,
390 };
391 let mut column: u64 = 0;
394 let mut first_cr: Option<u64> = None;
395 let mut ended = false;
396 loop {
397 if i == buf.len() {
398 if ended {
399 break;
400 }
401 buf.clear();
402 i = 0;
403 buf.resize(CR_ONLY_PEEK, 0);
404 let n = loop {
405 match stream.read(&mut buf) {
406 Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
407 r => break r.map_err(|e| map_stream_error(e, source, Some(compression)))?,
408 }
409 };
410 buf.truncate(n);
411 ended = n == 0;
412 continue;
413 }
414 let b = buf[i];
415 if b == b'\n' {
416 start.bytes += column + 1;
417 start.lines += 1;
418 column = 0;
419 first_cr = None;
420 } else if alphabet::is_whitespace(b) {
421 if b == b'\r' && first_cr.is_none() {
422 first_cr = Some(column);
423 }
424 column += 1;
425 } else if column == 0 {
426 break;
427 } else {
428 return Err(indented_line(
429 source,
430 start,
431 column,
432 first_cr,
433 b,
434 buf.get(i + 1),
435 ));
436 }
437 i += 1;
438 }
439 let mut rest = buf.split_off(i);
440 if !ended && rest.len() < CR_ONLY_PEEK && memchr::memchr(b'\n', &rest).is_none() {
443 let before = rest.len();
444 rest.resize(CR_ONLY_PEEK, 0);
445 let n = loop {
446 match stream.read(&mut rest[before..]) {
447 Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
448 r => break r.map_err(|e| map_stream_error(e, source, Some(compression)))?,
449 }
450 };
451 rest.truncate(before + n);
452 }
453 if let (None, Some(cr)) = (memchr::memchr(b'\n', &rest), memchr::memchr(b'\r', &rest)) {
454 return Err(DimaError::input_format(
455 Location::new(source.display_name())
456 .record(1)
457 .line(start.lines + 1)
458 .byte(start.bytes + cr as u64),
459 "carriage return inside a header line",
460 )
461 .with_hint(CR_ONLY_HINT));
462 }
463 Ok((Box::new(Cursor::new(rest).chain(stream)), start))
464}
465
466const CR_ONLY_HINT: &str = "the file uses classic Mac OS line endings (CR only); convert them, \
468 e.g. `tr '\\r' '\\n' < in.fasta > out.fasta`";
469
470fn indented_line(
474 source: &Source,
475 start: TextStart,
476 column: u64,
477 first_cr: Option<u64>,
478 found: u8,
479 next: Option<&u8>,
480) -> DimaError {
481 let at = |offset: u64| {
482 Location::new(source.display_name())
483 .line(start.lines + 1)
484 .column(offset + 1)
485 .byte(start.bytes + offset)
486 };
487 if let Some(cr) = first_cr {
488 return DimaError::input_format(at(cr), "carriage return before the first header line")
491 .with_hint(
492 "the file uses classic Mac OS line endings (CR only) or has a stray carriage \
493 return; convert carriage returns to line feeds, e.g. \
494 `tr '\\r' '\\n' < in.fasta > out.fasta`",
495 );
496 }
497 if found == b'>' {
498 return DimaError::input_format(at(column), "white space before the first header's '>'")
499 .with_hint("a header line starts with '>'; remove the white space before it");
500 }
501 DimaError::input_format(
502 at(column),
503 format!(
504 "expected a '>' header line, found {}",
505 crate::normalise::describe_byte(found)
506 ),
507 )
508 .with_hint(start_hint(found, next.copied()))
509}
510
511fn start_hint(found: u8, next: Option<u8>) -> &'static str {
513 match (found, next) {
514 (0, _) => {
515 "the file is binary, not FASTA text (or UTF-16/UTF-32 text without a byte-order \
516 mark; convert it with `iconv`)"
517 }
518 (0xC2, Some(0xA0) | None) => {
519 "0xC2 0xA0 is a no-break space (U+00A0), which is not white space in FASTA; delete \
520 it or replace it with a space"
521 }
522 _ => {
523 "DiMA reads aligned FASTA; if this is an archive (tar) or another format, extract \
524 or convert it first"
525 }
526 }
527}
528
529const CR_ONLY_PEEK: usize = 1 << 16;
531
532fn map_stream_error(e: io::Error, source: &Source, compression: Option<Compression>) -> DimaError {
535 let kind = e.kind();
536 match e.into_inner() {
537 Some(inner) => match inner.downcast::<StoredReadError>() {
538 Ok(stored) if stored.0.kind() == io::ErrorKind::IsADirectory => {
540 DimaError::usage(format!("{} is a directory", source.display_name()))
541 .with_hint("give the alignment file with -i, or redirect a file")
542 }
543 Ok(stored) => DimaError::io(IoAction::Read, source.target(), stored.0),
544 Err(other) => match other.downcast::<TrailingData>() {
545 Ok(t) => trailing(source, &t),
546 Err(other) => corrupt(source, compression, &other.to_string(), kind),
547 },
548 },
549 None => corrupt(source, compression, &kind.to_string(), kind),
550 }
551}
552
553fn trailing(source: &Source, t: &TrailingData) -> DimaError {
556 let (tool, ext) = if t.format == "bzip2" {
557 ("bzip2", "bz2")
558 } else {
559 ("gzip", "gz")
560 };
561 DimaError::input_format(Location::new(source.display_name()), t.to_string()).with_hint(format!(
562 "`{tool} -d` ignores such data with a warning; recompress the alignment without it, \
563 e.g. `{tool} -dc NAME.{ext} | {tool} > fixed.{ext}`"
564 ))
565}
566
567fn corrupt(
568 source: &Source,
569 compression: Option<Compression>,
570 detail: &str,
571 kind: io::ErrorKind,
572) -> DimaError {
573 let (name, check) = match compression {
574 Some(Compression::Gzip) => ("gzip", "gzip -t"),
575 Some(Compression::Bzip2) => ("bzip2", "bzip2 -t"),
576 Some(Compression::Zstd) => ("Zstandard", "zstd -t"),
577 Some(Compression::None) | None => ("text", ""),
578 };
579 let what = if kind == io::ErrorKind::UnexpectedEof {
580 "truncated"
581 } else {
582 "corrupt"
583 };
584 let e = DimaError::input_format(
585 Location::new(source.display_name()),
586 format!("{what} {name} data: {detail}"),
587 );
588 if check.is_empty() {
589 e
590 } else {
591 e.with_hint(format!(
592 "the file may be truncated or damaged; test it with `{check}` and download or \
593 recompress it again"
594 ))
595 }
596}
597
598pub struct Opened {
600 pub source: Source,
602 pub compression: Compression,
604 pub stored_size: Option<u64>,
606 pub modified: Option<SystemTime>,
608 pub batches: BatchReader,
610 pub checksum: ChecksumJob,
612}
613
614impl Opened {
615 pub fn finish(self) -> Result<InputInfo> {
620 let Self {
621 source,
622 compression,
623 batches,
624 checksum,
625 ..
626 } = self;
627 drop(batches);
628 let (checksum, size_bytes) = checksum.finish()?;
629 Ok(InputInfo {
630 name: source.provenance_name(),
631 size_bytes,
632 checksum,
633 compression,
634 })
635 }
636}
637
638pub fn open(source: &Source, progress: &Arc<Progress>) -> Result<Opened> {
645 open_with(source, progress, None)
646}
647
648pub(crate) fn open_with(
650 source: &Source,
651 progress: &Arc<Progress>,
652 thread_start: Option<&Hook>,
653) -> Result<Opened> {
654 let (raw, stored_size, modified) = match source {
655 Source::Path(p) => {
656 let directory = || {
657 DimaError::usage(format!("'{}' is a directory", p.display()))
658 .with_hint("give the path of a FASTA file")
659 };
660 let file = File::open(p).map_err(|e| {
662 if p.is_dir() {
663 directory()
664 } else {
665 DimaError::io(IoAction::Open, source.target(), e)
666 }
667 })?;
668 let meta = file
669 .metadata()
670 .map_err(|e| DimaError::io(IoAction::Inspect, source.target(), e))?;
671 if meta.is_dir() {
672 return Err(directory());
673 }
674 let size = meta.is_file().then_some(meta.len());
675 let modified = meta.is_file().then(|| meta.modified().ok()).flatten();
676 (RawHandle::File(Arc::new(file)), size, modified)
677 }
678 Source::Stdin => (RawHandle::Stdin, None, None),
679 };
680 if let Some(size) = stored_size {
681 progress.set_bytes_total(size);
682 }
683 let (checksum, tx) = ChecksumJob::spawn(thread_start.cloned())?;
684 let mut tee = Tee {
685 raw,
686 tx,
687 progress: Arc::clone(progress),
688 };
689 let drain = tee.duplicate();
690 let mut prefix = [0u8; 6];
691 let n = read_up_to(&mut tee, &mut prefix).map_err(|e| map_stream_error(e, source, None))?;
692 let prefix = prefix[..n].to_vec();
693 let compression = match sniff(&prefix) {
694 Sniffed::Supported(c) => c,
695 Sniffed::Unsupported(what, hint) => {
696 return Err(DimaError::input_format(
697 Location::new(source.display_name()).byte(0),
698 format!("the input is {what}, which DiMA cannot read"),
699 )
700 .with_hint(format!(
701 "{hint}; DiMA reads plain, gzip/BGZF, bzip2 and Zstandard FASTA"
702 )));
703 }
704 };
705
706 tracing::debug!(
707 source = %source.display_name(),
708 ?compression,
709 stored_size,
710 "opened input"
711 );
712 let chained = BufReader::with_capacity(RAW_BUFFER, Cursor::new(prefix).chain(tee));
713 let decoded: Box<dyn Read + Send> = match compression {
714 Compression::None => Box::new(chained),
715 Compression::Gzip => Box::new(Members::new(
716 chained,
717 flate2::bufread::GzDecoder::new,
718 flate2::bufread::GzDecoder::into_inner,
719 &[0x1F, 0x8B],
720 "gzip",
721 )),
722 Compression::Bzip2 => Box::new(Members::new(
723 chained,
724 bzip2::bufread::BzDecoder::new,
725 bzip2::bufread::BzDecoder::into_inner,
726 b"BZh",
727 "bzip2",
728 )),
729 Compression::Zstd => {
730 let mut d = zstd::stream::read::Decoder::with_buffer(chained)
731 .map_err(|e| map_stream_error(e, source, Some(compression)))?;
732 d.window_log_max(31)
734 .map_err(|e| map_stream_error(e, source, Some(compression)))?;
735 Box::new(d)
736 }
737 };
738 let (text, bom) = handle_bom(decoded, source, compression)?;
739 let (text, start) = skip_blank_lines(text, source, compression, bom)?;
740 Ok(Opened {
741 source: source.clone(),
742 compression,
743 stored_size,
744 modified,
745 batches: BatchReader::new(text, source.clone(), compression, start, Some(drain)),
746 checksum,
747 })
748}
749
750#[derive(Clone, Debug, PartialEq, Eq)]
752pub struct RawRecord {
753 pub range: Range<usize>,
755 pub header_end: usize,
757 pub line: u64,
759 pub byte: u64,
761}
762
763#[derive(Clone, Debug, Default)]
765pub struct RawBatch {
766 pub first_record: u64,
768 pub data: Vec<u8>,
770 pub records: Vec<RawRecord>,
772}
773
774impl RawBatch {
775 pub fn clear(&mut self) {
777 self.data.clear();
778 self.records.clear();
779 self.first_record = 0;
780 }
781
782 #[must_use]
784 pub fn len(&self) -> usize {
785 self.records.len()
786 }
787
788 #[must_use]
790 pub fn is_empty(&self) -> bool {
791 self.records.is_empty()
792 }
793
794 #[must_use]
796 pub fn header(&self, i: usize) -> &[u8] {
797 let r = &self.records[i];
798 let h = &self.data[r.range.start + 1..r.range.start + r.header_end];
799 h.strip_suffix(b"\r").unwrap_or(h)
800 }
801
802 #[must_use]
805 pub fn sequence(&self, i: usize) -> (&[u8], usize) {
806 let r = &self.records[i];
807 let start = (r.range.start + r.header_end + 1).min(r.range.end);
808 (&self.data[start..r.range.end], start - r.range.start)
809 }
810}
811
812#[derive(Clone, Copy, Debug, PartialEq, Eq)]
819pub struct BatchLimits {
820 pub max_records: usize,
822 pub max_bytes: usize,
824}
825
826impl Default for BatchLimits {
827 fn default() -> Self {
828 Self {
829 max_records: BATCH_MAX_RECORDS,
830 max_bytes: BATCH_MAX_BYTES,
831 }
832 }
833}
834
835struct Doubling;
837
838impl seq_io::policy::BufPolicy for Doubling {
839 fn grow_to(&mut self, current_size: usize) -> Option<usize> {
840 current_size.checked_mul(2)
841 }
842}
843
844pub struct BatchReader {
846 reader: seq_io::fasta::Reader<Box<dyn Read + Send>, Doubling>,
847 source: Source,
848 compression: Compression,
849 byte_base: u64,
850 line_base: u64,
852 next_record: u64,
853 limits: BatchLimits,
854 finished: bool,
855 drain: Option<Tee>,
856}
857
858impl BatchReader {
859 fn new(
860 text: Box<dyn Read + Send>,
861 source: Source,
862 compression: Compression,
863 start: TextStart,
864 drain: Option<Tee>,
865 ) -> Self {
866 Self {
867 reader: seq_io::fasta::Reader::with_capacity(text, 1 << 16).set_policy(Doubling),
868 source,
869 compression,
870 byte_base: start.bytes,
871 line_base: start.lines,
872 next_record: 1,
873 limits: BatchLimits::default(),
874 finished: false,
875 drain,
876 }
877 }
878
879 #[must_use]
881 pub fn with_limits(mut self, limits: BatchLimits) -> Self {
882 self.limits = limits;
883 self
884 }
885
886 #[must_use]
888 pub fn records_read(&self) -> u64 {
889 self.next_record - 1
890 }
891
892 #[must_use]
894 pub const fn is_finished(&self) -> bool {
895 self.finished
896 }
897
898 #[must_use]
900 pub fn source_name(&self) -> String {
901 self.source.display_name()
902 }
903
904 fn location(&self, line: u64, byte: u64) -> Location {
905 Location::new(self.source.display_name())
906 .line(line)
907 .byte(byte)
908 }
909
910 pub fn read_batch(&mut self, batch: &mut RawBatch, cancel: &AtomicBool) -> Result<bool> {
917 batch.clear();
918 batch.first_record = self.next_record;
919 if self.finished {
920 return Ok(false);
921 }
922 let is_first = self.next_record == 1;
923 loop {
924 if cancel.load(Ordering::Relaxed) {
925 return Err(DimaError::Cancelled);
926 }
927 let start = batch.data.len();
928 let Some(next) = self.reader.next() else {
929 self.finish_input()?;
930 break;
931 };
932 match next {
933 Ok(record) => {
934 let _ = record.write_unchanged(&mut batch.data);
937 }
938 Err(e) => return Err(self.map_parse_error(e)),
939 }
940 let position = self
941 .reader
942 .position()
943 .ok_or_else(|| DimaError::internal("record without a position"))?;
944 let (line, byte) = (
945 position.line() + self.line_base,
946 position.byte() + self.byte_base,
947 );
948 let raw = &batch.data[start..];
949 let header_end = memchr::memchr(b'\n', raw).unwrap_or(raw.len());
950 self.check_header(&raw[..header_end], line, byte)?;
951 batch.records.push(RawRecord {
952 range: start..batch.data.len(),
953 header_end,
954 line,
955 byte,
956 });
957 self.next_record += 1;
958 let n = batch.records.len();
959 let (max_records, max_bytes, min_records) = if is_first {
960 (BATCH_MAX_RECORDS, BATCH_MAX_BYTES, PROBE_MIN_RECORDS)
961 } else {
962 (self.limits.max_records.max(1), self.limits.max_bytes, 1)
963 };
964 if n >= max_records || (batch.data.len() >= max_bytes && n >= min_records) {
965 break;
966 }
967 }
968 if batch.is_empty() {
969 if is_first {
970 return Err(DimaError::input_format(
971 Location::new(self.source.display_name()),
972 "the input contains no FASTA records",
973 )
974 .with_hint(
975 "DiMA reads an aligned FASTA file: records start with a '>' header line",
976 ));
977 }
978 return Ok(false);
979 }
980 Ok(true)
981 }
982
983 fn check_header(&self, header_line: &[u8], line: u64, byte: u64) -> Result<()> {
984 let body = header_line.strip_suffix(b"\r").unwrap_or(header_line);
985 let at = |i: usize| {
986 self.location(line, byte + i as u64)
987 .record(self.next_record)
988 };
989 if let Some(i) = memchr::memchr(b'\r', body) {
990 let hint = if line == 1 {
994 "the file probably uses classic Mac OS line endings (CR only); convert them, \
995 e.g. `tr '\\r' '\\n' < in.fasta > out.fasta`"
996 } else {
997 "remove the stray carriage return (0x0D) from the header, e.g. \
998 `tr -d '\\r' < in.fasta > out.fasta`"
999 };
1000 return Err(
1001 DimaError::input_format(at(i), "carriage return inside a header line")
1002 .with_hint(hint),
1003 );
1004 }
1005 if let Some(i) = memchr::memchr(0, body) {
1006 return Err(DimaError::input_format(at(i), "NUL byte in a header line")
1007 .with_hint("the file is binary or corrupt; FASTA is plain text"));
1008 }
1009 Ok(())
1010 }
1011
1012 fn finish_input(&mut self) -> Result<()> {
1015 self.finished = true;
1016 if let Some(mut drain) = self.drain.take() {
1017 io::copy(&mut drain, &mut io::sink())
1018 .map_err(|e| map_stream_error(e, &self.source, Some(self.compression)))?;
1019 }
1020 Ok(())
1021 }
1022
1023 fn map_parse_error(&self, e: seq_io::fasta::Error) -> DimaError {
1024 match e {
1025 seq_io::fasta::Error::Io(e) => {
1026 map_stream_error(e, &self.source, Some(self.compression))
1027 }
1028 seq_io::fasta::Error::InvalidStart { line, found } => {
1029 let loc =
1030 Location::new(self.source.display_name()).line(line as u64 + self.line_base);
1031 if found == b'@' {
1032 DimaError::input_format(loc, "this looks like FASTQ ('@' header)").with_hint(
1033 "DiMA analyses aligned FASTA; align the sequences and save them as FASTA",
1034 )
1035 } else {
1036 DimaError::input_format(
1037 loc,
1038 format!(
1039 "expected a '>' header line, found {}",
1040 crate::normalise::describe_byte(found)
1041 ),
1042 )
1043 .with_hint(start_hint(found, None))
1044 }
1045 }
1046 seq_io::fasta::Error::BufferLimit => {
1047 DimaError::internal("record buffer limit reached (no limit should apply)")
1048 }
1049 }
1050 }
1051}
1052
1053pub fn sha256_file(path: &Path) -> Result<String> {
1059 let target = || IoTarget::Path(path.to_path_buf());
1060 let mut f = File::open(path).map_err(|e| DimaError::io(IoAction::Open, target(), e))?;
1061 let mut h = Sha256::new();
1062 let mut buf = vec![0u8; RAW_BUFFER];
1063 loop {
1064 let n = match f.read(&mut buf) {
1065 Ok(0) => break,
1066 Ok(n) => n,
1067 Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
1068 Err(e) => return Err(DimaError::io(IoAction::Read, target(), e)),
1069 };
1070 h.update(&buf[..n]);
1071 }
1072 Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())
1073}
1074
1075#[cfg(test)]
1077pub(crate) type TestRecords = Vec<(Vec<u8>, Vec<u8>)>;
1078
1079#[cfg(test)]
1082pub(crate) fn read_all(source: &Source) -> Result<(TestRecords, InputInfo)> {
1083 let progress = Arc::new(Progress::new());
1084 let mut opened = open(source, &progress)?;
1085 let cancel = AtomicBool::new(false);
1086 let mut out = Vec::new();
1087 let mut batch = RawBatch::default();
1088 while opened.batches.read_batch(&mut batch, &cancel)? {
1089 for i in 0..batch.len() {
1090 out.push((batch.header(i).to_vec(), batch.sequence(i).0.to_vec()));
1091 }
1092 }
1093 let info = opened.finish()?;
1094 Ok((out, info))
1095}
1096
1097#[cfg(test)]
1098mod tests {
1099 use super::*;
1100 use std::io::Write;
1101
1102 fn tmp(bytes: &[u8], name: &str) -> (tempfile::TempDir, PathBuf) {
1103 let dir = tempfile::tempdir().unwrap();
1104 let p = dir.path().join(name);
1105 std::fs::write(&p, bytes).unwrap();
1106 (dir, p)
1107 }
1108
1109 fn sha(bytes: &[u8]) -> String {
1110 Sha256::digest(bytes)
1111 .iter()
1112 .map(|b| format!("{b:02x}"))
1113 .collect()
1114 }
1115
1116 const FASTA: &[u8] = b">a|x\nACGT\nAC\n>b|y\r\nACGTAC\r\n>c\nAC GT-A\n";
1117
1118 fn gzip(data: &[u8]) -> Vec<u8> {
1119 let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1120 e.write_all(data).unwrap();
1121 e.finish().unwrap()
1122 }
1123
1124 fn bzip(data: &[u8]) -> Vec<u8> {
1125 let mut e = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::default());
1126 e.write_all(data).unwrap();
1127 e.finish().unwrap()
1128 }
1129
1130 fn zst(data: &[u8]) -> Vec<u8> {
1131 zstd::encode_all(data, 3).unwrap()
1132 }
1133
1134 #[test]
1135 fn records_and_checksums_for_every_compression() {
1136 let half = FASTA.len() / 2;
1138 let cases: Vec<(&str, Vec<u8>, Compression)> = vec![
1139 ("plain", FASTA.to_vec(), Compression::None),
1140 ("gz", gzip(FASTA), Compression::Gzip),
1141 (
1142 "gz2",
1143 [gzip(&FASTA[..half]), gzip(&FASTA[half..])].concat(),
1144 Compression::Gzip,
1145 ),
1146 (
1147 "gz-padded",
1148 [gzip(FASTA), vec![0; 5000]].concat(),
1149 Compression::Gzip,
1150 ),
1151 ("bz2", bzip(FASTA), Compression::Bzip2),
1152 (
1153 "bz2x2",
1154 [bzip(&FASTA[..half]), bzip(&FASTA[half..])].concat(),
1155 Compression::Bzip2,
1156 ),
1157 (
1158 "bz2-padded",
1159 [bzip(FASTA), vec![0; 3]].concat(),
1160 Compression::Bzip2,
1161 ),
1162 ("zst", zst(FASTA), Compression::Zstd),
1163 (
1164 "zstx2",
1165 [zst(&FASTA[..half]), zst(&FASTA[half..])].concat(),
1166 Compression::Zstd,
1167 ),
1168 ];
1169 for (name, bytes, compression) in cases {
1170 let (_d, p) = tmp(&bytes, name);
1171 let (records, info) = read_all(&Source::Path(p)).unwrap();
1172 assert_eq!(info.compression, compression, "{name}");
1173 assert_eq!(info.size_bytes, bytes.len() as u64, "{name}");
1174 assert_eq!(info.checksum.value, sha(&bytes), "{name}");
1175 assert_eq!(info.name, name);
1176 let heads: Vec<_> = records.iter().map(|r| r.0.clone()).collect();
1177 assert_eq!(
1178 heads,
1179 [b"a|x".to_vec(), b"b|y".to_vec(), b"c".to_vec()],
1180 "{name}"
1181 );
1182 assert_eq!(records[0].1, b"ACGT\nAC\n", "{name}");
1183 assert_eq!(records[1].1, b"ACGTAC\r\n", "{name}");
1184 assert_eq!(records[2].1, b"AC GT-A\n", "{name}");
1185 }
1186 }
1187
1188 #[test]
1189 fn zstd_skippable_frame_and_long_window() {
1190 let mut skippable = vec![0x50, 0x2A, 0x4D, 0x18, 4, 0, 0, 0, 1, 2, 3, 4];
1191 skippable.extend(zst(FASTA));
1192 let (_d, p) = tmp(&skippable, "s.zst");
1193 let (records, info) = read_all(&Source::Path(p)).unwrap();
1194 assert_eq!((records.len(), info.compression), (3, Compression::Zstd));
1195 let mut enc = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
1196 enc.long_distance_matching(true).unwrap();
1197 enc.window_log(31).unwrap();
1198 enc.write_all(FASTA).unwrap();
1199 let (_d, p) = tmp(&enc.finish().unwrap(), "l.zst");
1200 assert_eq!(read_all(&Source::Path(p)).unwrap().0.len(), 3);
1201 }
1202
1203 #[test]
1204 fn bom_and_encodings() {
1205 let mut with_bom = vec![0xEF, 0xBB, 0xBF];
1206 with_bom.extend_from_slice(FASTA);
1207 let (_d, p) = tmp(&gzip(&with_bom), "bom.gz");
1208 let (records, _) = read_all(&Source::Path(p)).unwrap();
1209 assert_eq!(records[0].0, b"a|x");
1210 for (bad, encoding) in [
1211 (&[0xFF, 0xFE, b'>', 0][..], "UTF-16"),
1212 (&[0xFE, 0xFF, 0, b'>'], "UTF-16"),
1213 (&[0, 0, 0xFE, 0xFF], "UTF-32"),
1214 (&[0xFF, 0xFE, 0, 0], "UTF-32"),
1215 ] {
1216 let (_d, p) = tmp(bad, "u16");
1217 let e = read_all(&Source::Path(p)).err().unwrap();
1218 assert!(
1219 e.to_string().contains(&format!("is {encoding} encoded")),
1220 "{e}"
1221 );
1222 assert!(e.hint().unwrap().contains(&format!("iconv -f {encoding}")));
1223 }
1224 }
1225
1226 #[test]
1227 fn unsupported_formats_are_named() {
1228 for (bytes, what) in [
1229 (&[0x1F, 0x9D, 0x90, 0x3E][..], ".Z"),
1230 (&[0x04, 0x22, 0x4D, 0x18, 0, 0], "LZ4"),
1231 (&[0x5D, 0, 0, 0x80, 0, 0], "LZMA"),
1232 (b"PK\x03\x04abc", "zip"),
1233 (&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C], "7z"),
1234 (&[0xFD, b'7', b'z', b'X', b'Z', 0x00, 0x00, 0x04], "xz"),
1235 ] {
1236 let (_d, p) = tmp(bytes, "x");
1237 let e = read_all(&Source::Path(p)).err().unwrap();
1238 assert!(e.to_string().contains(what), "{e}");
1239 assert_eq!(e.class(), crate::error::ExitClass::InvalidInput);
1240 let hint = e.hint().unwrap();
1241 assert!(
1242 hint.ends_with("DiMA reads plain, gzip/BGZF, bzip2 and Zstandard FASTA"),
1243 "{hint}"
1244 );
1245 }
1246 let (_d, p) = tmp(&[0xFD, b'7', b'z', b'X', b'Z', 0x00], "a.fasta.xz");
1248 let e = read_all(&Source::Path(p)).err().unwrap();
1249 assert!(
1250 e.to_string()
1251 .contains("the input is xz (.xz), which DiMA cannot read"),
1252 "{e}"
1253 );
1254 assert!(e.hint().unwrap().contains("`xz -dk NAME.xz`"));
1255 assert!(e.hint().unwrap().contains("zstd"));
1256 }
1257
1258 #[test]
1259 fn structural_errors() {
1260 let check = |bytes: &[u8], needle: &str| {
1261 let (_d, p) = tmp(bytes, "f.fa");
1262 let e = read_all(&Source::Path(p))
1263 .err()
1264 .unwrap_or_else(|| panic!("{needle}: no error"));
1265 assert!(e.to_string().contains(needle), "{needle}: {e}");
1266 assert_eq!(e.class(), crate::error::ExitClass::InvalidInput, "{needle}");
1267 e
1268 };
1269 let e = check(b"", "no FASTA records");
1270 assert!(e.hint().is_some());
1271 let e = check(b"\n\n \n\t\x0B\x0C\r\n \t", "no FASTA records");
1273 assert!(e.hint().unwrap().contains("'>' header line"), "{e}");
1274 let e = check(b"ACGT\n>a\nACGT\n", "expected a '>' header line");
1275 assert!(e.to_string().contains("line 1"));
1276 check(b"@r1\nACGT\n+\nIIII\n", "FASTQ");
1277 let e = check(b">a\rACGT\r>b\rACGT\r", "carriage return inside a header");
1278 assert!(e.hint().unwrap().contains("classic Mac OS"), "{e}");
1279 let e = check(
1280 b">a\nACGT\n>b\nACGT\n>c\rd\nACGT\n",
1281 "carriage return inside a header",
1282 );
1283 assert!(e.to_string().contains("record 3"), "{e}");
1284 assert!(e.hint().unwrap().contains("tr -d"), "{e}");
1285 let e = check(b">a\x00b\nACGT\n", "NUL byte in a header");
1286 assert!(e.to_string().contains("record 1"), "{e}");
1287 let e = check(b" \n\tx\n>a\nACGT\n", "found 'x'");
1288 assert!(e.to_string().contains("line 2, column 2, byte 3"), "{e}");
1289 let text = e.to_string();
1291 assert!(!text[text.find("found").unwrap()..].contains('\\'), "{e}");
1292 let e = check(
1293 b"\n >a\nACGT\n",
1294 "white space before the first header's '>'",
1295 );
1296 assert!(e.to_string().contains("line 2, column 3, byte 3"), "{e}");
1297 assert!(e.hint().unwrap().contains("remove the white space"), "{e}");
1298 for text in [&b"\xC2\xA0>a\nAC\n"[..], b" \xC2\xA0>a\nAC\n"] {
1300 let e = check(text, "found 0xC2");
1301 assert!(e.hint().unwrap().contains("no-break space"), "{e}");
1302 }
1303 let e = check(b"\x00\x00", "found 0x00");
1304 assert!(e.hint().unwrap().contains("binary"), "{e}");
1305 let truncated = gzip(FASTA);
1306 let e = check(&truncated[..truncated.len() - 8], "gzip data");
1307 assert!(e.hint().unwrap().contains("gzip -t"));
1308 let mut bz = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::best());
1310 bz.write_all(FASTA).unwrap();
1311 let bz = bz.finish().unwrap();
1312 let e = check(&bz[..bz.len() / 2], "truncated bzip2 data");
1313 assert!(e.hint().unwrap().contains("bzip2 -t"), "{e}");
1314 let zst = zstd::encode_all(FASTA, 3).unwrap();
1315 let e = check(&zst[..zst.len() / 2], "Zstandard data");
1316 assert!(e.hint().unwrap().contains("zstd -t"), "{e}");
1317 let e = check(&[0x1F, 0x8B], "truncated gzip data");
1318 assert!(e.hint().unwrap().contains("gzip -t"), "{e}");
1319 let mut utf32 = vec![0xFF, 0xFE, 0x00, 0x00];
1321 utf32.extend(">a\nAC\n".bytes().flat_map(|b| [b, 0, 0, 0]));
1322 let e = check(&utf32, "UTF-32");
1323 assert!(e.hint().unwrap().contains("iconv -f UTF-32"), "{e}");
1324 let e = check(&[0xFE, 0xFF, 0, b'>'], "UTF-16");
1325 assert!(e.hint().unwrap().contains("iconv -f UTF-16"), "{e}");
1326 let mut junk = gzip(FASTA);
1328 junk.extend(b"\0\0junk");
1329 let e = check(&junk, "6 bytes that are not gzip data");
1330 assert!(e.hint().unwrap().contains("gzip -dc"), "{e}");
1331 }
1332
1333 #[test]
1334 fn missing_input_and_directories() {
1335 let e = read_all(&Source::Path("/nonexistent/x.fa".into()))
1336 .err()
1337 .unwrap();
1338 assert_eq!(e.class(), crate::error::ExitClass::Io);
1339 assert!(e.to_string().contains("cannot open '/nonexistent/x.fa'"));
1340 let dir = tempfile::tempdir().unwrap();
1341 let e = read_all(&Source::Path(dir.path().to_path_buf()))
1342 .err()
1343 .unwrap();
1344 assert_eq!(e.class(), crate::error::ExitClass::Usage);
1345 }
1346
1347 #[test]
1348 fn locations_after_many_blank_lines() {
1349 for (blank, width) in [(&b"\n"[..], 1u64), (&b"\r\n"[..], 2)] {
1351 let mut text = blank.repeat(70_000);
1352 text.extend_from_slice(b">a\nACGTACGTACGT\n>b\nACGTACGTAC1T\n");
1353 let (_d, p) = tmp(&text, "blank.fa");
1354 let mut opened = open(&Source::Path(p), &Arc::new(Progress::new())).unwrap();
1355 let mut batch = RawBatch::default();
1356 opened
1357 .batches
1358 .read_batch(&mut batch, &AtomicBool::new(false))
1359 .unwrap();
1360 let lines: Vec<u64> = batch.records.iter().map(|r| r.line).collect();
1361 let bytes: Vec<u64> = batch.records.iter().map(|r| r.byte).collect();
1362 assert_eq!(lines, [70_001, 70_003]);
1363 assert_eq!(bytes, [70_000 * width, 70_000 * width + 16]);
1364 }
1365 for (lines, count, width) in [
1368 (&b" \t\x0B\x0C\r\n"[..], 70_000u64, 6u64),
1369 (&[b' '; 100_000][..], 1, 100_000),
1370 ] {
1371 let mut text = lines.repeat(usize::try_from(count).unwrap());
1372 if count == 1 {
1373 text.push(b'\n');
1374 }
1375 text.extend_from_slice(b">a\nACGT\n");
1376 let (_d, p) = tmp(&text, "ws.fa");
1377 let mut opened = open(&Source::Path(p), &Arc::new(Progress::new())).unwrap();
1378 let mut batch = RawBatch::default();
1379 opened
1380 .batches
1381 .read_batch(&mut batch, &AtomicBool::new(false))
1382 .unwrap();
1383 let skipped = if count == 1 { width + 1 } else { count * width };
1384 assert_eq!(
1385 (batch.records[0].line, batch.records[0].byte),
1386 (count + 1, skipped)
1387 );
1388 }
1389 let mut text = vec![b' '; 100_000];
1390 text.extend_from_slice(b"x\n>a\nAC\n");
1391 let (_d, p) = tmp(&text, "ws.fa");
1392 let e = read_all(&Source::Path(p)).err().unwrap();
1393 assert!(
1394 e.to_string().contains("line 1, column 100001, byte 100000"),
1395 "{e}"
1396 );
1397 let (_d, p) = tmp(&b"\n".repeat(70_000), "empty.fa");
1399 let e = read_all(&Source::Path(p)).err().unwrap();
1400 assert!(e.to_string().contains("no FASTA records"), "{e}");
1401 let (_d, p) = tmp(b"\n\r>a\nAC\n", "cr.fa");
1402 let e = read_all(&Source::Path(p)).err().unwrap();
1403 assert!(e.to_string().contains("line 2"), "{e}");
1404 assert!(e.hint().unwrap().contains("classic Mac OS"), "{e}");
1405 for (start, at) in [
1407 (&b"\xEF\xBB\xBF\n>a\r"[..], "record 1 (line 2, byte 6)"),
1408 (b">MERS_spike|Camel\r", "record 1 (line 1, byte 17)"),
1409 ] {
1410 let mut mac = start.to_vec();
1411 mac.extend(b"ACGT\r".repeat(100_000));
1412 let (_d, p) = tmp(&mac, "mac.fa");
1413 let e = read_all(&Source::Path(p)).err().unwrap();
1414 assert!(e.to_string().contains(at), "{e}");
1415 assert!(e.hint().unwrap().contains("classic Mac OS"), "{e}");
1416 }
1417 }
1418
1419 #[test]
1420 fn positions_and_batches() {
1421 let mut text = Vec::new();
1423 for i in 0..10_007 {
1424 write!(text, ">r{i}\r\nAC\r\nGT\r\n").unwrap();
1425 }
1426 let (_d, p) = tmp(&text, "b.fa");
1427 let progress = Arc::new(Progress::new());
1428 let opened = open(&Source::Path(p), &progress).unwrap();
1429 let mut reader = opened.batches.with_limits(BatchLimits {
1430 max_records: 3,
1431 max_bytes: usize::MAX,
1432 });
1433 let cancel = AtomicBool::new(false);
1434 let mut batch = RawBatch::default();
1435 let mut sizes = Vec::new();
1436 let mut lines = Vec::new();
1437 while reader.read_batch(&mut batch, &cancel).unwrap() {
1438 sizes.push((batch.first_record, batch.len()));
1439 for (i, r) in batch.records.iter().enumerate() {
1440 lines.push((r.line, r.byte));
1441 let n = batch.first_record as usize - 1 + i;
1442 assert_eq!(batch.header(i), format!("r{n}").as_bytes());
1443 assert_eq!(batch.sequence(i).0, b"AC\r\nGT\r\n");
1444 }
1445 }
1446 assert_eq!(sizes, [(1, 10_000), (10_001, 3), (10_004, 3), (10_007, 1)]);
1447 assert_eq!(lines[0], (1, 0));
1448 assert_eq!(lines[1], (4, 13));
1449 assert_eq!(reader.records_read(), 10_007);
1450 assert!(reader.is_finished());
1451 let (_d, p) = tmp(FASTA, "c.fa");
1453 let mut opened = open(&Source::Path(p), &progress).unwrap();
1454 cancel.store(true, Ordering::Relaxed);
1455 assert!(matches!(
1456 opened.batches.read_batch(&mut batch, &cancel),
1457 Err(DimaError::Cancelled)
1458 ));
1459 }
1460
1461 #[test]
1462 fn first_batch_is_the_probe_whatever_the_limits() {
1463 let mut text = Vec::new();
1464 for i in 0..1_050 {
1465 write!(text, ">r{i}\n{}\n", "A".repeat(100)).unwrap();
1466 }
1467 let (_d, p) = tmp(&text, "m.fa");
1468 let progress = Arc::new(Progress::new());
1469 let opened = open(&Source::Path(p), &progress).unwrap();
1470 let mut reader = opened.batches.with_limits(BatchLimits {
1471 max_records: 1000,
1472 max_bytes: 10,
1473 });
1474 let cancel = AtomicBool::new(false);
1475 let mut batch = RawBatch::default();
1476 assert!(reader.read_batch(&mut batch, &cancel).unwrap());
1477 assert_eq!(
1478 batch.len(),
1479 1_050,
1480 "the probe ignores the limits of later batches"
1481 );
1482 }
1483
1484 #[test]
1485 fn query_names() {
1486 let q = |s: &str| query_name_from_source(&Source::Path(s.into()));
1487 assert_eq!(
1488 q("data/HIV1_FLT_2022_pol_DNA.fasta.bz2").as_deref(),
1489 Some("HIV1_FLT_2022_pol_DNA")
1490 );
1491 assert_eq!(q("x/MERS.Spike.FA.GZ").as_deref(), Some("MERS.Spike"));
1492 assert_eq!(q("a.afa").as_deref(), Some("a"));
1493 assert_eq!(q("noext").as_deref(), Some("noext"));
1494 assert_eq!(q(".fasta"), None);
1495 assert_eq!(q("x.fa.fasta.gz").as_deref(), Some("x"));
1496 assert_eq!(q("spike.fasta.xz.zst").as_deref(), Some("spike"));
1498 assert_eq!(query_name_from_source(&Source::Stdin), None);
1499 assert_eq!(Source::Path("d/e/f.fa".into()).provenance_name(), "f.fa");
1500 assert_eq!(Source::Stdin.provenance_name(), "-");
1501 for alias in [
1504 "/dev/fd/63",
1505 "/dev/stdin",
1506 "/proc/self/fd/0",
1507 "/proc/4242/fd/11",
1508 ] {
1509 let s = Source::Path(alias.into());
1510 assert_eq!(
1511 query_name_from_source(&s),
1512 if cfg!(unix) {
1513 None
1514 } else {
1515 Some(alias.rsplit('/').next().unwrap().to_owned())
1516 },
1517 "{alias}"
1518 );
1519 if cfg!(unix) {
1520 assert_eq!(s.provenance_name(), "-", "{alias}");
1521 assert_eq!(
1522 crate::run_report::file_name(std::path::Path::new(alias)),
1523 "-"
1524 );
1525 }
1526 }
1527 assert_eq!(
1528 q("/dev/fdx/63.fa"),
1529 Some("63".into()),
1530 "only the descriptor paths"
1531 );
1532 }
1533
1534 #[test]
1535 fn sha256_of_file() {
1536 let (_d, p) = tmp(b"abc", "h");
1537 assert_eq!(
1538 sha256_file(&p).unwrap(),
1539 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
1540 );
1541 }
1542
1543 #[cfg(unix)]
1544 #[test]
1545 fn fifo_input() {
1546 let dir = tempfile::tempdir().unwrap();
1547 let p = dir.path().join("pipe");
1548 let status = std::process::Command::new("mkfifo")
1549 .arg(&p)
1550 .status()
1551 .unwrap();
1552 assert!(status.success());
1553 let writer_path = p.clone();
1554 let data = gzip(FASTA);
1555 let expected = sha(&data);
1556 let t = thread::spawn(move || {
1557 let mut f = std::fs::OpenOptions::new()
1558 .write(true)
1559 .open(writer_path)
1560 .unwrap();
1561 for chunk in data.chunks(3) {
1563 f.write_all(chunk).unwrap();
1564 f.flush().unwrap();
1565 }
1566 });
1567 let (records, info) = read_all(&Source::Path(p)).unwrap();
1568 t.join().unwrap();
1569 assert_eq!(records.len(), 3);
1570 assert_eq!(info.checksum.value, expected);
1571 assert_eq!(info.compression, Compression::Gzip);
1572 }
1573}