Skip to main content

dima_core/
input.rs

1//! Opening an input and reading it as batches of raw FASTA records.
2//!
3//! Pipeline: the stored bytes are read once (files, FIFOs, `<(…)` and standard input alike) and
4//! teed to a SHA-256 thread; the format is recognised from the first bytes (gzip/BGZF, bzip2,
5//! Zstandard or plain text; unsupported formats, xz among them (D116), get an explicit error with
6//! a hint); the decompressed
7//! text has one UTF-8 byte-order mark and the blank lines before the first record removed
8//! (UTF-16/32 is refused with an `iconv` hint); and `seq_io` splits it into records, which are
9//! copied byte for byte into batches for the parallel normaliser. Every structural error stops
10//! the run with the source and as much of the record, line and byte as applies.
11
12use std::fs::File;
13use std::io::{self, BufReader, Cursor, Read};
14use std::ops::Range;
15use std::path::{Path, PathBuf};
16use std::sync::Arc;
17use std::sync::atomic::{AtomicBool, Ordering};
18use std::sync::mpsc::{self, SyncSender};
19use std::thread;
20use std::time::SystemTime;
21
22use sha2::{Digest, Sha256};
23
24use crate::alphabet;
25use crate::config::{BATCH_MAX_BYTES, BATCH_MAX_RECORDS, PROBE_MIN_RECORDS};
26use crate::control::{Hook, Progress};
27use crate::error::{DimaError, IoAction, IoTarget, Location, Result};
28use crate::model::{Checksum, Compression, InputInfo};
29
30mod members;
31
32use members::{Members, TrailingData};
33
34/// Size of the buffer between the stored bytes and the decoder.
35const RAW_BUFFER: usize = 1 << 20;
36/// Chunks in flight to the checksum thread.
37const HASH_QUEUE: usize = 16;
38
39/// Where an input comes from.
40#[derive(Clone, Debug, PartialEq, Eq)]
41pub enum Source {
42    /// A file system path (regular file, FIFO, `/dev/stdin`, process substitution, …).
43    Path(PathBuf),
44    /// Standard input.
45    Stdin,
46}
47
48/// `true` for a path that names an open descriptor rather than a file: `/dev/stdin`,
49/// `/dev/stdout`, `/dev/stderr`, `/dev/fd/N` (process substitution, `<(...)`) and
50/// `/proc/<pid>/fd/N`. Its file name is a descriptor number chosen by the shell, so it names
51/// nothing (D45: such inputs are named like standard input).
52#[must_use]
53pub fn is_descriptor_alias(p: &Path) -> bool {
54    if !cfg!(unix) {
55        return false;
56    }
57    let components: Vec<_> = p.components().map(|c| c.as_os_str().to_owned()).collect();
58    let is = |i: usize, s: &str| components.get(i).is_some_and(|c| c == s);
59    (is(1, "dev") && (is(2, "stdout") || is(2, "stderr") || is(2, "stdin") || is(2, "fd")))
60        || (is(1, "proc") && is(3, "fd"))
61}
62
63impl Source {
64    /// Name used in messages: the path as given, or `standard input`.
65    #[must_use]
66    pub fn display_name(&self) -> String {
67        match self {
68            Self::Path(p) => p.display().to_string(),
69            Self::Stdin => "standard input".to_owned(),
70        }
71    }
72
73    /// Name recorded in the provenance: the file name without directories, or `-` for
74    /// standard input and descriptor paths ([`is_descriptor_alias`]).
75    #[must_use]
76    pub fn provenance_name(&self) -> String {
77        match self {
78            Self::Path(p) if is_descriptor_alias(p) => "-".to_owned(),
79            Self::Path(p) => p.file_name().map_or_else(
80                || p.display().to_string(),
81                |n| n.to_string_lossy().into_owned(),
82            ),
83            Self::Stdin => "-".to_owned(),
84        }
85    }
86
87    fn target(&self) -> IoTarget {
88        match self {
89            Self::Path(p) => IoTarget::Path(p.clone()),
90            Self::Stdin => IoTarget::Stdin,
91        }
92    }
93}
94
95/// Extensions of the compressions DiMA reads (file dialogs offer them).
96pub const COMPRESSION_EXTENSIONS: &[&str] = &[".gz", ".bgz", ".bz2", ".zst"];
97/// Extensions of FASTA alignments.
98pub const FASTA_EXTENSIONS: &[&str] = &[
99    ".fasta", ".fas", ".fa", ".fna", ".ffn", ".faa", ".frn", ".afa", ".aln", ".mfa",
100];
101/// Also removed from query names, although xz is not read (D116): `spike.fasta.xz.zst` is still
102/// `spike`.
103const UNREAD_COMPRESSION_EXTENSIONS: &[&str] = &[".xz"];
104
105/// The query name implied by an input's file name (D45): the file name without known
106/// compression and FASTA extensions, or `None` for standard input or an empty stem.
107#[must_use]
108pub fn query_name_from_source(source: &Source) -> Option<String> {
109    let Source::Path(p) = source else {
110        return None;
111    };
112    if is_descriptor_alias(p) {
113        return None;
114    }
115    let mut name = p.file_name()?.to_string_lossy().into_owned();
116    let strip = |name: &mut String, list: &[&str]| {
117        let lower = name.to_ascii_lowercase();
118        if let Some(ext) = list.iter().find(|e| lower.ends_with(*e)) {
119            name.truncate(name.len() - ext.len());
120            true
121        } else {
122            false
123        }
124    };
125    // Removed repeatedly and case-insensitively, after MultiQC's sample-name cleaning (Ewels et
126    // al. 2016).
127    while strip(&mut name, COMPRESSION_EXTENSIONS)
128        || strip(&mut name, UNREAD_COMPRESSION_EXTENSIONS)
129    {}
130    while strip(&mut name, FASTA_EXTENSIONS) {}
131    let name = name.trim().to_owned();
132    (!name.is_empty()).then_some(name)
133}
134
135/// The stored bytes, shared between the decoding chain and the final drain.
136#[derive(Clone)]
137enum RawHandle {
138    File(Arc<File>),
139    Stdin,
140}
141
142impl Read for RawHandle {
143    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
144        match self {
145            Self::File(f) => (&**f).read(buf),
146            Self::Stdin => io::stdin().read(buf),
147        }
148    }
149}
150
151/// Marks an error raised while reading the stored bytes (an I/O error, exit 3), as opposed to
152/// one raised while decoding them (corrupt data, exit 4).
153#[derive(Debug)]
154pub(crate) struct StoredReadError(pub(crate) io::Error);
155
156impl std::fmt::Display for StoredReadError {
157    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158        self.0.fmt(f)
159    }
160}
161
162impl std::error::Error for StoredReadError {
163    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
164        Some(&self.0)
165    }
166}
167
168/// Reads the stored bytes, forwarding a copy of every chunk to the checksum thread and counting
169/// them for progress.
170struct Tee {
171    raw: RawHandle,
172    tx: SyncSender<Vec<u8>>,
173    progress: Arc<Progress>,
174}
175
176impl Tee {
177    fn duplicate(&self) -> Self {
178        Self {
179            raw: self.raw.clone(),
180            tx: self.tx.clone(),
181            progress: Arc::clone(&self.progress),
182        }
183    }
184}
185
186impl Read for Tee {
187    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
188        loop {
189            match self.raw.read(buf) {
190                Ok(n) => {
191                    if n > 0 {
192                        // A closed receiver only means the checksum is no longer wanted.
193                        let _ = self.tx.send(buf[..n].to_vec());
194                        self.progress.add_bytes_read(n as u64);
195                    }
196                    return Ok(n);
197                }
198                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
199                Err(e) => return Err(io::Error::new(e.kind(), StoredReadError(e))),
200            }
201        }
202    }
203}
204
205/// The SHA-256 of the stored bytes, computed on its own thread.
206pub struct ChecksumJob {
207    /// Always `Some` until [`ChecksumJob::finish`].
208    handle: Option<thread::JoinHandle<([u8; 32], u64)>>,
209}
210
211impl ChecksumJob {
212    fn spawn(thread_start: Option<Hook>) -> Result<(Self, SyncSender<Vec<u8>>)> {
213        let (tx, rx) = mpsc::sync_channel::<Vec<u8>>(HASH_QUEUE);
214        let handle = thread::Builder::new()
215            .name("dima-sha256".into())
216            .spawn(move || {
217                if let Some(h) = &thread_start {
218                    h.call();
219                }
220                let mut hasher = Sha256::new();
221                let mut size = 0u64;
222                for chunk in rx {
223                    hasher.update(&chunk);
224                    size += chunk.len() as u64;
225                }
226                (hasher.finalize().into(), size)
227            })
228            .map_err(|e| DimaError::system("the checksum thread", e))?;
229        Ok((
230            Self {
231                handle: Some(handle),
232            },
233            tx,
234        ))
235    }
236
237    /// Waits for the digest (every sender must have been dropped) and returns it with the
238    /// number of stored bytes.
239    ///
240    /// # Errors
241    /// [`DimaError::Internal`] if the checksum thread panicked.
242    pub fn finish(mut self) -> Result<(Checksum, u64)> {
243        let (digest, size) = self
244            .handle
245            .take()
246            .ok_or_else(|| DimaError::internal("the checksum was finished twice"))?
247            .join()
248            .map_err(|_| DimaError::internal("the checksum thread panicked"))?;
249        let value = digest.iter().fold(String::with_capacity(64), |mut s, b| {
250            use std::fmt::Write as _;
251            let _ = write!(s, "{b:02x}");
252            s
253        });
254        Ok((
255            Checksum {
256                kind: "sha-256".to_owned(),
257                value,
258            },
259            size,
260        ))
261    }
262}
263
264/// Reads until `buf` is full or the stream ends; returns the number of bytes read.
265fn read_up_to(r: &mut impl Read, buf: &mut [u8]) -> io::Result<usize> {
266    let mut filled = 0;
267    while filled < buf.len() {
268        match r.read(&mut buf[filled..]) {
269            Ok(0) => break,
270            Ok(n) => filled += n,
271            Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
272            Err(e) => return Err(e),
273        }
274    }
275    Ok(filled)
276}
277
278/// What the first bytes of the stored data say.
279#[derive(Clone, Copy, Debug, PartialEq, Eq)]
280enum Sniffed {
281    Supported(Compression),
282    Unsupported(&'static str, &'static str),
283}
284
285fn sniff(prefix: &[u8]) -> Sniffed {
286    use Compression as C;
287    match prefix {
288        [0x1F, 0x8B, ..] => Sniffed::Supported(C::Gzip),
289        [b'B', b'Z', b'h', b'1'..=b'9', ..] => Sniffed::Supported(C::Bzip2),
290        [0xFD, b'7', b'z', b'X', b'Z', 0x00, ..] => Sniffed::Unsupported(
291            "xz (.xz)",
292            "decompress it first (`xz -dk NAME.xz`, or 7-Zip on Windows) or recompress it with \
293             Zstandard (`xz -dc NAME.xz | zstd -19 -o NAME.zst`)",
294        ),
295        [0x28, 0xB5, 0x2F, 0xFD, ..] => Sniffed::Supported(C::Zstd),
296        // A Zstandard skippable frame (0x184D2A50–5F), as written first by pzstd.
297        [b, 0x2A, 0x4D, 0x18, ..] if b & 0xF0 == 0x50 => Sniffed::Supported(C::Zstd),
298        [0x1F, 0x9D, ..] => Sniffed::Unsupported(
299            "Unix compress (.Z)",
300            "decompress it first (`gzip -d` or `uncompress`) or recompress with gzip or zstd",
301        ),
302        [0x04, 0x22, 0x4D, 0x18, ..] => Sniffed::Unsupported(
303            "LZ4",
304            "decompress it first (`lz4 -d`) or recompress with gzip or zstd",
305        ),
306        [0x5D, 0x00, 0x00, ..] => Sniffed::Unsupported(
307            "legacy LZMA (.lzma)",
308            "decompress it first (`unlzma`) or recompress with gzip or zstd",
309        ),
310        [b'P', b'K', 0x03, 0x04, ..] => Sniffed::Unsupported(
311            "a zip archive",
312            "extract the FASTA file from the archive first",
313        ),
314        [b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C, ..] => Sniffed::Unsupported(
315            "a 7z archive",
316            "extract the FASTA file from the archive first",
317        ),
318        _ => Sniffed::Supported(C::None),
319    }
320}
321
322/// Strips one UTF-8 byte-order mark and refuses UTF-16/32 text.
323fn handle_bom(
324    mut stream: Box<dyn Read + Send>,
325    source: &Source,
326    compression: Compression,
327) -> Result<(Box<dyn Read + Send>, u64)> {
328    let mut head = [0u8; 4];
329    // The first block is decoded here, so a damaged file can fail already.
330    let n = read_up_to(&mut stream, &mut head)
331        .map_err(|e| map_stream_error(e, source, Some(compression)))?;
332    let head = &head[..n];
333    // UTF-32 first: its little-endian mark starts like UTF-16's.
334    let encoding = match head {
335        [0xFF, 0xFE, 0x00, 0x00] | [0x00, 0x00, 0xFE, 0xFF] => Some("UTF-32"),
336        [0xFF, 0xFE, ..] | [0xFE, 0xFF, ..] => Some("UTF-16"),
337        _ => None,
338    };
339    if let Some(encoding) = encoding {
340        return Err(DimaError::input_format(
341            Location::new(source.display_name()).byte(0),
342            format!("the text is {encoding} encoded"),
343        )
344        .with_hint(format!(
345            "convert it to UTF-8 or ASCII, e.g. `iconv -f {encoding} -t UTF-8 in.fasta > out.fasta`"
346        )));
347    }
348    let skip = if head.starts_with(&[0xEF, 0xBB, 0xBF]) {
349        3
350    } else {
351        0
352    };
353    let rest = head[skip..].to_vec();
354    Ok((Box::new(Cursor::new(rest).chain(stream)), skip as u64))
355}
356
357/// Where the text the FASTA parser reads starts in the decoded input.
358#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
359struct TextStart {
360    /// Bytes before it (the byte-order mark and the lines skipped).
361    bytes: u64,
362    /// Lines skipped before it.
363    lines: u64,
364}
365
366/// Skips the lines before the first record that are empty or hold only white space (space,
367/// TAB, CR, VT, FF: the bytes ignored inside sequence lines, D37), as MAFFT, Clustal Omega,
368/// EMBOSS and HMMER's Easel do. The parser would refuse a white-space line and miscounts lines
369/// and bytes after more than one buffer of blank lines (seq_io 0.3.4 `fasta.rs` `init`), so
370/// the lines are skipped and counted here and every location is offset instead. Only the
371/// current line's white space is counted, never kept, so memory stays bounded however long the
372/// lines are. A line whose first other byte follows white space (an indented `>`, a stray or
373/// classic Mac OS carriage return) is refused here with its location.
374///
375/// Text with classic Mac OS line endings (CR only) is refused here too, from the text already
376/// read (up to 64 KiB; never waiting for more, so a pipe that is still being written never
377/// blocks the caller): the parser would otherwise hold the whole file as one line before finding
378/// the CR.
379fn skip_blank_lines(
380    mut stream: Box<dyn Read + Send>,
381    source: &Source,
382    compression: Compression,
383    bom: u64,
384) -> Result<(Box<dyn Read + Send>, TextStart)> {
385    let mut buf: Vec<u8> = Vec::new();
386    let mut i = 0;
387    let mut start = TextStart {
388        bytes: bom,
389        lines: 0,
390    };
391    // The current line so far: its white-space bytes (examined and dropped) and the offset of
392    // its first carriage return.
393    let mut column: u64 = 0;
394    let mut first_cr: Option<u64> = None;
395    let mut ended = false;
396    loop {
397        if i == buf.len() {
398            if ended {
399                break;
400            }
401            buf.clear();
402            i = 0;
403            buf.resize(CR_ONLY_PEEK, 0);
404            let n = loop {
405                match stream.read(&mut buf) {
406                    Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
407                    r => break r.map_err(|e| map_stream_error(e, source, Some(compression)))?,
408                }
409            };
410            buf.truncate(n);
411            ended = n == 0;
412            continue;
413        }
414        let b = buf[i];
415        if b == b'\n' {
416            start.bytes += column + 1;
417            start.lines += 1;
418            column = 0;
419            first_cr = None;
420        } else if alphabet::is_whitespace(b) {
421            if b == b'\r' && first_cr.is_none() {
422                first_cr = Some(column);
423            }
424            column += 1;
425        } else if column == 0 {
426            break;
427        } else {
428            return Err(indented_line(
429                source,
430                start,
431                column,
432                first_cr,
433                b,
434                buf.get(i + 1),
435            ));
436        }
437        i += 1;
438    }
439    let mut rest = buf.split_off(i);
440    // The first read can be only the few bytes the format check looked at: without a line feed
441    // so far, read once more (what is available, not until 64 KiB).
442    if !ended && rest.len() < CR_ONLY_PEEK && memchr::memchr(b'\n', &rest).is_none() {
443        let before = rest.len();
444        rest.resize(CR_ONLY_PEEK, 0);
445        let n = loop {
446            match stream.read(&mut rest[before..]) {
447                Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
448                r => break r.map_err(|e| map_stream_error(e, source, Some(compression)))?,
449            }
450        };
451        rest.truncate(before + n);
452    }
453    if let (None, Some(cr)) = (memchr::memchr(b'\n', &rest), memchr::memchr(b'\r', &rest)) {
454        return Err(DimaError::input_format(
455            Location::new(source.display_name())
456                .record(1)
457                .line(start.lines + 1)
458                .byte(start.bytes + cr as u64),
459            "carriage return inside a header line",
460        )
461        .with_hint(CR_ONLY_HINT));
462    }
463    Ok((Box::new(Cursor::new(rest).chain(stream)), start))
464}
465
466/// Hint for text with classic Mac OS line endings.
467const CR_ONLY_HINT: &str = "the file uses classic Mac OS line endings (CR only); convert them, \
468                            e.g. `tr '\\r' '\\n' < in.fasta > out.fasta`";
469
470/// The error for a line before the first record that has white space and then `found` (with
471/// `next` after it, when read): `column` white-space bytes, the first carriage return at
472/// `first_cr`.
473fn indented_line(
474    source: &Source,
475    start: TextStart,
476    column: u64,
477    first_cr: Option<u64>,
478    found: u8,
479    next: Option<&u8>,
480) -> DimaError {
481    let at = |offset: u64| {
482        Location::new(source.display_name())
483            .line(start.lines + 1)
484            .column(offset + 1)
485            .byte(start.bytes + offset)
486    };
487    if let Some(cr) = first_cr {
488        // A carriage return that does not end a line: classic Mac OS line endings, or a stray
489        // one; turning it into a line feed fixes both.
490        return DimaError::input_format(at(cr), "carriage return before the first header line")
491            .with_hint(
492                "the file uses classic Mac OS line endings (CR only) or has a stray carriage \
493                 return; convert carriage returns to line feeds, e.g. \
494                 `tr '\\r' '\\n' < in.fasta > out.fasta`",
495            );
496    }
497    if found == b'>' {
498        return DimaError::input_format(at(column), "white space before the first header's '>'")
499            .with_hint("a header line starts with '>'; remove the white space before it");
500    }
501    DimaError::input_format(
502        at(column),
503        format!(
504            "expected a '>' header line, found {}",
505            crate::normalise::describe_byte(found)
506        ),
507    )
508    .with_hint(start_hint(found, next.copied()))
509}
510
511/// Hint for a first line that does not start with `>`: its first byte `found`, then `next`.
512fn start_hint(found: u8, next: Option<u8>) -> &'static str {
513    match (found, next) {
514        (0, _) => {
515            "the file is binary, not FASTA text (or UTF-16/UTF-32 text without a byte-order \
516             mark; convert it with `iconv`)"
517        }
518        (0xC2, Some(0xA0) | None) => {
519            "0xC2 0xA0 is a no-break space (U+00A0), which is not white space in FASTA; delete \
520             it or replace it with a space"
521        }
522        _ => {
523            "DiMA reads aligned FASTA; if this is an archive (tar) or another format, extract \
524             or convert it first"
525        }
526    }
527}
528
529/// How much of the text is examined for classic Mac OS line endings.
530const CR_ONLY_PEEK: usize = 1 << 16;
531
532/// Turns an error raised while reading the text into a [`DimaError`]: stored-byte read errors
533/// are I/O errors; everything else is corrupt or truncated compressed data.
534fn map_stream_error(e: io::Error, source: &Source, compression: Option<Compression>) -> DimaError {
535    let kind = e.kind();
536    match e.into_inner() {
537        Some(inner) => match inner.downcast::<StoredReadError>() {
538            // `dima analyze < folder`: the same mistake as `-i folder`.
539            Ok(stored) if stored.0.kind() == io::ErrorKind::IsADirectory => {
540                DimaError::usage(format!("{} is a directory", source.display_name()))
541                    .with_hint("give the alignment file with -i, or redirect a file")
542            }
543            Ok(stored) => DimaError::io(IoAction::Read, source.target(), stored.0),
544            Err(other) => match other.downcast::<TrailingData>() {
545                Ok(t) => trailing(source, &t),
546                Err(other) => corrupt(source, compression, &other.to_string(), kind),
547            },
548        },
549        None => corrupt(source, compression, &kind.to_string(), kind),
550    }
551}
552
553/// Data after the end of the compressed data (not padding): `gzip -d` and `bzip2 -d` ignore it
554/// with a warning, so their test commands would call the file fine.
555fn trailing(source: &Source, t: &TrailingData) -> DimaError {
556    let (tool, ext) = if t.format == "bzip2" {
557        ("bzip2", "bz2")
558    } else {
559        ("gzip", "gz")
560    };
561    DimaError::input_format(Location::new(source.display_name()), t.to_string()).with_hint(format!(
562        "`{tool} -d` ignores such data with a warning; recompress the alignment without it, \
563             e.g. `{tool} -dc NAME.{ext} | {tool} > fixed.{ext}`"
564    ))
565}
566
567fn corrupt(
568    source: &Source,
569    compression: Option<Compression>,
570    detail: &str,
571    kind: io::ErrorKind,
572) -> DimaError {
573    let (name, check) = match compression {
574        Some(Compression::Gzip) => ("gzip", "gzip -t"),
575        Some(Compression::Bzip2) => ("bzip2", "bzip2 -t"),
576        Some(Compression::Zstd) => ("Zstandard", "zstd -t"),
577        Some(Compression::None) | None => ("text", ""),
578    };
579    let what = if kind == io::ErrorKind::UnexpectedEof {
580        "truncated"
581    } else {
582        "corrupt"
583    };
584    let e = DimaError::input_format(
585        Location::new(source.display_name()),
586        format!("{what} {name} data: {detail}"),
587    );
588    if check.is_empty() {
589        e
590    } else {
591        e.with_hint(format!(
592            "the file may be truncated or damaged; test it with `{check}` and download or \
593             recompress it again"
594        ))
595    }
596}
597
598/// An opened input, ready to be read as batches.
599pub struct Opened {
600    /// The source.
601    pub source: Source,
602    /// Compression detected from the content.
603    pub compression: Compression,
604    /// Size of the stored bytes, for regular files.
605    pub stored_size: Option<u64>,
606    /// Modification time, for regular files.
607    pub modified: Option<SystemTime>,
608    /// The batch reader.
609    pub batches: BatchReader,
610    /// The checksum job; finish it after the batch reader is dropped.
611    pub checksum: ChecksumJob,
612}
613
614impl Opened {
615    /// Drops the reader (closing the checksum stream) and returns the input's identity.
616    ///
617    /// # Errors
618    /// See [`ChecksumJob::finish`].
619    pub fn finish(self) -> Result<InputInfo> {
620        let Self {
621            source,
622            compression,
623            batches,
624            checksum,
625            ..
626        } = self;
627        drop(batches);
628        let (checksum, size_bytes) = checksum.finish()?;
629        Ok(InputInfo {
630            name: source.provenance_name(),
631            size_bytes,
632            checksum,
633            compression,
634        })
635    }
636}
637
638/// Opens `source` for reading.
639///
640/// # Errors
641/// * [`DimaError::Io`] if the source cannot be opened or read (exit 3);
642/// * [`DimaError::Usage`] if the path is a directory;
643/// * [`DimaError::InputFormat`] for unsupported compression or UTF-16/32 text.
644pub fn open(source: &Source, progress: &Arc<Progress>) -> Result<Opened> {
645    open_with(source, progress, None)
646}
647
648/// [`open`], running `thread_start` first in the checksum thread.
649pub(crate) fn open_with(
650    source: &Source,
651    progress: &Arc<Progress>,
652    thread_start: Option<&Hook>,
653) -> Result<Opened> {
654    let (raw, stored_size, modified) = match source {
655        Source::Path(p) => {
656            let directory = || {
657                DimaError::usage(format!("'{}' is a directory", p.display()))
658                    .with_hint("give the path of a FASTA file")
659            };
660            // Windows refuses to open a directory as a file ("access denied"): say what it is.
661            let file = File::open(p).map_err(|e| {
662                if p.is_dir() {
663                    directory()
664                } else {
665                    DimaError::io(IoAction::Open, source.target(), e)
666                }
667            })?;
668            let meta = file
669                .metadata()
670                .map_err(|e| DimaError::io(IoAction::Inspect, source.target(), e))?;
671            if meta.is_dir() {
672                return Err(directory());
673            }
674            let size = meta.is_file().then_some(meta.len());
675            let modified = meta.is_file().then(|| meta.modified().ok()).flatten();
676            (RawHandle::File(Arc::new(file)), size, modified)
677        }
678        Source::Stdin => (RawHandle::Stdin, None, None),
679    };
680    if let Some(size) = stored_size {
681        progress.set_bytes_total(size);
682    }
683    let (checksum, tx) = ChecksumJob::spawn(thread_start.cloned())?;
684    let mut tee = Tee {
685        raw,
686        tx,
687        progress: Arc::clone(progress),
688    };
689    let drain = tee.duplicate();
690    let mut prefix = [0u8; 6];
691    let n = read_up_to(&mut tee, &mut prefix).map_err(|e| map_stream_error(e, source, None))?;
692    let prefix = prefix[..n].to_vec();
693    let compression = match sniff(&prefix) {
694        Sniffed::Supported(c) => c,
695        Sniffed::Unsupported(what, hint) => {
696            return Err(DimaError::input_format(
697                Location::new(source.display_name()).byte(0),
698                format!("the input is {what}, which DiMA cannot read"),
699            )
700            .with_hint(format!(
701                "{hint}; DiMA reads plain, gzip/BGZF, bzip2 and Zstandard FASTA"
702            )));
703        }
704    };
705
706    tracing::debug!(
707        source = %source.display_name(),
708        ?compression,
709        stored_size,
710        "opened input"
711    );
712    let chained = BufReader::with_capacity(RAW_BUFFER, Cursor::new(prefix).chain(tee));
713    let decoded: Box<dyn Read + Send> = match compression {
714        Compression::None => Box::new(chained),
715        Compression::Gzip => Box::new(Members::new(
716            chained,
717            flate2::bufread::GzDecoder::new,
718            flate2::bufread::GzDecoder::into_inner,
719            &[0x1F, 0x8B],
720            "gzip",
721        )),
722        Compression::Bzip2 => Box::new(Members::new(
723            chained,
724            bzip2::bufread::BzDecoder::new,
725            bzip2::bufread::BzDecoder::into_inner,
726            b"BZh",
727            "bzip2",
728        )),
729        Compression::Zstd => {
730            let mut d = zstd::stream::read::Decoder::with_buffer(chained)
731                .map_err(|e| map_stream_error(e, source, Some(compression)))?;
732            // Accept frames written with `--long=31` (up to 2 GiB windows).
733            d.window_log_max(31)
734                .map_err(|e| map_stream_error(e, source, Some(compression)))?;
735            Box::new(d)
736        }
737    };
738    let (text, bom) = handle_bom(decoded, source, compression)?;
739    let (text, start) = skip_blank_lines(text, source, compression, bom)?;
740    Ok(Opened {
741        source: source.clone(),
742        compression,
743        stored_size,
744        modified,
745        batches: BatchReader::new(text, source.clone(), compression, start, Some(drain)),
746        checksum,
747    })
748}
749
750/// A record copied verbatim from the input: `>header` line and sequence lines.
751#[derive(Clone, Debug, PartialEq, Eq)]
752pub struct RawRecord {
753    /// Range of the whole record in [`RawBatch::data`] (from `>` to the final line feed).
754    pub range: Range<usize>,
755    /// Offset of the line feed that ends the header, relative to `range.start`.
756    pub header_end: usize,
757    /// 1-based line number of the header.
758    pub line: u64,
759    /// 0-based byte offset of the `>` in the decompressed text.
760    pub byte: u64,
761}
762
763/// A batch of consecutive records sharing one buffer.
764#[derive(Clone, Debug, Default)]
765pub struct RawBatch {
766    /// 1-based number of the first record.
767    pub first_record: u64,
768    /// Record bytes.
769    pub data: Vec<u8>,
770    /// Records.
771    pub records: Vec<RawRecord>,
772}
773
774impl RawBatch {
775    /// Empties the batch, keeping its allocations.
776    pub fn clear(&mut self) {
777        self.data.clear();
778        self.records.clear();
779        self.first_record = 0;
780    }
781
782    /// Number of records.
783    #[must_use]
784    pub fn len(&self) -> usize {
785        self.records.len()
786    }
787
788    /// `true` when the batch has no records.
789    #[must_use]
790    pub fn is_empty(&self) -> bool {
791        self.records.is_empty()
792    }
793
794    /// Header line of record `i` (without `>` and line ending).
795    #[must_use]
796    pub fn header(&self, i: usize) -> &[u8] {
797        let r = &self.records[i];
798        let h = &self.data[r.range.start + 1..r.range.start + r.header_end];
799        h.strip_suffix(b"\r").unwrap_or(h)
800    }
801
802    /// Sequence bytes of record `i`, line endings included, and the offset of its first byte
803    /// relative to the record start.
804    #[must_use]
805    pub fn sequence(&self, i: usize) -> (&[u8], usize) {
806        let r = &self.records[i];
807        let start = (r.range.start + r.header_end + 1).min(r.range.end);
808        (&self.data[start..r.range.end], start - r.range.start)
809    }
810}
811
812/// Limits of the batches after the first.
813///
814/// The first batch is the probe (D40) and is always defined by the documented constants: up to
815/// 10,000 records, closing early once it reaches 64 MiB *and* holds at least 1,000 records (or
816/// the whole input). Later batches only bound memory, so changing these limits never changes a
817/// result.
818#[derive(Clone, Copy, Debug, PartialEq, Eq)]
819pub struct BatchLimits {
820    /// Maximum records per batch (at least 1).
821    pub max_records: usize,
822    /// A batch closes once its data reach this size.
823    pub max_bytes: usize,
824}
825
826impl Default for BatchLimits {
827    fn default() -> Self {
828        Self {
829            max_records: BATCH_MAX_RECORDS,
830            max_bytes: BATCH_MAX_BYTES,
831        }
832    }
833}
834
835/// seq_io buffer policy: double without a cap (records of any size are accepted).
836struct Doubling;
837
838impl seq_io::policy::BufPolicy for Doubling {
839    fn grow_to(&mut self, current_size: usize) -> Option<usize> {
840        current_size.checked_mul(2)
841    }
842}
843
844/// Splits the text into batches of records.
845pub struct BatchReader {
846    reader: seq_io::fasta::Reader<Box<dyn Read + Send>, Doubling>,
847    source: Source,
848    compression: Compression,
849    byte_base: u64,
850    /// Blank lines skipped before the parser's first line.
851    line_base: u64,
852    next_record: u64,
853    limits: BatchLimits,
854    finished: bool,
855    drain: Option<Tee>,
856}
857
858impl BatchReader {
859    fn new(
860        text: Box<dyn Read + Send>,
861        source: Source,
862        compression: Compression,
863        start: TextStart,
864        drain: Option<Tee>,
865    ) -> Self {
866        Self {
867            reader: seq_io::fasta::Reader::with_capacity(text, 1 << 16).set_policy(Doubling),
868            source,
869            compression,
870            byte_base: start.bytes,
871            line_base: start.lines,
872            next_record: 1,
873            limits: BatchLimits::default(),
874            finished: false,
875            drain,
876        }
877    }
878
879    /// Replaces the batch limits (tests use tiny batches to exercise batch boundaries).
880    #[must_use]
881    pub fn with_limits(mut self, limits: BatchLimits) -> Self {
882        self.limits = limits;
883        self
884    }
885
886    /// Records read so far.
887    #[must_use]
888    pub fn records_read(&self) -> u64 {
889        self.next_record - 1
890    }
891
892    /// `true` once the end of the input has been reached.
893    #[must_use]
894    pub const fn is_finished(&self) -> bool {
895        self.finished
896    }
897
898    /// The input's display name.
899    #[must_use]
900    pub fn source_name(&self) -> String {
901        self.source.display_name()
902    }
903
904    fn location(&self, line: u64, byte: u64) -> Location {
905        Location::new(self.source.display_name())
906            .line(line)
907            .byte(byte)
908    }
909
910    /// Reads the next batch into `batch` (cleared first). Returns `Ok(false)` once the input is
911    /// exhausted and every stored byte has been read (so the checksum is complete).
912    ///
913    /// # Errors
914    /// Structural FASTA errors ([`DimaError::InputFormat`]), I/O errors, corrupt compressed
915    /// data, an empty input, or [`DimaError::Cancelled`].
916    pub fn read_batch(&mut self, batch: &mut RawBatch, cancel: &AtomicBool) -> Result<bool> {
917        batch.clear();
918        batch.first_record = self.next_record;
919        if self.finished {
920            return Ok(false);
921        }
922        let is_first = self.next_record == 1;
923        loop {
924            if cancel.load(Ordering::Relaxed) {
925                return Err(DimaError::Cancelled);
926            }
927            let start = batch.data.len();
928            let Some(next) = self.reader.next() else {
929                self.finish_input()?;
930                break;
931            };
932            match next {
933                Ok(record) => {
934                    // `write_unchanged` copies the record as stored (adding a final line feed if
935                    // the file lacks one); writing to a Vec cannot fail.
936                    let _ = record.write_unchanged(&mut batch.data);
937                }
938                Err(e) => return Err(self.map_parse_error(e)),
939            }
940            let position = self
941                .reader
942                .position()
943                .ok_or_else(|| DimaError::internal("record without a position"))?;
944            let (line, byte) = (
945                position.line() + self.line_base,
946                position.byte() + self.byte_base,
947            );
948            let raw = &batch.data[start..];
949            let header_end = memchr::memchr(b'\n', raw).unwrap_or(raw.len());
950            self.check_header(&raw[..header_end], line, byte)?;
951            batch.records.push(RawRecord {
952                range: start..batch.data.len(),
953                header_end,
954                line,
955                byte,
956            });
957            self.next_record += 1;
958            let n = batch.records.len();
959            let (max_records, max_bytes, min_records) = if is_first {
960                (BATCH_MAX_RECORDS, BATCH_MAX_BYTES, PROBE_MIN_RECORDS)
961            } else {
962                (self.limits.max_records.max(1), self.limits.max_bytes, 1)
963            };
964            if n >= max_records || (batch.data.len() >= max_bytes && n >= min_records) {
965                break;
966            }
967        }
968        if batch.is_empty() {
969            if is_first {
970                return Err(DimaError::input_format(
971                    Location::new(self.source.display_name()),
972                    "the input contains no FASTA records",
973                )
974                .with_hint(
975                    "DiMA reads an aligned FASTA file: records start with a '>' header line",
976                ));
977            }
978            return Ok(false);
979        }
980        Ok(true)
981    }
982
983    fn check_header(&self, header_line: &[u8], line: u64, byte: u64) -> Result<()> {
984        let body = header_line.strip_suffix(b"\r").unwrap_or(header_line);
985        let at = |i: usize| {
986            self.location(line, byte + i as u64)
987                .record(self.next_record)
988        };
989        if let Some(i) = memchr::memchr(b'\r', body) {
990            // With classic Mac OS line endings (CR only) the whole file is its first line;
991            // elsewhere a CR is a stray character, and turning it into a line break would
992            // split the header.
993            let hint = if line == 1 {
994                "the file probably uses classic Mac OS line endings (CR only); convert them, \
995                 e.g. `tr '\\r' '\\n' < in.fasta > out.fasta`"
996            } else {
997                "remove the stray carriage return (0x0D) from the header, e.g. \
998                 `tr -d '\\r' < in.fasta > out.fasta`"
999            };
1000            return Err(
1001                DimaError::input_format(at(i), "carriage return inside a header line")
1002                    .with_hint(hint),
1003            );
1004        }
1005        if let Some(i) = memchr::memchr(0, body) {
1006            return Err(DimaError::input_format(at(i), "NUL byte in a header line")
1007                .with_hint("the file is binary or corrupt; FASTA is plain text"));
1008        }
1009        Ok(())
1010    }
1011
1012    /// At the end of the text: reads any stored bytes the decoder did not need, so the checksum
1013    /// covers the whole input.
1014    fn finish_input(&mut self) -> Result<()> {
1015        self.finished = true;
1016        if let Some(mut drain) = self.drain.take() {
1017            io::copy(&mut drain, &mut io::sink())
1018                .map_err(|e| map_stream_error(e, &self.source, Some(self.compression)))?;
1019        }
1020        Ok(())
1021    }
1022
1023    fn map_parse_error(&self, e: seq_io::fasta::Error) -> DimaError {
1024        match e {
1025            seq_io::fasta::Error::Io(e) => {
1026                map_stream_error(e, &self.source, Some(self.compression))
1027            }
1028            seq_io::fasta::Error::InvalidStart { line, found } => {
1029                let loc =
1030                    Location::new(self.source.display_name()).line(line as u64 + self.line_base);
1031                if found == b'@' {
1032                    DimaError::input_format(loc, "this looks like FASTQ ('@' header)").with_hint(
1033                        "DiMA analyses aligned FASTA; align the sequences and save them as FASTA",
1034                    )
1035                } else {
1036                    DimaError::input_format(
1037                        loc,
1038                        format!(
1039                            "expected a '>' header line, found {}",
1040                            crate::normalise::describe_byte(found)
1041                        ),
1042                    )
1043                    .with_hint(start_hint(found, None))
1044                }
1045            }
1046            seq_io::fasta::Error::BufferLimit => {
1047                DimaError::internal("record buffer limit reached (no limit should apply)")
1048            }
1049        }
1050    }
1051}
1052
1053/// Reads a whole regular file's checksum without parsing it (used by tests and by the GUI's
1054/// change detection).
1055///
1056/// # Errors
1057/// [`DimaError::Io`] if the file cannot be read.
1058pub fn sha256_file(path: &Path) -> Result<String> {
1059    let target = || IoTarget::Path(path.to_path_buf());
1060    let mut f = File::open(path).map_err(|e| DimaError::io(IoAction::Open, target(), e))?;
1061    let mut h = Sha256::new();
1062    let mut buf = vec![0u8; RAW_BUFFER];
1063    loop {
1064        let n = match f.read(&mut buf) {
1065            Ok(0) => break,
1066            Ok(n) => n,
1067            Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
1068            Err(e) => return Err(DimaError::io(IoAction::Read, target(), e)),
1069        };
1070        h.update(&buf[..n]);
1071    }
1072    Ok(h.finalize().iter().map(|b| format!("{b:02x}")).collect())
1073}
1074
1075/// `(header, sequence bytes)` pairs.
1076#[cfg(test)]
1077pub(crate) type TestRecords = Vec<(Vec<u8>, Vec<u8>)>;
1078
1079/// Reads the whole input into memory as a single list of records (header, sequence bytes);
1080/// for tests and small inspections only.
1081#[cfg(test)]
1082pub(crate) fn read_all(source: &Source) -> Result<(TestRecords, InputInfo)> {
1083    let progress = Arc::new(Progress::new());
1084    let mut opened = open(source, &progress)?;
1085    let cancel = AtomicBool::new(false);
1086    let mut out = Vec::new();
1087    let mut batch = RawBatch::default();
1088    while opened.batches.read_batch(&mut batch, &cancel)? {
1089        for i in 0..batch.len() {
1090            out.push((batch.header(i).to_vec(), batch.sequence(i).0.to_vec()));
1091        }
1092    }
1093    let info = opened.finish()?;
1094    Ok((out, info))
1095}
1096
1097#[cfg(test)]
1098mod tests {
1099    use super::*;
1100    use std::io::Write;
1101
1102    fn tmp(bytes: &[u8], name: &str) -> (tempfile::TempDir, PathBuf) {
1103        let dir = tempfile::tempdir().unwrap();
1104        let p = dir.path().join(name);
1105        std::fs::write(&p, bytes).unwrap();
1106        (dir, p)
1107    }
1108
1109    fn sha(bytes: &[u8]) -> String {
1110        Sha256::digest(bytes)
1111            .iter()
1112            .map(|b| format!("{b:02x}"))
1113            .collect()
1114    }
1115
1116    const FASTA: &[u8] = b">a|x\nACGT\nAC\n>b|y\r\nACGTAC\r\n>c\nAC GT-A\n";
1117
1118    fn gzip(data: &[u8]) -> Vec<u8> {
1119        let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
1120        e.write_all(data).unwrap();
1121        e.finish().unwrap()
1122    }
1123
1124    fn bzip(data: &[u8]) -> Vec<u8> {
1125        let mut e = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::default());
1126        e.write_all(data).unwrap();
1127        e.finish().unwrap()
1128    }
1129
1130    fn zst(data: &[u8]) -> Vec<u8> {
1131        zstd::encode_all(data, 3).unwrap()
1132    }
1133
1134    #[test]
1135    fn records_and_checksums_for_every_compression() {
1136        // Each format, and two concatenated streams/members/frames of each.
1137        let half = FASTA.len() / 2;
1138        let cases: Vec<(&str, Vec<u8>, Compression)> = vec![
1139            ("plain", FASTA.to_vec(), Compression::None),
1140            ("gz", gzip(FASTA), Compression::Gzip),
1141            (
1142                "gz2",
1143                [gzip(&FASTA[..half]), gzip(&FASTA[half..])].concat(),
1144                Compression::Gzip,
1145            ),
1146            (
1147                "gz-padded",
1148                [gzip(FASTA), vec![0; 5000]].concat(),
1149                Compression::Gzip,
1150            ),
1151            ("bz2", bzip(FASTA), Compression::Bzip2),
1152            (
1153                "bz2x2",
1154                [bzip(&FASTA[..half]), bzip(&FASTA[half..])].concat(),
1155                Compression::Bzip2,
1156            ),
1157            (
1158                "bz2-padded",
1159                [bzip(FASTA), vec![0; 3]].concat(),
1160                Compression::Bzip2,
1161            ),
1162            ("zst", zst(FASTA), Compression::Zstd),
1163            (
1164                "zstx2",
1165                [zst(&FASTA[..half]), zst(&FASTA[half..])].concat(),
1166                Compression::Zstd,
1167            ),
1168        ];
1169        for (name, bytes, compression) in cases {
1170            let (_d, p) = tmp(&bytes, name);
1171            let (records, info) = read_all(&Source::Path(p)).unwrap();
1172            assert_eq!(info.compression, compression, "{name}");
1173            assert_eq!(info.size_bytes, bytes.len() as u64, "{name}");
1174            assert_eq!(info.checksum.value, sha(&bytes), "{name}");
1175            assert_eq!(info.name, name);
1176            let heads: Vec<_> = records.iter().map(|r| r.0.clone()).collect();
1177            assert_eq!(
1178                heads,
1179                [b"a|x".to_vec(), b"b|y".to_vec(), b"c".to_vec()],
1180                "{name}"
1181            );
1182            assert_eq!(records[0].1, b"ACGT\nAC\n", "{name}");
1183            assert_eq!(records[1].1, b"ACGTAC\r\n", "{name}");
1184            assert_eq!(records[2].1, b"AC GT-A\n", "{name}");
1185        }
1186    }
1187
1188    #[test]
1189    fn zstd_skippable_frame_and_long_window() {
1190        let mut skippable = vec![0x50, 0x2A, 0x4D, 0x18, 4, 0, 0, 0, 1, 2, 3, 4];
1191        skippable.extend(zst(FASTA));
1192        let (_d, p) = tmp(&skippable, "s.zst");
1193        let (records, info) = read_all(&Source::Path(p)).unwrap();
1194        assert_eq!((records.len(), info.compression), (3, Compression::Zstd));
1195        let mut enc = zstd::stream::write::Encoder::new(Vec::new(), 3).unwrap();
1196        enc.long_distance_matching(true).unwrap();
1197        enc.window_log(31).unwrap();
1198        enc.write_all(FASTA).unwrap();
1199        let (_d, p) = tmp(&enc.finish().unwrap(), "l.zst");
1200        assert_eq!(read_all(&Source::Path(p)).unwrap().0.len(), 3);
1201    }
1202
1203    #[test]
1204    fn bom_and_encodings() {
1205        let mut with_bom = vec![0xEF, 0xBB, 0xBF];
1206        with_bom.extend_from_slice(FASTA);
1207        let (_d, p) = tmp(&gzip(&with_bom), "bom.gz");
1208        let (records, _) = read_all(&Source::Path(p)).unwrap();
1209        assert_eq!(records[0].0, b"a|x");
1210        for (bad, encoding) in [
1211            (&[0xFF, 0xFE, b'>', 0][..], "UTF-16"),
1212            (&[0xFE, 0xFF, 0, b'>'], "UTF-16"),
1213            (&[0, 0, 0xFE, 0xFF], "UTF-32"),
1214            (&[0xFF, 0xFE, 0, 0], "UTF-32"),
1215        ] {
1216            let (_d, p) = tmp(bad, "u16");
1217            let e = read_all(&Source::Path(p)).err().unwrap();
1218            assert!(
1219                e.to_string().contains(&format!("is {encoding} encoded")),
1220                "{e}"
1221            );
1222            assert!(e.hint().unwrap().contains(&format!("iconv -f {encoding}")));
1223        }
1224    }
1225
1226    #[test]
1227    fn unsupported_formats_are_named() {
1228        for (bytes, what) in [
1229            (&[0x1F, 0x9D, 0x90, 0x3E][..], ".Z"),
1230            (&[0x04, 0x22, 0x4D, 0x18, 0, 0], "LZ4"),
1231            (&[0x5D, 0, 0, 0x80, 0, 0], "LZMA"),
1232            (b"PK\x03\x04abc", "zip"),
1233            (&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C], "7z"),
1234            (&[0xFD, b'7', b'z', b'X', b'Z', 0x00, 0x00, 0x04], "xz"),
1235        ] {
1236            let (_d, p) = tmp(bytes, "x");
1237            let e = read_all(&Source::Path(p)).err().unwrap();
1238            assert!(e.to_string().contains(what), "{e}");
1239            assert_eq!(e.class(), crate::error::ExitClass::InvalidInput);
1240            let hint = e.hint().unwrap();
1241            assert!(
1242                hint.ends_with("DiMA reads plain, gzip/BGZF, bzip2 and Zstandard FASTA"),
1243                "{hint}"
1244            );
1245        }
1246        // xz is refused with the ways to read it (D116).
1247        let (_d, p) = tmp(&[0xFD, b'7', b'z', b'X', b'Z', 0x00], "a.fasta.xz");
1248        let e = read_all(&Source::Path(p)).err().unwrap();
1249        assert!(
1250            e.to_string()
1251                .contains("the input is xz (.xz), which DiMA cannot read"),
1252            "{e}"
1253        );
1254        assert!(e.hint().unwrap().contains("`xz -dk NAME.xz`"));
1255        assert!(e.hint().unwrap().contains("zstd"));
1256    }
1257
1258    #[test]
1259    fn structural_errors() {
1260        let check = |bytes: &[u8], needle: &str| {
1261            let (_d, p) = tmp(bytes, "f.fa");
1262            let e = read_all(&Source::Path(p))
1263                .err()
1264                .unwrap_or_else(|| panic!("{needle}: no error"));
1265            assert!(e.to_string().contains(needle), "{needle}: {e}");
1266            assert_eq!(e.class(), crate::error::ExitClass::InvalidInput, "{needle}");
1267            e
1268        };
1269        let e = check(b"", "no FASTA records");
1270        assert!(e.hint().is_some());
1271        // Blank and white-space-only lines before the first record are skipped.
1272        let e = check(b"\n\n  \n\t\x0B\x0C\r\n \t", "no FASTA records");
1273        assert!(e.hint().unwrap().contains("'>' header line"), "{e}");
1274        let e = check(b"ACGT\n>a\nACGT\n", "expected a '>' header line");
1275        assert!(e.to_string().contains("line 1"));
1276        check(b"@r1\nACGT\n+\nIIII\n", "FASTQ");
1277        let e = check(b">a\rACGT\r>b\rACGT\r", "carriage return inside a header");
1278        assert!(e.hint().unwrap().contains("classic Mac OS"), "{e}");
1279        let e = check(
1280            b">a\nACGT\n>b\nACGT\n>c\rd\nACGT\n",
1281            "carriage return inside a header",
1282        );
1283        assert!(e.to_string().contains("record 3"), "{e}");
1284        assert!(e.hint().unwrap().contains("tr -d"), "{e}");
1285        let e = check(b">a\x00b\nACGT\n", "NUL byte in a header");
1286        assert!(e.to_string().contains("record 1"), "{e}");
1287        let e = check(b" \n\tx\n>a\nACGT\n", "found 'x'");
1288        assert!(e.to_string().contains("line 2, column 2, byte 3"), "{e}");
1289        // The byte is shown plainly, never escaped (the path before it may hold backslashes).
1290        let text = e.to_string();
1291        assert!(!text[text.find("found").unwrap()..].contains('\\'), "{e}");
1292        let e = check(
1293            b"\n  >a\nACGT\n",
1294            "white space before the first header's '>'",
1295        );
1296        assert!(e.to_string().contains("line 2, column 3, byte 3"), "{e}");
1297        assert!(e.hint().unwrap().contains("remove the white space"), "{e}");
1298        // A no-break space is not white space; at a line start or after white space.
1299        for text in [&b"\xC2\xA0>a\nAC\n"[..], b" \xC2\xA0>a\nAC\n"] {
1300            let e = check(text, "found 0xC2");
1301            assert!(e.hint().unwrap().contains("no-break space"), "{e}");
1302        }
1303        let e = check(b"\x00\x00", "found 0x00");
1304        assert!(e.hint().unwrap().contains("binary"), "{e}");
1305        let truncated = gzip(FASTA);
1306        let e = check(&truncated[..truncated.len() - 8], "gzip data");
1307        assert!(e.hint().unwrap().contains("gzip -t"));
1308        // Damage found while decoding the first block names the compression too.
1309        let mut bz = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::best());
1310        bz.write_all(FASTA).unwrap();
1311        let bz = bz.finish().unwrap();
1312        let e = check(&bz[..bz.len() / 2], "truncated bzip2 data");
1313        assert!(e.hint().unwrap().contains("bzip2 -t"), "{e}");
1314        let zst = zstd::encode_all(FASTA, 3).unwrap();
1315        let e = check(&zst[..zst.len() / 2], "Zstandard data");
1316        assert!(e.hint().unwrap().contains("zstd -t"), "{e}");
1317        let e = check(&[0x1F, 0x8B], "truncated gzip data");
1318        assert!(e.hint().unwrap().contains("gzip -t"), "{e}");
1319        // UTF-32 is named as such (UTF-16's conversion would not help).
1320        let mut utf32 = vec![0xFF, 0xFE, 0x00, 0x00];
1321        utf32.extend(">a\nAC\n".bytes().flat_map(|b| [b, 0, 0, 0]));
1322        let e = check(&utf32, "UTF-32");
1323        assert!(e.hint().unwrap().contains("iconv -f UTF-32"), "{e}");
1324        let e = check(&[0xFE, 0xFF, 0, b'>'], "UTF-16");
1325        assert!(e.hint().unwrap().contains("iconv -f UTF-16"), "{e}");
1326        // Data after the compressed data that is not zero padding.
1327        let mut junk = gzip(FASTA);
1328        junk.extend(b"\0\0junk");
1329        let e = check(&junk, "6 bytes that are not gzip data");
1330        assert!(e.hint().unwrap().contains("gzip -dc"), "{e}");
1331    }
1332
1333    #[test]
1334    fn missing_input_and_directories() {
1335        let e = read_all(&Source::Path("/nonexistent/x.fa".into()))
1336            .err()
1337            .unwrap();
1338        assert_eq!(e.class(), crate::error::ExitClass::Io);
1339        assert!(e.to_string().contains("cannot open '/nonexistent/x.fa'"));
1340        let dir = tempfile::tempdir().unwrap();
1341        let e = read_all(&Source::Path(dir.path().to_path_buf()))
1342            .err()
1343            .unwrap();
1344        assert_eq!(e.class(), crate::error::ExitClass::Usage);
1345    }
1346
1347    #[test]
1348    fn locations_after_many_blank_lines() {
1349        // More blank lines than one parser buffer holds (64 KiB), LF and CR LF.
1350        for (blank, width) in [(&b"\n"[..], 1u64), (&b"\r\n"[..], 2)] {
1351            let mut text = blank.repeat(70_000);
1352            text.extend_from_slice(b">a\nACGTACGTACGT\n>b\nACGTACGTAC1T\n");
1353            let (_d, p) = tmp(&text, "blank.fa");
1354            let mut opened = open(&Source::Path(p), &Arc::new(Progress::new())).unwrap();
1355            let mut batch = RawBatch::default();
1356            opened
1357                .batches
1358                .read_batch(&mut batch, &AtomicBool::new(false))
1359                .unwrap();
1360            let lines: Vec<u64> = batch.records.iter().map(|r| r.line).collect();
1361            let bytes: Vec<u64> = batch.records.iter().map(|r| r.byte).collect();
1362            assert_eq!(lines, [70_001, 70_003]);
1363            assert_eq!(bytes, [70_000 * width, 70_000 * width + 16]);
1364        }
1365        // White-space-only lines, and one line of white space longer than a buffer: counted
1366        // exactly, never kept.
1367        for (lines, count, width) in [
1368            (&b" \t\x0B\x0C\r\n"[..], 70_000u64, 6u64),
1369            (&[b' '; 100_000][..], 1, 100_000),
1370        ] {
1371            let mut text = lines.repeat(usize::try_from(count).unwrap());
1372            if count == 1 {
1373                text.push(b'\n');
1374            }
1375            text.extend_from_slice(b">a\nACGT\n");
1376            let (_d, p) = tmp(&text, "ws.fa");
1377            let mut opened = open(&Source::Path(p), &Arc::new(Progress::new())).unwrap();
1378            let mut batch = RawBatch::default();
1379            opened
1380                .batches
1381                .read_batch(&mut batch, &AtomicBool::new(false))
1382                .unwrap();
1383            let skipped = if count == 1 { width + 1 } else { count * width };
1384            assert_eq!(
1385                (batch.records[0].line, batch.records[0].byte),
1386                (count + 1, skipped)
1387            );
1388        }
1389        let mut text = vec![b' '; 100_000];
1390        text.extend_from_slice(b"x\n>a\nAC\n");
1391        let (_d, p) = tmp(&text, "ws.fa");
1392        let e = read_all(&Source::Path(p)).err().unwrap();
1393        assert!(
1394            e.to_string().contains("line 1, column 100001, byte 100000"),
1395            "{e}"
1396        );
1397        // Blank lines only: no records; a lone CR before content is refused.
1398        let (_d, p) = tmp(&b"\n".repeat(70_000), "empty.fa");
1399        let e = read_all(&Source::Path(p)).err().unwrap();
1400        assert!(e.to_string().contains("no FASTA records"), "{e}");
1401        let (_d, p) = tmp(b"\n\r>a\nAC\n", "cr.fa");
1402        let e = read_all(&Source::Path(p)).err().unwrap();
1403        assert!(e.to_string().contains("line 2"), "{e}");
1404        assert!(e.hint().unwrap().contains("classic Mac OS"), "{e}");
1405        // Classic Mac OS line endings are found at the start, not after the whole file.
1406        for (start, at) in [
1407            (&b"\xEF\xBB\xBF\n>a\r"[..], "record 1 (line 2, byte 6)"),
1408            (b">MERS_spike|Camel\r", "record 1 (line 1, byte 17)"),
1409        ] {
1410            let mut mac = start.to_vec();
1411            mac.extend(b"ACGT\r".repeat(100_000));
1412            let (_d, p) = tmp(&mac, "mac.fa");
1413            let e = read_all(&Source::Path(p)).err().unwrap();
1414            assert!(e.to_string().contains(at), "{e}");
1415            assert!(e.hint().unwrap().contains("classic Mac OS"), "{e}");
1416        }
1417    }
1418
1419    #[test]
1420    fn positions_and_batches() {
1421        // 10,000 records form the first batch (the probe); later batches follow the limits.
1422        let mut text = Vec::new();
1423        for i in 0..10_007 {
1424            write!(text, ">r{i}\r\nAC\r\nGT\r\n").unwrap();
1425        }
1426        let (_d, p) = tmp(&text, "b.fa");
1427        let progress = Arc::new(Progress::new());
1428        let opened = open(&Source::Path(p), &progress).unwrap();
1429        let mut reader = opened.batches.with_limits(BatchLimits {
1430            max_records: 3,
1431            max_bytes: usize::MAX,
1432        });
1433        let cancel = AtomicBool::new(false);
1434        let mut batch = RawBatch::default();
1435        let mut sizes = Vec::new();
1436        let mut lines = Vec::new();
1437        while reader.read_batch(&mut batch, &cancel).unwrap() {
1438            sizes.push((batch.first_record, batch.len()));
1439            for (i, r) in batch.records.iter().enumerate() {
1440                lines.push((r.line, r.byte));
1441                let n = batch.first_record as usize - 1 + i;
1442                assert_eq!(batch.header(i), format!("r{n}").as_bytes());
1443                assert_eq!(batch.sequence(i).0, b"AC\r\nGT\r\n");
1444            }
1445        }
1446        assert_eq!(sizes, [(1, 10_000), (10_001, 3), (10_004, 3), (10_007, 1)]);
1447        assert_eq!(lines[0], (1, 0));
1448        assert_eq!(lines[1], (4, 13));
1449        assert_eq!(reader.records_read(), 10_007);
1450        assert!(reader.is_finished());
1451        // Cancellation stops reading.
1452        let (_d, p) = tmp(FASTA, "c.fa");
1453        let mut opened = open(&Source::Path(p), &progress).unwrap();
1454        cancel.store(true, Ordering::Relaxed);
1455        assert!(matches!(
1456            opened.batches.read_batch(&mut batch, &cancel),
1457            Err(DimaError::Cancelled)
1458        ));
1459    }
1460
1461    #[test]
1462    fn first_batch_is_the_probe_whatever_the_limits() {
1463        let mut text = Vec::new();
1464        for i in 0..1_050 {
1465            write!(text, ">r{i}\n{}\n", "A".repeat(100)).unwrap();
1466        }
1467        let (_d, p) = tmp(&text, "m.fa");
1468        let progress = Arc::new(Progress::new());
1469        let opened = open(&Source::Path(p), &progress).unwrap();
1470        let mut reader = opened.batches.with_limits(BatchLimits {
1471            max_records: 1000,
1472            max_bytes: 10,
1473        });
1474        let cancel = AtomicBool::new(false);
1475        let mut batch = RawBatch::default();
1476        assert!(reader.read_batch(&mut batch, &cancel).unwrap());
1477        assert_eq!(
1478            batch.len(),
1479            1_050,
1480            "the probe ignores the limits of later batches"
1481        );
1482    }
1483
1484    #[test]
1485    fn query_names() {
1486        let q = |s: &str| query_name_from_source(&Source::Path(s.into()));
1487        assert_eq!(
1488            q("data/HIV1_FLT_2022_pol_DNA.fasta.bz2").as_deref(),
1489            Some("HIV1_FLT_2022_pol_DNA")
1490        );
1491        assert_eq!(q("x/MERS.Spike.FA.GZ").as_deref(), Some("MERS.Spike"));
1492        assert_eq!(q("a.afa").as_deref(), Some("a"));
1493        assert_eq!(q("noext").as_deref(), Some("noext"));
1494        assert_eq!(q(".fasta"), None);
1495        assert_eq!(q("x.fa.fasta.gz").as_deref(), Some("x"));
1496        // Recompressed from xz as the hint for xz input suggests.
1497        assert_eq!(q("spike.fasta.xz.zst").as_deref(), Some("spike"));
1498        assert_eq!(query_name_from_source(&Source::Stdin), None);
1499        assert_eq!(Source::Path("d/e/f.fa".into()).provenance_name(), "f.fa");
1500        assert_eq!(Source::Stdin.provenance_name(), "-");
1501        // Process substitution and descriptor paths are named like standard input: their
1502        // numbers are the shell's.
1503        for alias in [
1504            "/dev/fd/63",
1505            "/dev/stdin",
1506            "/proc/self/fd/0",
1507            "/proc/4242/fd/11",
1508        ] {
1509            let s = Source::Path(alias.into());
1510            assert_eq!(
1511                query_name_from_source(&s),
1512                if cfg!(unix) {
1513                    None
1514                } else {
1515                    Some(alias.rsplit('/').next().unwrap().to_owned())
1516                },
1517                "{alias}"
1518            );
1519            if cfg!(unix) {
1520                assert_eq!(s.provenance_name(), "-", "{alias}");
1521                assert_eq!(
1522                    crate::run_report::file_name(std::path::Path::new(alias)),
1523                    "-"
1524                );
1525            }
1526        }
1527        assert_eq!(
1528            q("/dev/fdx/63.fa"),
1529            Some("63".into()),
1530            "only the descriptor paths"
1531        );
1532    }
1533
1534    #[test]
1535    fn sha256_of_file() {
1536        let (_d, p) = tmp(b"abc", "h");
1537        assert_eq!(
1538            sha256_file(&p).unwrap(),
1539            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
1540        );
1541    }
1542
1543    #[cfg(unix)]
1544    #[test]
1545    fn fifo_input() {
1546        let dir = tempfile::tempdir().unwrap();
1547        let p = dir.path().join("pipe");
1548        let status = std::process::Command::new("mkfifo")
1549            .arg(&p)
1550            .status()
1551            .unwrap();
1552        assert!(status.success());
1553        let writer_path = p.clone();
1554        let data = gzip(FASTA);
1555        let expected = sha(&data);
1556        let t = thread::spawn(move || {
1557            let mut f = std::fs::OpenOptions::new()
1558                .write(true)
1559                .open(writer_path)
1560                .unwrap();
1561            // Short writes exercise the sniffing loop on pipes.
1562            for chunk in data.chunks(3) {
1563                f.write_all(chunk).unwrap();
1564                f.flush().unwrap();
1565            }
1566        });
1567        let (records, info) = read_all(&Source::Path(p)).unwrap();
1568        t.join().unwrap();
1569        assert_eq!(records.len(), 3);
1570        assert_eq!(info.checksum.value, expected);
1571        assert_eq!(info.compression, Compression::Gzip);
1572    }
1573}