Skip to main content

dima_core/
header.rs

1//! FASTA header metadata: `|`-separated fields named by `--header-format` (Tharanga et al.
2//! 2025, "Metadata"), cleaned once and interned so the engine only handles small integers.
3//!
4//! * **Format.** Field names are trimmed and NFC-normalised, and must be non-empty and unique;
5//!   `--metadata-fields` (default: all fields) must name a subset of them.
6//! * **Split first.** The whole header line after `>` is split on `|` *before* any cleaning,
7//!   so cleaning can never create or remove a delimiter.
8//! * **Encoding (D48).** Invalid UTF-8 is an error in strict mode (with the record and byte);
9//!   permissive and report modes replace it with U+FFFD and count the replacements.
10//! * **Value cleaning (D59).** Remove format characters (General_Category Cf, e.g. zero-width
11//!   spaces and bidirectional marks) and control characters except TAB; trim Unicode
12//!   White_Space; normalise to NFC (UAX #15; W3C charmod-norm); an empty value becomes the fill
13//!   value. Interior text is never changed. Every change is counted.
14//! * **Mismatches (D18).** A header whose field count differs from the format is an error in
15//!   strict mode (all headers are read first and the first 20 offenders are listed); in
16//!   permissive and report modes its values become the fill value and a diagnostic is raised.
17//! * **Diagnostics.** Near-duplicate values (equal after case folding, whitespace collapsing and
18//!   NFKC), missing-like tokens (kept literally; INSDC missing-value vocabulary and common NA
19//!   spellings) and identifier-like fields (every value unique).
20
21use std::collections::{BTreeMap, HashMap};
22
23use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
24use unicode_properties::{GeneralCategory, UnicodeGeneralCategory};
25
26use crate::config::{ID_DIAG_MIN_RECORDS, MISMATCH_LOG_MAX, ValidationMode};
27use crate::diagnostics::{Code, Diagnostic, Tally, agree, plural};
28use crate::error::{DimaError, Location, Result};
29use crate::model::{CodePointCount, HeaderCleaning, HeaderMismatch};
30
31/// Splits a `--header-format` string such as `Accession|Host|Year` into names.
32#[must_use]
33pub fn split_format(text: &str) -> Vec<String> {
34    text.split('|').map(str::to_owned).collect()
35}
36
37/// A field name or fill value as used: trimmed and NFC-normalised.
38pub(crate) fn clean_name(name: &str) -> String {
39    name.trim().nfc().collect()
40}
41
42/// Validated header format: field names, stratified fields and fill value.
43#[derive(Clone, Debug, PartialEq, Eq)]
44pub struct HeaderSpec {
45    fields: Vec<String>,
46    stratified: Vec<usize>,
47    fill: String,
48}
49
50impl HeaderSpec {
51    /// Validates the format, the stratified subset and the fill value.
52    ///
53    /// # Errors
54    /// A usage error naming the offending entry (and the valid names) when a name is empty or
55    /// repeated, a metadata field is not in the format, or the fill value is empty.
56    pub fn new(format: &[String], metadata_fields: Option<&[String]>, fill: &str) -> Result<Self> {
57        if format.is_empty() {
58            return Err(DimaError::usage("the header format names no fields"));
59        }
60        let fields: Vec<String> = format.iter().map(|f| clean_name(f)).collect();
61        for (i, f) in fields.iter().enumerate() {
62            if f.is_empty() {
63                return Err(
64                    DimaError::usage(format!("header format field {} has no name", i + 1))
65                        .with_hint(
66                            "name every '|'-separated field, e.g. 'Accession|Host|Country|Year'",
67                        ),
68                );
69            }
70            if fields[..i].contains(f) {
71                return Err(DimaError::usage(format!(
72                    "header format field {:?} appears more than once",
73                    f
74                ))
75                .with_hint("give every field a distinct name"));
76            }
77        }
78        let stratified = match metadata_fields {
79            None => (0..fields.len()).collect(),
80            Some(names) => {
81                if names.is_empty() {
82                    return Err(DimaError::usage("no metadata fields were given"));
83                }
84                let mut idx = Vec::with_capacity(names.len());
85                for n in names {
86                    let n = clean_name(n);
87                    let Some(i) = fields.iter().position(|f| *f == n) else {
88                        return Err(DimaError::usage(format!(
89                            "metadata field {n:?} is not in the header format"
90                        ))
91                        .with_hint(format!("the header format names: {}", fields.join(", "))));
92                    };
93                    if idx.contains(&i) {
94                        return Err(DimaError::usage(format!(
95                            "metadata field {n:?} is listed more than once"
96                        )));
97                    }
98                    idx.push(i);
99                }
100                // Keep the header-format order regardless of the order given.
101                idx.sort_unstable();
102                idx
103            }
104        };
105        let fill = clean_name(fill);
106        if fill.is_empty() {
107            return Err(DimaError::usage("the header fill value must not be empty")
108                .with_hint("empty values are what the fill value replaces; use e.g. Unknown"));
109        }
110        Ok(Self {
111            fields,
112            stratified,
113            fill,
114        })
115    }
116
117    /// Field names, in header order.
118    #[must_use]
119    pub fn fields(&self) -> &[String] {
120        &self.fields
121    }
122
123    /// Indices (into [`HeaderSpec::fields`]) of the stratified fields, in header order.
124    #[must_use]
125    pub fn stratified(&self) -> &[usize] {
126        &self.stratified
127    }
128
129    /// Names of the stratified fields, in header order.
130    #[must_use]
131    pub fn stratified_names(&self) -> Vec<String> {
132        self.stratified
133            .iter()
134            .map(|&i| self.fields[i].clone())
135            .collect()
136    }
137
138    /// The fill value.
139    #[must_use]
140    pub fn fill(&self) -> &str {
141        &self.fill
142    }
143}
144
145/// Cleaning counters (D59), mergeable in input order.
146#[derive(Clone, Debug, Default, PartialEq, Eq)]
147pub struct CleanStats {
148    /// Values with leading or trailing white space.
149    pub trimmed: u64,
150    /// Values changed by NFC.
151    pub nfc_changed: u64,
152    /// Stratified values replaced by the fill value: empty values, and every stratified field of
153    /// a header with the wrong number of fields.
154    pub fill_uses: u64,
155    /// Invisible code points removed, by code point.
156    pub invisible: BTreeMap<char, u64>,
157}
158
159impl CleanStats {
160    /// Adds counts of later input.
161    pub fn merge(&mut self, other: Self) {
162        self.trimmed += other.trimmed;
163        self.nfc_changed += other.nfc_changed;
164        self.fill_uses += other.fill_uses;
165        for (c, n) in other.invisible {
166            *self.invisible.entry(c).or_default() += n;
167        }
168    }
169
170    /// `true` when nothing was changed.
171    #[must_use]
172    pub fn is_empty(&self) -> bool {
173        self.trimmed == 0
174            && self.nfc_changed == 0
175            && self.fill_uses == 0
176            && self.invisible.is_empty()
177    }
178}
179
180/// `true` for code points removed from header values: format characters (Cf) and controls (Cc)
181/// other than TAB.
182fn is_invisible(c: char) -> bool {
183    (c.is_control() && c != '\t') || c.general_category() == GeneralCategory::Format
184}
185
186/// Cleans one header value (D59): remove Cf and Cc-except-TAB, trim White_Space, NFC; an empty
187/// result becomes `fill`.
188pub fn clean_value(raw: &str, fill: &str, stats: &mut CleanStats) -> String {
189    // Fast path: printable ASCII needs at most trimming.
190    if raw.bytes().all(|b| (0x20..0x7F).contains(&b)) {
191        let t = raw.trim();
192        if t.len() != raw.len() {
193            stats.trimmed += 1;
194        }
195        if t.is_empty() {
196            stats.fill_uses += 1;
197            return fill.to_owned();
198        }
199        return t.to_owned();
200    }
201    let mut visible = String::with_capacity(raw.len());
202    for c in raw.chars() {
203        if is_invisible(c) {
204            *stats.invisible.entry(c).or_default() += 1;
205        } else {
206            visible.push(c);
207        }
208    }
209    let t = visible.trim();
210    if t.len() != visible.len() {
211        stats.trimmed += 1;
212    }
213    let value = if is_nfc_quick(t.chars()) == IsNormalized::Yes {
214        t.to_owned()
215    } else {
216        let n: String = t.nfc().collect();
217        if n != t {
218            stats.nfc_changed += 1;
219        }
220        n
221    };
222    if value.is_empty() {
223        stats.fill_uses += 1;
224        return fill.to_owned();
225    }
226    value
227}
228
229/// Header-processing counters for one batch (or the whole run), mergeable in input order.
230#[derive(Clone, Debug, Default, PartialEq, Eq)]
231pub struct HeaderStats {
232    /// Headers processed.
233    pub records: u64,
234    /// Invalid UTF-8 sequences replaced by U+FFFD.
235    pub utf8_replacements: u64,
236    /// Value cleaning.
237    pub cleaning: CleanStats,
238    /// Headers whose field count differs from the format.
239    pub mismatch_count: u64,
240    /// The first mismatches, in input order.
241    pub mismatches: Vec<HeaderMismatch>,
242}
243
244impl HeaderStats {
245    /// Adds counters of later input.
246    pub fn merge(&mut self, other: Self) {
247        self.records += other.records;
248        self.utf8_replacements += other.utf8_replacements;
249        self.cleaning.merge(other.cleaning);
250        self.mismatch_count += other.mismatch_count;
251        let room = MISMATCH_LOG_MAX.saturating_sub(self.mismatches.len());
252        self.mismatches
253            .extend(other.mismatches.into_iter().take(room));
254    }
255}
256
257/// Where a header is in the input, for messages.
258#[derive(Clone, Copy, Debug)]
259pub struct HeaderPosition<'a> {
260    /// Input name.
261    pub source: &'a str,
262    /// 1-based record number.
263    pub record: u64,
264    /// 1-based line number.
265    pub line: u64,
266    /// Byte offset of the `>`.
267    pub byte: u64,
268}
269
270/// Parses one header (the bytes after `>`) into the cleaned values of the stratified fields.
271///
272/// Returns the fill value for every stratified field when the field count does not match
273/// (recording the mismatch); the caller turns recorded mismatches into an error in strict mode
274/// once all headers have been read.
275///
276/// # Errors
277/// [`DimaError::InputData`] for invalid UTF-8 in strict mode.
278pub fn parse_header(
279    header: &[u8],
280    spec: &HeaderSpec,
281    mode: ValidationMode,
282    at: HeaderPosition<'_>,
283    stats: &mut HeaderStats,
284) -> Result<Vec<String>> {
285    stats.records += 1;
286    let text = match std::str::from_utf8(header) {
287        Ok(t) => std::borrow::Cow::Borrowed(t),
288        Err(e) => {
289            if mode == ValidationMode::Strict {
290                let offset = at.byte + 1 + e.valid_up_to() as u64;
291                return Err(DimaError::input_data(
292                    Some(
293                        Location::new(at.source)
294                            .record(at.record)
295                            .line(at.line)
296                            .byte(offset),
297                    ),
298                    "header is not valid UTF-8",
299                )
300                .with_hint(
301                    "convert the file to UTF-8, e.g. `iconv -f LATIN1 -t UTF-8 in.fasta > \
302                     out.fasta`, or use --validation permissive to replace invalid bytes",
303                ));
304            }
305            stats.utf8_replacements += header
306                .utf8_chunks()
307                .filter(|c| !c.invalid().is_empty())
308                .count() as u64;
309            String::from_utf8_lossy(header)
310        }
311    };
312    let parts: Vec<&str> = text.split('|').collect();
313    if parts.len() != spec.fields.len() {
314        stats.mismatch_count += 1;
315        if stats.mismatches.len() < MISMATCH_LOG_MAX {
316            stats.mismatches.push(HeaderMismatch {
317                record: at.record,
318                line: at.line,
319                header: text.clone().into_owned(),
320                expected: spec.fields.len() as u64,
321                found: parts.len() as u64,
322            });
323        }
324        stats.cleaning.fill_uses += spec.stratified.len() as u64;
325        return Ok(vec![spec.fill.clone(); spec.stratified.len()]);
326    }
327    Ok(spec
328        .stratified
329        .iter()
330        .map(|&i| clean_value(parts[i], &spec.fill, &mut stats.cleaning))
331        .collect())
332}
333
334/// The error raised in strict mode when headers do not match the format.
335#[must_use]
336pub fn mismatch_error(source: &str, spec: &HeaderSpec, stats: &HeaderStats) -> DimaError {
337    let mut msg = format!(
338        "{} not have the {} of the header format ({})",
339        plural(stats.mismatch_count, "header does", "headers do"),
340        plural(spec.fields.len() as u64, "field", "fields"),
341        spec.fields.join("|")
342    );
343    for m in &stats.mismatches {
344        msg.push_str(&format!(
345            "\n  record {} (line {}): {}: {}",
346            m.record,
347            m.line,
348            plural(m.found, "field", "fields"),
349            truncate_for_display(&m.header, 120)
350        ));
351    }
352    if stats.mismatch_count > stats.mismatches.len() as u64 {
353        msg.push_str(&format!(
354            "\n  and {} more",
355            stats.mismatch_count - stats.mismatches.len() as u64
356        ));
357    }
358    DimaError::input_data(Some(Location::new(source)), msg).with_hint(
359        "fix the headers, change --header-format, or use --validation permissive to give \
360         mismatched headers the fill value",
361    )
362}
363
364fn truncate_for_display(s: &str, max_chars: usize) -> String {
365    match s.char_indices().nth(max_chars) {
366        Some((i, _)) => format!("{}...", &s[..i]),
367        None => s.to_owned(),
368    }
369}
370
371/// Interned header values: each distinct (field, value) pair gets a small integer id.
372#[derive(Clone, Debug, Default)]
373pub struct ValueDictionary {
374    ids: Vec<HashMap<String, u32>>,
375    entries: Vec<(usize, String, u64)>,
376}
377
378impl ValueDictionary {
379    /// An empty dictionary for `fields` stratified fields.
380    #[must_use]
381    pub fn new(fields: usize) -> Self {
382        Self {
383            ids: vec![HashMap::new(); fields],
384            entries: Vec::new(),
385        }
386    }
387
388    /// The id of `value` in stratified field `field` (0-based among stratified fields),
389    /// counting one more record.
390    ///
391    /// # Errors
392    /// [`DimaError::Overflow`] beyond 2³² distinct values.
393    pub fn intern(&mut self, field: usize, value: String) -> Result<u32> {
394        if let Some(&id) = self.ids[field].get(&value) {
395            self.entries[id as usize].2 += 1;
396            return Ok(id);
397        }
398        let id = u32::try_from(self.entries.len()).map_err(|_| DimaError::Overflow {
399            what: "distinct header values",
400        })?;
401        self.ids[field].insert(value.clone(), id);
402        self.entries.push((field, value, 1));
403        Ok(id)
404    }
405
406    /// `(stratified field index, value, records)` of an id.
407    #[must_use]
408    pub fn entry(&self, id: u32) -> (usize, &str, u64) {
409        let (f, v, n) = &self.entries[id as usize];
410        (*f, v, *n)
411    }
412
413    /// Number of distinct values.
414    #[must_use]
415    pub fn len(&self) -> usize {
416        self.entries.len()
417    }
418
419    /// `true` when no value was interned.
420    #[must_use]
421    pub fn is_empty(&self) -> bool {
422        self.entries.is_empty()
423    }
424
425    /// Distinct values of one stratified field with their record counts, sorted by value bytes.
426    #[must_use]
427    pub fn values_of(&self, field: usize) -> Vec<(&str, u64)> {
428        let mut v: Vec<(&str, u64)> = self
429            .entries
430            .iter()
431            .filter(|e| e.0 == field)
432            .map(|e| (e.1.as_str(), e.2))
433            .collect();
434        v.sort_unstable_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
435        v
436    }
437}
438
439/// Tokens that usually mean "missing" (matched case-insensitively): the INSDC missing-value
440/// vocabulary and common spellings of NA.
441const MISSING_LIKE: &[&str] = &[
442    "missing",
443    "not applicable",
444    "not collected",
445    "not provided",
446    "restricted access",
447    "na",
448    "n/a",
449    "nan",
450    "null",
451    "none",
452];
453
454fn is_missing_like(v: &str, fill: &str) -> bool {
455    if v == fill {
456        return false;
457    }
458    let l = v.to_lowercase();
459    MISSING_LIKE.contains(&l.as_str()) || l.starts_with("missing:")
460}
461
462/// Key under which values count as near-duplicates: lower case, whitespace runs collapsed to a
463/// single space, NFKC.
464fn near_duplicate_key(v: &str) -> String {
465    let lower = v.to_lowercase();
466    let collapsed = lower.split_whitespace().collect::<Vec<_>>().join(" ");
467    collapsed.nfkc().collect()
468}
469
470/// Diagnostics about the values of the stratified fields: missing-like tokens and
471/// near-duplicates, per field.
472#[must_use]
473pub fn value_diagnostics(spec: &HeaderSpec, dict: &ValueDictionary) -> Vec<Diagnostic> {
474    let names = spec.stratified_names();
475    let mut out = Vec::new();
476    for (f, name) in names.iter().enumerate() {
477        let values = dict.values_of(f);
478        let mut missing = Tally::default();
479        for &(v, n) in &values {
480            if is_missing_like(v, spec.fill()) {
481                missing.add(n, || format!("{v:?} ({})", plural(n, "record", "records")));
482            }
483        }
484        if !missing.is_empty() {
485            out.push(
486                Diagnostic::new(
487                    Code::MissingLikeValues,
488                    format!(
489                        "field {name:?}: {} that usually {} \"missing\"; {}",
490                        plural(missing.count, "record has a value", "records have values"),
491                        agree(missing.count, "means", "mean"),
492                        agree(
493                            missing.count,
494                            "it is kept as a literal value",
495                            "they are kept as literal values"
496                        )
497                    ),
498                    missing,
499                )
500                .with_hint(
501                    "if they mean missing, empty them in the headers so they receive the fill \
502                     value, or edit them consistently",
503                ),
504            );
505        }
506        let mut groups: BTreeMap<String, Vec<(&str, u64)>> = BTreeMap::new();
507        for &(v, n) in &values {
508            groups
509                .entry(near_duplicate_key(v))
510                .or_default()
511                .push((v, n));
512        }
513        let mut dups = Tally::default();
514        for g in groups.values().filter(|g| g.len() > 1) {
515            dups.add(1, || {
516                g.iter()
517                    .map(|(v, n)| format!("{v:?} ({n})"))
518                    .collect::<Vec<_>>()
519                    .join(" / ")
520            });
521        }
522        if !dups.is_empty() {
523            out.push(
524                Diagnostic::new(
525                    Code::NearDuplicateValues,
526                    format!(
527                        "field {name:?}: {} of values {} only in case, spacing or character \
528                         forms and {} counted separately",
529                        plural(dups.count, "group", "groups"),
530                        agree(dups.count, "differs", "differ"),
531                        agree(dups.count, "is", "are")
532                    ),
533                    dups,
534                )
535                .with_hint("make the spellings consistent in the headers if they mean the same"),
536            );
537        }
538    }
539    out
540}
541
542/// Diagnostic for stratified fields whose values were all distinct in the probe (likely
543/// identifiers such as accessions), which multiply the metadata held per variant.
544#[must_use]
545pub fn identifier_diagnostic(
546    spec: &HeaderSpec,
547    probe_values: &[Vec<String>],
548    probe_records: u64,
549) -> Option<Diagnostic> {
550    if probe_records < ID_DIAG_MIN_RECORDS as u64 {
551        return None;
552    }
553    let names = spec.stratified_names();
554    let mut tally = Tally::default();
555    let mut flagged = Vec::new();
556    for (f, values) in probe_values.iter().enumerate() {
557        let mut distinct: Vec<&String> = values.iter().collect();
558        distinct.sort_unstable();
559        distinct.dedup();
560        if distinct.len() as u64 == probe_records {
561            tally.add(1, || names[f].clone());
562            flagged.push(names[f].clone());
563        }
564    }
565    if tally.is_empty() {
566        return None;
567    }
568    let keep: Vec<&String> = names.iter().filter(|n| !flagged.contains(n)).collect();
569    let hint = if keep.is_empty() {
570        "every stratified field is identifier-like; omit --header-format if no stratification \
571         is needed"
572            .to_owned()
573    } else {
574        format!(
575            "to stratify without them: --metadata-fields '{}'",
576            keep.iter()
577                .map(|s| s.as_str())
578                .collect::<Vec<_>>()
579                .join("|")
580        )
581    };
582    Some(
583        Diagnostic::new(
584            Code::IdentifierLikeField,
585            format!(
586                "{} {} had a different value in each of the first {probe_records} records; \
587                 identifier-like fields add one metadata entry per sequence to every variant, \
588                 which makes the results large",
589                agree(flagged.len() as u64, "field", "fields"),
590                flagged
591                    .iter()
592                    .map(|f| format!("{f:?}"))
593                    .collect::<Vec<_>>()
594                    .join(", ")
595            ),
596            tally,
597        )
598        .with_hint(hint),
599    )
600}
601
602/// Provenance of the cleaning (D59, D97).
603#[must_use]
604pub fn cleaning_record(stats: &CleanStats) -> HeaderCleaning {
605    let (a, b, c) = unicode_normalization::UNICODE_VERSION;
606    HeaderCleaning {
607        unicode_version: format!("{a}.{b}.{c}"),
608        trimmed: stats.trimmed,
609        invisible_removed: stats
610            .invisible
611            .iter()
612            .map(|(c, n)| CodePointCount {
613                code_point: format!("U+{:04X}", u32::from(*c)),
614                count: *n,
615            })
616            .collect(),
617        nfc_changed: stats.nfc_changed,
618        fill_uses: stats.fill_uses,
619    }
620}
621
622/// Diagnostic summarising value cleaning, if anything was changed.
623#[must_use]
624pub fn cleaning_diagnostic(stats: &CleanStats) -> Option<Diagnostic> {
625    if stats.is_empty() {
626        return None;
627    }
628    let invisible: u64 = stats.invisible.values().sum();
629    let mut tally = Tally::default();
630    for (c, n) in &stats.invisible {
631        tally.add(*n, || {
632            format!(
633                "U+{:04X} ({})",
634                u32::from(*c),
635                crate::human::plural(*n, "time", "times")
636            )
637        });
638    }
639    tally.count = stats.trimmed + stats.nfc_changed + stats.fill_uses + invisible;
640    Some(Diagnostic::new(
641        Code::HeaderValuesCleaned,
642        format!(
643            "header values cleaned: {} trimmed, {} removed, {} normalised to NFC, {} filled",
644            stats.trimmed,
645            plural(invisible, "invisible character", "invisible characters"),
646            stats.nfc_changed,
647            plural(
648                stats.fill_uses,
649                "empty or mismatched value",
650                "empty or mismatched values"
651            )
652        ),
653        tally,
654    ))
655}
656
657#[cfg(test)]
658mod tests {
659    use super::*;
660
661    fn spec(format: &str) -> HeaderSpec {
662        HeaderSpec::new(&split_format(format), None, "Unknown").unwrap()
663    }
664
665    #[test]
666    fn spec_validation() {
667        let s = HeaderSpec::new(
668            &split_format(" Accession | Host|Year"),
669            Some(&["Year".into(), "Accession".into()]),
670            "NA",
671        )
672        .unwrap();
673        assert_eq!(s.fields(), ["Accession", "Host", "Year"]);
674        assert_eq!(
675            s.stratified(),
676            [0, 2],
677            "header order regardless of the order given"
678        );
679        assert_eq!(s.stratified_names(), ["Accession", "Year"]);
680        for (format, fields, fill, needle) in [
681            ("A||B", None, "x", "field 2 has no name"),
682            ("A|B|A", None, "x", "appears more than once"),
683            (
684                "A|B",
685                Some(vec!["C".to_owned()]),
686                "x",
687                "not in the header format",
688            ),
689            ("A|B", Some(vec![]), "x", "no metadata fields"),
690            (
691                "A|B",
692                Some(vec!["A".to_owned(), " A".to_owned()]),
693                "x",
694                "more than once",
695            ),
696            ("A|B", None, " \u{3000}", "fill value"),
697        ] {
698            let e = HeaderSpec::new(&split_format(format), fields.as_deref(), fill).unwrap_err();
699            assert!(e.to_string().contains(needle), "{format}: {e}");
700            assert_eq!(e.class(), crate::error::ExitClass::Usage);
701        }
702        let e = HeaderSpec::new(&split_format("A|B"), Some(&["Hots".into()]), "x").unwrap_err();
703        assert!(e.hint().unwrap().contains("A, B"));
704        // NFC: a decomposed name equals its composed form.
705        let e =
706            HeaderSpec::new(&["Cafe\u{301}".into(), "Caf\u{e9}".into()], None, "x").unwrap_err();
707        assert!(e.to_string().contains("more than once"));
708    }
709
710    #[test]
711    fn value_cleaning() {
712        let mut st = CleanStats::default();
713        assert_eq!(clean_value("  Human ", "U", &mut st), "Human");
714        assert_eq!(clean_value("", "U", &mut st), "U");
715        assert_eq!(clean_value("\u{200B}\u{FEFF}", "U", &mut st), "U");
716        assert_eq!(
717            clean_value("Sa\u{300}o\tPaulo", "U", &mut st),
718            "S\u{e0}o\tPaulo"
719        );
720        assert_eq!(
721            clean_value("\u{202F}Riyadh\u{2003}", "U", &mut st),
722            "Riyadh"
723        );
724        assert_eq!(clean_value("Ri\u{200D}yadh\u{7}", "U", &mut st), "Riyadh");
725        assert_eq!(
726            clean_value("x\u{85}", "U", &mut st),
727            "x",
728            "NEL is a C1 control (and White_Space)"
729        );
730        // Interior text, including visible spaces, is untouched; NFKC is never applied.
731        assert_eq!(
732            clean_value("Saudi  Arabia fi", "U", &mut st),
733            "Saudi  Arabia fi"
734        );
735        assert_eq!(st.trimmed, 2);
736        assert_eq!(st.nfc_changed, 1);
737        assert_eq!(st.fill_uses, 2);
738        assert_eq!(st.invisible.get(&'\u{200B}'), Some(&1));
739        assert_eq!(st.invisible.get(&'\u{7}'), Some(&1));
740        assert_eq!(
741            st.invisible.get(&'\u{202F}'),
742            None,
743            "U+202F is a visible (white) space"
744        );
745        let rec = cleaning_record(&st);
746        assert!(
747            rec.invisible_removed
748                .iter()
749                .any(|c| c.code_point == "U+200B")
750        );
751        assert!(cleaning_diagnostic(&st).is_some());
752        assert!(cleaning_diagnostic(&CleanStats::default()).is_none());
753    }
754
755    #[test]
756    fn header_parsing_and_mismatches() {
757        let s = spec("Accession|Host|Year");
758        let at = HeaderPosition {
759            source: "x.fa",
760            record: 3,
761            line: 7,
762            byte: 100,
763        };
764        let mut st = HeaderStats::default();
765        let v = parse_header(b"KX1| Camel |2015", &s, ValidationMode::Strict, at, &mut st).unwrap();
766        assert_eq!(v, ["KX1", "Camel", "2015"]);
767        let v = parse_header(b"KX2|Human", &s, ValidationMode::Strict, at, &mut st).unwrap();
768        assert_eq!(v, ["Unknown"; 3]);
769        assert_eq!(st.mismatch_count, 1);
770        assert_eq!(st.mismatches[0].found, 2);
771        assert_eq!(st.cleaning.fill_uses, 3);
772        let e = mismatch_error("x.fa", &s, &st);
773        assert!(
774            e.to_string()
775                .contains("record 3 (line 7): 2 fields: KX2|Human"),
776            "{e}"
777        );
778        // Invalid UTF-8.
779        let bad = b"KX3|Cam\xE9l|2015";
780        let e = parse_header(bad, &s, ValidationMode::Strict, at, &mut st).unwrap_err();
781        assert!(e.to_string().contains("record 3 (line 7, byte 108)"), "{e}");
782        assert!(e.hint().unwrap().contains("iconv"));
783        let v = parse_header(bad, &s, ValidationMode::Permissive, at, &mut st).unwrap();
784        assert_eq!(v[1], "Cam\u{FFFD}l");
785        assert_eq!(st.utf8_replacements, 1);
786        // Pipes inside values are not possible: splitting happens first.
787        let v = parse_header(
788            "A|B|\u{FF5C}".as_bytes(),
789            &s,
790            ValidationMode::Strict,
791            at,
792            &mut st,
793        )
794        .unwrap();
795        assert_eq!(
796            v[2], "\u{FF5C}",
797            "a full-width bar stays a value; NFC does not map it to '|'"
798        );
799    }
800
801    #[test]
802    fn mismatch_log_is_capped_in_order() {
803        let s = spec("A|B");
804        let mut total = HeaderStats::default();
805        for batch in 0..3 {
806            let mut st = HeaderStats::default();
807            for r in 0..10 {
808                let at = HeaderPosition {
809                    source: "x",
810                    record: batch * 10 + r + 1,
811                    line: 1,
812                    byte: 0,
813                };
814                parse_header(b"only-one", &s, ValidationMode::Permissive, at, &mut st).unwrap();
815            }
816            total.merge(st);
817        }
818        assert_eq!(total.mismatch_count, 30);
819        assert_eq!(total.mismatches.len(), MISMATCH_LOG_MAX);
820        assert_eq!(total.mismatches[19].record, 20);
821        let e = mismatch_error("x", &s, &total);
822        assert!(e.to_string().ends_with("\n  and 10 more"));
823    }
824
825    #[test]
826    fn dictionary_and_value_diagnostics() {
827        let s = spec("Host|Country");
828        let mut d = ValueDictionary::new(2);
829        for (h, c) in [
830            ("Human", "Saudi Arabia"),
831            ("human", "Saudi  Arabia"),
832            ("Camel", "NA"),
833            ("Human", "missing: control sample"),
834            ("Camel", "Unknown"),
835        ] {
836            d.intern(0, h.into()).unwrap();
837            d.intern(1, c.into()).unwrap();
838        }
839        assert_eq!(d.len(), 8);
840        assert_eq!(d.values_of(0), [("Camel", 2), ("Human", 2), ("human", 1)]);
841        let diags = value_diagnostics(&s, &d);
842        let codes: Vec<_> = diags.iter().map(|d| d.code.as_str()).collect();
843        assert_eq!(
844            codes,
845            [
846                "near_duplicate_values",
847                "missing_like_values",
848                "near_duplicate_values"
849            ]
850        );
851        assert!(
852            diags[1].message.contains("2 records"),
853            "the fill value itself is not flagged: {}",
854            diags[1].message
855        );
856        assert!(diags[0].examples[0].contains("\"Human\" (2) / \"human\" (1)"));
857    }
858
859    #[test]
860    fn identifier_like_fields() {
861        let s = spec("Accession|Host");
862        let ids: Vec<String> = (0..150).map(|i| format!("ACC{i}")).collect();
863        let hosts: Vec<String> = (0..150)
864            .map(|i| ["Camel", "Human"][i % 2].to_owned())
865            .collect();
866        let d = identifier_diagnostic(&s, &[ids.clone(), hosts.clone()], 150).unwrap();
867        assert!(d.message.contains("\"Accession\""));
868        assert_eq!(
869            d.hint.as_deref(),
870            Some("to stratify without them: --metadata-fields 'Host'")
871        );
872        assert!(
873            identifier_diagnostic(&s, &[ids[..50].to_vec(), hosts[..50].to_vec()], 50).is_none(),
874            "too few records"
875        );
876        assert!(identifier_diagnostic(&s, &[hosts.clone(), hosts], 150).is_none());
877    }
878}