1use std::collections::{BTreeMap, HashMap};
22
23use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
24use unicode_properties::{GeneralCategory, UnicodeGeneralCategory};
25
26use crate::config::{ID_DIAG_MIN_RECORDS, MISMATCH_LOG_MAX, ValidationMode};
27use crate::diagnostics::{Code, Diagnostic, Tally, agree, plural};
28use crate::error::{DimaError, Location, Result};
29use crate::model::{CodePointCount, HeaderCleaning, HeaderMismatch};
30
31#[must_use]
33pub fn split_format(text: &str) -> Vec<String> {
34 text.split('|').map(str::to_owned).collect()
35}
36
37pub(crate) fn clean_name(name: &str) -> String {
39 name.trim().nfc().collect()
40}
41
42#[derive(Clone, Debug, PartialEq, Eq)]
44pub struct HeaderSpec {
45 fields: Vec<String>,
46 stratified: Vec<usize>,
47 fill: String,
48}
49
50impl HeaderSpec {
51 pub fn new(format: &[String], metadata_fields: Option<&[String]>, fill: &str) -> Result<Self> {
57 if format.is_empty() {
58 return Err(DimaError::usage("the header format names no fields"));
59 }
60 let fields: Vec<String> = format.iter().map(|f| clean_name(f)).collect();
61 for (i, f) in fields.iter().enumerate() {
62 if f.is_empty() {
63 return Err(
64 DimaError::usage(format!("header format field {} has no name", i + 1))
65 .with_hint(
66 "name every '|'-separated field, e.g. 'Accession|Host|Country|Year'",
67 ),
68 );
69 }
70 if fields[..i].contains(f) {
71 return Err(DimaError::usage(format!(
72 "header format field {:?} appears more than once",
73 f
74 ))
75 .with_hint("give every field a distinct name"));
76 }
77 }
78 let stratified = match metadata_fields {
79 None => (0..fields.len()).collect(),
80 Some(names) => {
81 if names.is_empty() {
82 return Err(DimaError::usage("no metadata fields were given"));
83 }
84 let mut idx = Vec::with_capacity(names.len());
85 for n in names {
86 let n = clean_name(n);
87 let Some(i) = fields.iter().position(|f| *f == n) else {
88 return Err(DimaError::usage(format!(
89 "metadata field {n:?} is not in the header format"
90 ))
91 .with_hint(format!("the header format names: {}", fields.join(", "))));
92 };
93 if idx.contains(&i) {
94 return Err(DimaError::usage(format!(
95 "metadata field {n:?} is listed more than once"
96 )));
97 }
98 idx.push(i);
99 }
100 idx.sort_unstable();
102 idx
103 }
104 };
105 let fill = clean_name(fill);
106 if fill.is_empty() {
107 return Err(DimaError::usage("the header fill value must not be empty")
108 .with_hint("empty values are what the fill value replaces; use e.g. Unknown"));
109 }
110 Ok(Self {
111 fields,
112 stratified,
113 fill,
114 })
115 }
116
117 #[must_use]
119 pub fn fields(&self) -> &[String] {
120 &self.fields
121 }
122
123 #[must_use]
125 pub fn stratified(&self) -> &[usize] {
126 &self.stratified
127 }
128
129 #[must_use]
131 pub fn stratified_names(&self) -> Vec<String> {
132 self.stratified
133 .iter()
134 .map(|&i| self.fields[i].clone())
135 .collect()
136 }
137
138 #[must_use]
140 pub fn fill(&self) -> &str {
141 &self.fill
142 }
143}
144
145#[derive(Clone, Debug, Default, PartialEq, Eq)]
147pub struct CleanStats {
148 pub trimmed: u64,
150 pub nfc_changed: u64,
152 pub fill_uses: u64,
155 pub invisible: BTreeMap<char, u64>,
157}
158
159impl CleanStats {
160 pub fn merge(&mut self, other: Self) {
162 self.trimmed += other.trimmed;
163 self.nfc_changed += other.nfc_changed;
164 self.fill_uses += other.fill_uses;
165 for (c, n) in other.invisible {
166 *self.invisible.entry(c).or_default() += n;
167 }
168 }
169
170 #[must_use]
172 pub fn is_empty(&self) -> bool {
173 self.trimmed == 0
174 && self.nfc_changed == 0
175 && self.fill_uses == 0
176 && self.invisible.is_empty()
177 }
178}
179
180fn is_invisible(c: char) -> bool {
183 (c.is_control() && c != '\t') || c.general_category() == GeneralCategory::Format
184}
185
186pub fn clean_value(raw: &str, fill: &str, stats: &mut CleanStats) -> String {
189 if raw.bytes().all(|b| (0x20..0x7F).contains(&b)) {
191 let t = raw.trim();
192 if t.len() != raw.len() {
193 stats.trimmed += 1;
194 }
195 if t.is_empty() {
196 stats.fill_uses += 1;
197 return fill.to_owned();
198 }
199 return t.to_owned();
200 }
201 let mut visible = String::with_capacity(raw.len());
202 for c in raw.chars() {
203 if is_invisible(c) {
204 *stats.invisible.entry(c).or_default() += 1;
205 } else {
206 visible.push(c);
207 }
208 }
209 let t = visible.trim();
210 if t.len() != visible.len() {
211 stats.trimmed += 1;
212 }
213 let value = if is_nfc_quick(t.chars()) == IsNormalized::Yes {
214 t.to_owned()
215 } else {
216 let n: String = t.nfc().collect();
217 if n != t {
218 stats.nfc_changed += 1;
219 }
220 n
221 };
222 if value.is_empty() {
223 stats.fill_uses += 1;
224 return fill.to_owned();
225 }
226 value
227}
228
229#[derive(Clone, Debug, Default, PartialEq, Eq)]
231pub struct HeaderStats {
232 pub records: u64,
234 pub utf8_replacements: u64,
236 pub cleaning: CleanStats,
238 pub mismatch_count: u64,
240 pub mismatches: Vec<HeaderMismatch>,
242}
243
244impl HeaderStats {
245 pub fn merge(&mut self, other: Self) {
247 self.records += other.records;
248 self.utf8_replacements += other.utf8_replacements;
249 self.cleaning.merge(other.cleaning);
250 self.mismatch_count += other.mismatch_count;
251 let room = MISMATCH_LOG_MAX.saturating_sub(self.mismatches.len());
252 self.mismatches
253 .extend(other.mismatches.into_iter().take(room));
254 }
255}
256
257#[derive(Clone, Copy, Debug)]
259pub struct HeaderPosition<'a> {
260 pub source: &'a str,
262 pub record: u64,
264 pub line: u64,
266 pub byte: u64,
268}
269
270pub fn parse_header(
279 header: &[u8],
280 spec: &HeaderSpec,
281 mode: ValidationMode,
282 at: HeaderPosition<'_>,
283 stats: &mut HeaderStats,
284) -> Result<Vec<String>> {
285 stats.records += 1;
286 let text = match std::str::from_utf8(header) {
287 Ok(t) => std::borrow::Cow::Borrowed(t),
288 Err(e) => {
289 if mode == ValidationMode::Strict {
290 let offset = at.byte + 1 + e.valid_up_to() as u64;
291 return Err(DimaError::input_data(
292 Some(
293 Location::new(at.source)
294 .record(at.record)
295 .line(at.line)
296 .byte(offset),
297 ),
298 "header is not valid UTF-8",
299 )
300 .with_hint(
301 "convert the file to UTF-8, e.g. `iconv -f LATIN1 -t UTF-8 in.fasta > \
302 out.fasta`, or use --validation permissive to replace invalid bytes",
303 ));
304 }
305 stats.utf8_replacements += header
306 .utf8_chunks()
307 .filter(|c| !c.invalid().is_empty())
308 .count() as u64;
309 String::from_utf8_lossy(header)
310 }
311 };
312 let parts: Vec<&str> = text.split('|').collect();
313 if parts.len() != spec.fields.len() {
314 stats.mismatch_count += 1;
315 if stats.mismatches.len() < MISMATCH_LOG_MAX {
316 stats.mismatches.push(HeaderMismatch {
317 record: at.record,
318 line: at.line,
319 header: text.clone().into_owned(),
320 expected: spec.fields.len() as u64,
321 found: parts.len() as u64,
322 });
323 }
324 stats.cleaning.fill_uses += spec.stratified.len() as u64;
325 return Ok(vec![spec.fill.clone(); spec.stratified.len()]);
326 }
327 Ok(spec
328 .stratified
329 .iter()
330 .map(|&i| clean_value(parts[i], &spec.fill, &mut stats.cleaning))
331 .collect())
332}
333
334#[must_use]
336pub fn mismatch_error(source: &str, spec: &HeaderSpec, stats: &HeaderStats) -> DimaError {
337 let mut msg = format!(
338 "{} not have the {} of the header format ({})",
339 plural(stats.mismatch_count, "header does", "headers do"),
340 plural(spec.fields.len() as u64, "field", "fields"),
341 spec.fields.join("|")
342 );
343 for m in &stats.mismatches {
344 msg.push_str(&format!(
345 "\n record {} (line {}): {}: {}",
346 m.record,
347 m.line,
348 plural(m.found, "field", "fields"),
349 truncate_for_display(&m.header, 120)
350 ));
351 }
352 if stats.mismatch_count > stats.mismatches.len() as u64 {
353 msg.push_str(&format!(
354 "\n and {} more",
355 stats.mismatch_count - stats.mismatches.len() as u64
356 ));
357 }
358 DimaError::input_data(Some(Location::new(source)), msg).with_hint(
359 "fix the headers, change --header-format, or use --validation permissive to give \
360 mismatched headers the fill value",
361 )
362}
363
364fn truncate_for_display(s: &str, max_chars: usize) -> String {
365 match s.char_indices().nth(max_chars) {
366 Some((i, _)) => format!("{}...", &s[..i]),
367 None => s.to_owned(),
368 }
369}
370
371#[derive(Clone, Debug, Default)]
373pub struct ValueDictionary {
374 ids: Vec<HashMap<String, u32>>,
375 entries: Vec<(usize, String, u64)>,
376}
377
378impl ValueDictionary {
379 #[must_use]
381 pub fn new(fields: usize) -> Self {
382 Self {
383 ids: vec![HashMap::new(); fields],
384 entries: Vec::new(),
385 }
386 }
387
388 pub fn intern(&mut self, field: usize, value: String) -> Result<u32> {
394 if let Some(&id) = self.ids[field].get(&value) {
395 self.entries[id as usize].2 += 1;
396 return Ok(id);
397 }
398 let id = u32::try_from(self.entries.len()).map_err(|_| DimaError::Overflow {
399 what: "distinct header values",
400 })?;
401 self.ids[field].insert(value.clone(), id);
402 self.entries.push((field, value, 1));
403 Ok(id)
404 }
405
406 #[must_use]
408 pub fn entry(&self, id: u32) -> (usize, &str, u64) {
409 let (f, v, n) = &self.entries[id as usize];
410 (*f, v, *n)
411 }
412
413 #[must_use]
415 pub fn len(&self) -> usize {
416 self.entries.len()
417 }
418
419 #[must_use]
421 pub fn is_empty(&self) -> bool {
422 self.entries.is_empty()
423 }
424
425 #[must_use]
427 pub fn values_of(&self, field: usize) -> Vec<(&str, u64)> {
428 let mut v: Vec<(&str, u64)> = self
429 .entries
430 .iter()
431 .filter(|e| e.0 == field)
432 .map(|e| (e.1.as_str(), e.2))
433 .collect();
434 v.sort_unstable_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes()));
435 v
436 }
437}
438
439const MISSING_LIKE: &[&str] = &[
442 "missing",
443 "not applicable",
444 "not collected",
445 "not provided",
446 "restricted access",
447 "na",
448 "n/a",
449 "nan",
450 "null",
451 "none",
452];
453
454fn is_missing_like(v: &str, fill: &str) -> bool {
455 if v == fill {
456 return false;
457 }
458 let l = v.to_lowercase();
459 MISSING_LIKE.contains(&l.as_str()) || l.starts_with("missing:")
460}
461
462fn near_duplicate_key(v: &str) -> String {
465 let lower = v.to_lowercase();
466 let collapsed = lower.split_whitespace().collect::<Vec<_>>().join(" ");
467 collapsed.nfkc().collect()
468}
469
470#[must_use]
473pub fn value_diagnostics(spec: &HeaderSpec, dict: &ValueDictionary) -> Vec<Diagnostic> {
474 let names = spec.stratified_names();
475 let mut out = Vec::new();
476 for (f, name) in names.iter().enumerate() {
477 let values = dict.values_of(f);
478 let mut missing = Tally::default();
479 for &(v, n) in &values {
480 if is_missing_like(v, spec.fill()) {
481 missing.add(n, || format!("{v:?} ({})", plural(n, "record", "records")));
482 }
483 }
484 if !missing.is_empty() {
485 out.push(
486 Diagnostic::new(
487 Code::MissingLikeValues,
488 format!(
489 "field {name:?}: {} that usually {} \"missing\"; {}",
490 plural(missing.count, "record has a value", "records have values"),
491 agree(missing.count, "means", "mean"),
492 agree(
493 missing.count,
494 "it is kept as a literal value",
495 "they are kept as literal values"
496 )
497 ),
498 missing,
499 )
500 .with_hint(
501 "if they mean missing, empty them in the headers so they receive the fill \
502 value, or edit them consistently",
503 ),
504 );
505 }
506 let mut groups: BTreeMap<String, Vec<(&str, u64)>> = BTreeMap::new();
507 for &(v, n) in &values {
508 groups
509 .entry(near_duplicate_key(v))
510 .or_default()
511 .push((v, n));
512 }
513 let mut dups = Tally::default();
514 for g in groups.values().filter(|g| g.len() > 1) {
515 dups.add(1, || {
516 g.iter()
517 .map(|(v, n)| format!("{v:?} ({n})"))
518 .collect::<Vec<_>>()
519 .join(" / ")
520 });
521 }
522 if !dups.is_empty() {
523 out.push(
524 Diagnostic::new(
525 Code::NearDuplicateValues,
526 format!(
527 "field {name:?}: {} of values {} only in case, spacing or character \
528 forms and {} counted separately",
529 plural(dups.count, "group", "groups"),
530 agree(dups.count, "differs", "differ"),
531 agree(dups.count, "is", "are")
532 ),
533 dups,
534 )
535 .with_hint("make the spellings consistent in the headers if they mean the same"),
536 );
537 }
538 }
539 out
540}
541
542#[must_use]
545pub fn identifier_diagnostic(
546 spec: &HeaderSpec,
547 probe_values: &[Vec<String>],
548 probe_records: u64,
549) -> Option<Diagnostic> {
550 if probe_records < ID_DIAG_MIN_RECORDS as u64 {
551 return None;
552 }
553 let names = spec.stratified_names();
554 let mut tally = Tally::default();
555 let mut flagged = Vec::new();
556 for (f, values) in probe_values.iter().enumerate() {
557 let mut distinct: Vec<&String> = values.iter().collect();
558 distinct.sort_unstable();
559 distinct.dedup();
560 if distinct.len() as u64 == probe_records {
561 tally.add(1, || names[f].clone());
562 flagged.push(names[f].clone());
563 }
564 }
565 if tally.is_empty() {
566 return None;
567 }
568 let keep: Vec<&String> = names.iter().filter(|n| !flagged.contains(n)).collect();
569 let hint = if keep.is_empty() {
570 "every stratified field is identifier-like; omit --header-format if no stratification \
571 is needed"
572 .to_owned()
573 } else {
574 format!(
575 "to stratify without them: --metadata-fields '{}'",
576 keep.iter()
577 .map(|s| s.as_str())
578 .collect::<Vec<_>>()
579 .join("|")
580 )
581 };
582 Some(
583 Diagnostic::new(
584 Code::IdentifierLikeField,
585 format!(
586 "{} {} had a different value in each of the first {probe_records} records; \
587 identifier-like fields add one metadata entry per sequence to every variant, \
588 which makes the results large",
589 agree(flagged.len() as u64, "field", "fields"),
590 flagged
591 .iter()
592 .map(|f| format!("{f:?}"))
593 .collect::<Vec<_>>()
594 .join(", ")
595 ),
596 tally,
597 )
598 .with_hint(hint),
599 )
600}
601
602#[must_use]
604pub fn cleaning_record(stats: &CleanStats) -> HeaderCleaning {
605 let (a, b, c) = unicode_normalization::UNICODE_VERSION;
606 HeaderCleaning {
607 unicode_version: format!("{a}.{b}.{c}"),
608 trimmed: stats.trimmed,
609 invisible_removed: stats
610 .invisible
611 .iter()
612 .map(|(c, n)| CodePointCount {
613 code_point: format!("U+{:04X}", u32::from(*c)),
614 count: *n,
615 })
616 .collect(),
617 nfc_changed: stats.nfc_changed,
618 fill_uses: stats.fill_uses,
619 }
620}
621
622#[must_use]
624pub fn cleaning_diagnostic(stats: &CleanStats) -> Option<Diagnostic> {
625 if stats.is_empty() {
626 return None;
627 }
628 let invisible: u64 = stats.invisible.values().sum();
629 let mut tally = Tally::default();
630 for (c, n) in &stats.invisible {
631 tally.add(*n, || {
632 format!(
633 "U+{:04X} ({})",
634 u32::from(*c),
635 crate::human::plural(*n, "time", "times")
636 )
637 });
638 }
639 tally.count = stats.trimmed + stats.nfc_changed + stats.fill_uses + invisible;
640 Some(Diagnostic::new(
641 Code::HeaderValuesCleaned,
642 format!(
643 "header values cleaned: {} trimmed, {} removed, {} normalised to NFC, {} filled",
644 stats.trimmed,
645 plural(invisible, "invisible character", "invisible characters"),
646 stats.nfc_changed,
647 plural(
648 stats.fill_uses,
649 "empty or mismatched value",
650 "empty or mismatched values"
651 )
652 ),
653 tally,
654 ))
655}
656
657#[cfg(test)]
658mod tests {
659 use super::*;
660
661 fn spec(format: &str) -> HeaderSpec {
662 HeaderSpec::new(&split_format(format), None, "Unknown").unwrap()
663 }
664
665 #[test]
666 fn spec_validation() {
667 let s = HeaderSpec::new(
668 &split_format(" Accession | Host|Year"),
669 Some(&["Year".into(), "Accession".into()]),
670 "NA",
671 )
672 .unwrap();
673 assert_eq!(s.fields(), ["Accession", "Host", "Year"]);
674 assert_eq!(
675 s.stratified(),
676 [0, 2],
677 "header order regardless of the order given"
678 );
679 assert_eq!(s.stratified_names(), ["Accession", "Year"]);
680 for (format, fields, fill, needle) in [
681 ("A||B", None, "x", "field 2 has no name"),
682 ("A|B|A", None, "x", "appears more than once"),
683 (
684 "A|B",
685 Some(vec!["C".to_owned()]),
686 "x",
687 "not in the header format",
688 ),
689 ("A|B", Some(vec![]), "x", "no metadata fields"),
690 (
691 "A|B",
692 Some(vec!["A".to_owned(), " A".to_owned()]),
693 "x",
694 "more than once",
695 ),
696 ("A|B", None, " \u{3000}", "fill value"),
697 ] {
698 let e = HeaderSpec::new(&split_format(format), fields.as_deref(), fill).unwrap_err();
699 assert!(e.to_string().contains(needle), "{format}: {e}");
700 assert_eq!(e.class(), crate::error::ExitClass::Usage);
701 }
702 let e = HeaderSpec::new(&split_format("A|B"), Some(&["Hots".into()]), "x").unwrap_err();
703 assert!(e.hint().unwrap().contains("A, B"));
704 let e =
706 HeaderSpec::new(&["Cafe\u{301}".into(), "Caf\u{e9}".into()], None, "x").unwrap_err();
707 assert!(e.to_string().contains("more than once"));
708 }
709
710 #[test]
711 fn value_cleaning() {
712 let mut st = CleanStats::default();
713 assert_eq!(clean_value(" Human ", "U", &mut st), "Human");
714 assert_eq!(clean_value("", "U", &mut st), "U");
715 assert_eq!(clean_value("\u{200B}\u{FEFF}", "U", &mut st), "U");
716 assert_eq!(
717 clean_value("Sa\u{300}o\tPaulo", "U", &mut st),
718 "S\u{e0}o\tPaulo"
719 );
720 assert_eq!(
721 clean_value("\u{202F}Riyadh\u{2003}", "U", &mut st),
722 "Riyadh"
723 );
724 assert_eq!(clean_value("Ri\u{200D}yadh\u{7}", "U", &mut st), "Riyadh");
725 assert_eq!(
726 clean_value("x\u{85}", "U", &mut st),
727 "x",
728 "NEL is a C1 control (and White_Space)"
729 );
730 assert_eq!(
732 clean_value("Saudi Arabia fi", "U", &mut st),
733 "Saudi Arabia fi"
734 );
735 assert_eq!(st.trimmed, 2);
736 assert_eq!(st.nfc_changed, 1);
737 assert_eq!(st.fill_uses, 2);
738 assert_eq!(st.invisible.get(&'\u{200B}'), Some(&1));
739 assert_eq!(st.invisible.get(&'\u{7}'), Some(&1));
740 assert_eq!(
741 st.invisible.get(&'\u{202F}'),
742 None,
743 "U+202F is a visible (white) space"
744 );
745 let rec = cleaning_record(&st);
746 assert!(
747 rec.invisible_removed
748 .iter()
749 .any(|c| c.code_point == "U+200B")
750 );
751 assert!(cleaning_diagnostic(&st).is_some());
752 assert!(cleaning_diagnostic(&CleanStats::default()).is_none());
753 }
754
755 #[test]
756 fn header_parsing_and_mismatches() {
757 let s = spec("Accession|Host|Year");
758 let at = HeaderPosition {
759 source: "x.fa",
760 record: 3,
761 line: 7,
762 byte: 100,
763 };
764 let mut st = HeaderStats::default();
765 let v = parse_header(b"KX1| Camel |2015", &s, ValidationMode::Strict, at, &mut st).unwrap();
766 assert_eq!(v, ["KX1", "Camel", "2015"]);
767 let v = parse_header(b"KX2|Human", &s, ValidationMode::Strict, at, &mut st).unwrap();
768 assert_eq!(v, ["Unknown"; 3]);
769 assert_eq!(st.mismatch_count, 1);
770 assert_eq!(st.mismatches[0].found, 2);
771 assert_eq!(st.cleaning.fill_uses, 3);
772 let e = mismatch_error("x.fa", &s, &st);
773 assert!(
774 e.to_string()
775 .contains("record 3 (line 7): 2 fields: KX2|Human"),
776 "{e}"
777 );
778 let bad = b"KX3|Cam\xE9l|2015";
780 let e = parse_header(bad, &s, ValidationMode::Strict, at, &mut st).unwrap_err();
781 assert!(e.to_string().contains("record 3 (line 7, byte 108)"), "{e}");
782 assert!(e.hint().unwrap().contains("iconv"));
783 let v = parse_header(bad, &s, ValidationMode::Permissive, at, &mut st).unwrap();
784 assert_eq!(v[1], "Cam\u{FFFD}l");
785 assert_eq!(st.utf8_replacements, 1);
786 let v = parse_header(
788 "A|B|\u{FF5C}".as_bytes(),
789 &s,
790 ValidationMode::Strict,
791 at,
792 &mut st,
793 )
794 .unwrap();
795 assert_eq!(
796 v[2], "\u{FF5C}",
797 "a full-width bar stays a value; NFC does not map it to '|'"
798 );
799 }
800
801 #[test]
802 fn mismatch_log_is_capped_in_order() {
803 let s = spec("A|B");
804 let mut total = HeaderStats::default();
805 for batch in 0..3 {
806 let mut st = HeaderStats::default();
807 for r in 0..10 {
808 let at = HeaderPosition {
809 source: "x",
810 record: batch * 10 + r + 1,
811 line: 1,
812 byte: 0,
813 };
814 parse_header(b"only-one", &s, ValidationMode::Permissive, at, &mut st).unwrap();
815 }
816 total.merge(st);
817 }
818 assert_eq!(total.mismatch_count, 30);
819 assert_eq!(total.mismatches.len(), MISMATCH_LOG_MAX);
820 assert_eq!(total.mismatches[19].record, 20);
821 let e = mismatch_error("x", &s, &total);
822 assert!(e.to_string().ends_with("\n and 10 more"));
823 }
824
825 #[test]
826 fn dictionary_and_value_diagnostics() {
827 let s = spec("Host|Country");
828 let mut d = ValueDictionary::new(2);
829 for (h, c) in [
830 ("Human", "Saudi Arabia"),
831 ("human", "Saudi Arabia"),
832 ("Camel", "NA"),
833 ("Human", "missing: control sample"),
834 ("Camel", "Unknown"),
835 ] {
836 d.intern(0, h.into()).unwrap();
837 d.intern(1, c.into()).unwrap();
838 }
839 assert_eq!(d.len(), 8);
840 assert_eq!(d.values_of(0), [("Camel", 2), ("Human", 2), ("human", 1)]);
841 let diags = value_diagnostics(&s, &d);
842 let codes: Vec<_> = diags.iter().map(|d| d.code.as_str()).collect();
843 assert_eq!(
844 codes,
845 [
846 "near_duplicate_values",
847 "missing_like_values",
848 "near_duplicate_values"
849 ]
850 );
851 assert!(
852 diags[1].message.contains("2 records"),
853 "the fill value itself is not flagged: {}",
854 diags[1].message
855 );
856 assert!(diags[0].examples[0].contains("\"Human\" (2) / \"human\" (1)"));
857 }
858
859 #[test]
860 fn identifier_like_fields() {
861 let s = spec("Accession|Host");
862 let ids: Vec<String> = (0..150).map(|i| format!("ACC{i}")).collect();
863 let hosts: Vec<String> = (0..150)
864 .map(|i| ["Camel", "Human"][i % 2].to_owned())
865 .collect();
866 let d = identifier_diagnostic(&s, &[ids.clone(), hosts.clone()], 150).unwrap();
867 assert!(d.message.contains("\"Accession\""));
868 assert_eq!(
869 d.hint.as_deref(),
870 Some("to stratify without them: --metadata-fields 'Host'")
871 );
872 assert!(
873 identifier_diagnostic(&s, &[ids[..50].to_vec(), hosts[..50].to_vec()], 50).is_none(),
874 "too few records"
875 );
876 assert!(identifier_diagnostic(&s, &[hosts.clone(), hosts], 150).is_none());
877 }
878}