boxworks_hyphenate/lib.rs
1use boxworks::ds;
2use common::font;
3use common::font::TextIter;
4
5pub struct State {
6 // TODO: Support changing the language
7 pub hyphenator: hyphenate::Hyphenator,
8 pub left_hyphen_min: i32,
9 pub right_hyphen_min: i32,
10}
11pub struct Hyphenator<'a, Font> {
12 pub state: &'a State,
13 /// The lig/kern program of the font being hyphenated is required because
14 /// hyphenation breaks ligatures apart to insert discretionaries, and the
15 /// program is needed to reconstitute ligatures and kerns in the result.
16 pub font_repo: &'a font::Repo<Font>,
17}
18
19impl State {
20 /// Creates a hyphenator state with plain TeX's English patterns and defaults.
21 pub fn plain_tex_en_us() -> Self {
22 Self {
23 hyphenator: hyphenate::Hyphenator::plain_tex_en_us(),
24 left_hyphen_min: 2,
25 right_hyphen_min: 3,
26 }
27 }
28}
29
30impl<'a, Font: font::TextBuilder> boxworks::Hyphenator for Hyphenator<'a, Font> {
31 fn hyphenate(&self, list: &mut Vec<ds::Horizontal>) {
32 let out = hyphenate_impl(self, list);
33 *list = out;
34 }
35}
36
37fn hyphenate_impl<'a, Font: font::TextBuilder>(
38 hyphenater: &Hyphenator<'a, Font>,
39 list: &[ds::Horizontal],
40) -> Vec<ds::Horizontal> {
41 let lower_caser = hyphenate::AsciiLowerCaser::default();
42 let mut out = vec![];
43 let mut i = 0;
44 while let Some(elem) = list.get(i) {
45 i += 1;
46 out.push(elem.clone());
47 // Hyphenation starts only at glue nodes, as per TeX.2021.866.
48 if !matches!(elem, ds::Horizontal::Glue(_)) {
49 continue;
50 }
51
52 // TODO: hyf_char needs to be read from somewhere using the font numbers
53 // in char and lig nodes.
54 let hyf_char = 3_i32;
55
56 // Find the place to start hyphenating
57 // TeX.2021.896
58 let hyphenation_font: Option<font::Id> = loop {
59 let Some(elem) = list.get(i) else { break None };
60 enum Action {
61 Start { font: font::Id },
62 Continue,
63 // Equivalent to done1 in Knuth's TeX.
64 Abort,
65 }
66 use ds::Horizontal::*;
67 let action = match elem {
68 Char(char) => {
69 // The hyf_char logic runs in the label done2.
70 // TODO: this code assumes \uchyph=true (uppercase letters are hyphenated)
71 // and that \lccode has its PlainTeX values (has lower case character iff
72 // ASCII alphabetic). We should remove these assumptions by plumbing in some
73 // parameters.
74 if char.char.is_ascii_alphabetic() {
75 if hyf_char > 0 && hyf_char <= 255 {
76 Action::Start { font: char.font }
77 } else {
78 Action::Abort
79 }
80 } else {
81 Action::Continue
82 }
83 }
84 Ligature(ligature) => {
85 match ligature.original_chars.chars().next() {
86 None => Action::Continue,
87 Some(char) => {
88 // TODO: all of the TODOs in the Char arm.
89 if char.is_ascii_alphabetic() {
90 if hyf_char > 0 && hyf_char <= 255 {
91 Action::Start {
92 font: ligature.font,
93 }
94 } else {
95 Action::Abort
96 }
97 } else {
98 Action::Continue
99 }
100 }
101 }
102 }
103 Whatsit(whatsit) => {
104 // TeX.2021.1363 but given how we've architected the code, the logic in TeX.2021.1382
105 // (which changes the current language) should run here.
106 whatsit.hyphenation_hook();
107 Action::Continue
108 }
109 Kern(kern) => match kern.kind {
110 ds::KernKind::Normal => Action::Continue,
111 _ => Action::Abort,
112 },
113 _ => Action::Abort,
114 };
115 match action {
116 Action::Start { font } => {
117 break Some(font);
118 }
119 Action::Continue => {
120 i += 1;
121 out.push(elem.clone());
122 }
123 Action::Abort => {
124 i += 1;
125 out.push(elem.clone());
126 break None;
127 }
128 }
129 };
130 let Some(hyphenation_font) = hyphenation_font else {
131 continue;
132 };
133 let font = hyphenater.font_repo.get(hyphenation_font);
134 // It's still possible we won't hyphenate based on the node that ends the
135 // string of characters. So we save this `i` value here; if hyphenation is skipped
136 // we set `i` back to this.
137 let hyphenation_start_i = i;
138
139 let mut s = String::new();
140 // Accumulate the word to be hyphenated.
141 // TeX.2021.897
142 let right_boundary_override: Option<char> =
143 loop {
144 let Some(elem) = list.get(i) else { break None };
145 use ds::Horizontal::*;
146 match elem {
147 Char(char) => {
148 if char.font != hyphenation_font {
149 // TODO: add tests for this case including left/right boundary behaviour.
150 break None;
151 }
152 // TODO: plumb in \lccode and change this check.
153 if !char.char.is_ascii_alphabetic() {
154 break Some(char.char);
155 }
156 if s.len() + char.char.len_utf8() >= 64 {
157 // TeX only hyphenates words up to 64 bytes.
158 // TODO: this check is not quite right: unicode values in the range [128, 255)
159 // should count as 1 only.
160 break Some(char.char);
161 }
162 s.push(char.char);
163 }
164 Ligature(ligature) => {
165 // TeX.2021.898
166 if ligature.font != hyphenation_font {
167 // TODO: add tests for this case including left/right boundary behaviour.
168 break None;
169 }
170 if !ligature
171 .original_chars
172 .chars()
173 .all(|c| c.is_ascii_alphabetic())
174 {
175 break Some(ligature.original_chars.chars().next().expect(
176 "there must be at least one char for this branch to execute",
177 ));
178 }
179 if s.len() + ligature.original_chars.len() >= 64 {
180 // TeX only hyphenates words up to 64 bytes.
181 // TODO: this check is not quite right: unicode values in the range [128, 255)
182 // should count as 1 only.
183 break Some(ligature.original_chars.chars().next().expect(
184 "there must be at least one char for this branch to execute",
185 ));
186 }
187 s.push_str(&ligature.original_chars);
188 }
189 Kern(kern) => match kern.kind {
190 ds::KernKind::Normal => {
191 // Kerns from the font's lig/kern program are part of the word (TeX.2021.897).
192 }
193 _ => break None,
194 },
195 _ => break None,
196 }
197 // Consume the node whose characters have just been placed in s (or the normal kern).
198 i += 1;
199 };
200 // The first char node that triggered the word search will have been put in s.
201 assert!(!s.is_empty());
202
203 // Check if the word can be hyphenated based on the terminating node.
204 // TeX.2021.899
205 // We use a different index to iterate as all of the elements here still need to be
206 // copied to the output list.
207 let mut j = i;
208 let should_hyphenate = loop {
209 let Some(elem) = list.get(j) else { break true };
210 use ds::Horizontal::*;
211 match elem {
212 Char(_) | Ligature(_) => {
213 // This can happen if the font is different, or the 64 byte limit is already
214 // reached.
215 }
216 Kern(kern) => match kern.kind {
217 ds::KernKind::Normal => {
218 // Kerns from the font's lig/kern program don't end the word (TeX.2021.899).
219 }
220 _ => break true,
221 },
222 Whatsit(_) | Glue(_) | Penalty(_) | Insertion(_) | Adjust(_) | Mark(_) => {
223 break true;
224 }
225 HBox(_) | VBox(_) | Rule(_) | Discretionary(_) | Math(_) => {
226 // done1 in Knuth's TeX.f
227 break false;
228 }
229 }
230 j += 1;
231 };
232 if !should_hyphenate {
233 i = hyphenation_start_i;
234 continue;
235 }
236
237 let l = s.chars().count();
238
239 let mut indices = {
240 let indices = hyphenater
241 .state
242 .hyphenator
243 .calculate_indices(&lower_caser, &s);
244 // TeX.2021.1200
245 let left_hyphen_min: usize = match hyphenater.state.left_hyphen_min.try_into() {
246 Ok(0) | Err(_) => 1,
247 Ok(i) => i,
248 };
249 let right_hyphen_min: usize = match hyphenater.state.right_hyphen_min.try_into() {
250 Ok(0) | Err(_) => 1,
251 Ok(i) => i,
252 };
253 let hyph_max = l.saturating_sub(right_hyphen_min);
254 IndexIter::new(indices, left_hyphen_min, hyph_max)
255 };
256 let mut next_or = indices.next();
257
258 let mut main_iter = font.build_text(
259 s.chars(),
260 font::BuildTextOptions {
261 disable_left_boundary: false,
262 right_boundary_override,
263 },
264 );
265
266 use common::font::TextItem;
267
268 let mut chars_pushed = 0;
269 let mut elements_since_separation_point = 0_usize;
270 let mut start_of_separation_point = 0_usize;
271 // This corresponds to the loop in TeX.2021.913 but not 1-1.
272 //
273 // TeX's loop is across all cut prefixes whereas this loop is over each
274 // individual lig/kern element that is emitted.
275 // We iterate over all elements, but do have some logic at the start of the
276 // loop if we are at a separation point.
277 //
278 // Knuth's reconstitute method sets hyphen_passed>0 if either the main lig/kern
279 // program ran over a hyphen while processing the cut prefix,
280 // or if the hyphen lig/kern program is non-trivial and
281 // thus needs to run. His reconstitute method does *not* consider the regular case where the
282 // hyphen does not interact with the lig/kern program; this case is handled in the
283 // body of section 913.
284 loop {
285 if main_iter.is_separation_point() {
286 elements_since_separation_point = 0;
287 start_of_separation_point = chars_pushed;
288 }
289 let Some(elem) = main_iter.next() else { break };
290
291 let (num_chars, last_char, original_elem): (usize, Option<char>, ds::Horizontal) =
292 match elem {
293 TextItem::Char(c) => (
294 1,
295 Some(c),
296 ds::Char {
297 char: c,
298 font: hyphenation_font,
299 }
300 .into(),
301 ),
302 TextItem::Kern(scaled) => (
303 0,
304 None,
305 ds::Kern {
306 width: scaled,
307 kind: ds::KernKind::Normal,
308 }
309 .into(),
310 ),
311 TextItem::Ligature {
312 c,
313 original,
314 includes_left_boundary,
315 includes_right_boundary,
316 } => (
317 original.chars().count(),
318 Some(c),
319 ds::Ligature {
320 includes_left_boundary,
321 includes_right_boundary,
322 char: c,
323 font: hyphenation_font,
324 original_chars: original,
325 }
326 .into(),
327 ),
328 };
329 // It would be nice to debug assert that this is equal to the original element in the list,
330 // but there are buggy cases in TeX in which this is not the case.
331 // The unit tests cover these cases.
332 out.push(original_elem);
333 chars_pushed += num_chars;
334 elements_since_separation_point += 1;
335 let Some(last_char) = last_char else { continue };
336
337 let hyph_next = next_or.unwrap_or(usize::MAX);
338 if hyph_next > chars_pushed {
339 // No hyphen here.
340 continue;
341 }
342
343 let is_hyphen_rule = font.has_replacement(Some(last_char), Some('-'));
344
345 // If the hyphen is exactly at a separation point and if the lig/kern program with
346 // the hyphen is also at a separation point (e.g. is_hyphen_rule=false) then we advance
347 // to this separation point. Otherwise, the discretionary is built from looking at
348 // material since the last separation point.
349 if hyph_next == chars_pushed && main_iter.is_separation_point() && !is_hyphen_rule {
350 elements_since_separation_point = 0;
351 start_of_separation_point = chars_pushed;
352 }
353
354 // This is the loop in TeX.2021.914.
355 loop {
356 let hyph_next = next_or.unwrap_or(usize::MAX);
357 let pre_break_text = s[start_of_separation_point..hyph_next]
358 .chars()
359 .chain("-".chars());
360 let pre_break: Vec<ds::DiscretionaryElem> = font
361 .build_text(
362 pre_break_text,
363 font::BuildTextOptions {
364 // The pre-break text always starts at a separation point so we don't
365 // need to do any left boundary processing. Moreover, if we did the default
366 // left boundary processing we would get the wrong result because the pre-
367 // break text is not preceded by the start of a word.
368 // This is all covered in unit tests.
369 disable_left_boundary: true,
370 right_boundary_override: None,
371 },
372 )
373 .map(|elem| {
374 let d: ds::DiscretionaryElem = match elem {
375 TextItem::Char(c) => ds::Char {
376 char: c,
377 font: hyphenation_font,
378 }
379 .into(),
380 TextItem::Kern(scaled) => ds::Kern {
381 width: scaled,
382 kind: ds::KernKind::Normal,
383 }
384 .into(),
385 TextItem::Ligature {
386 c,
387 original,
388 includes_left_boundary,
389 includes_right_boundary,
390 } => ds::Ligature {
391 char: c,
392 font: hyphenation_font,
393 includes_left_boundary,
394 includes_right_boundary,
395 original_chars: original,
396 }
397 .into(),
398 };
399 d
400 })
401 .collect();
402
403 let post_break_text = &s[hyph_next..];
404 let mut post_break_iter = font.build_text(
405 post_break_text.chars(),
406 font::BuildTextOptions {
407 disable_left_boundary: false,
408 right_boundary_override,
409 },
410 );
411 let mut post_break: Vec<ds::DiscretionaryElem> = vec![];
412 let mut post_chars_pushed = hyph_next;
413
414 let disc_insert_point = out
415 .len()
416 .checked_sub(elements_since_separation_point)
417 .expect("not popping more elements than have been pushed");
418
419 // This is the loop in TeX.2021.916.
420 // We want to achieve synchronization for the two iterators.
421 //
422 // The following boolean handles the first if statement in TeX.2021.916.
423 let mut post_char_left_boundary =
424 font.has_replacement(None, post_break_text.chars().next());
425 loop {
426 if !post_char_left_boundary
427 && post_chars_pushed == chars_pushed
428 && post_break_iter.is_separation_point()
429 && main_iter.is_separation_point()
430 {
431 break;
432 }
433 if post_char_left_boundary || post_chars_pushed < chars_pushed {
434 post_char_left_boundary = false;
435 while let Some(elem) = post_break_iter.next() {
436 post_chars_pushed += match &elem {
437 TextItem::Char(_) => 1,
438 TextItem::Kern(_) => 0,
439 TextItem::Ligature { original, .. } => original.chars().count(),
440 };
441 post_break.push(match elem {
442 TextItem::Char(c) => ds::Char {
443 char: c,
444 font: hyphenation_font,
445 }
446 .into(),
447 TextItem::Kern(scaled) => ds::Kern {
448 width: scaled,
449 kind: ds::KernKind::Normal,
450 }
451 .into(),
452 TextItem::Ligature {
453 c,
454 original,
455 includes_left_boundary,
456 includes_right_boundary,
457 } => ds::Ligature {
458 char: c,
459 font: hyphenation_font,
460 includes_left_boundary,
461 includes_right_boundary,
462 original_chars: original,
463 }
464 .into(),
465 });
466 if post_break_iter.is_separation_point() {
467 break;
468 }
469 }
470 } else {
471 while let Some(elem) = main_iter.next() {
472 chars_pushed += match &elem {
473 TextItem::Char(_) => 1,
474 TextItem::Kern(_) => 0,
475 TextItem::Ligature { original, .. } => original.chars().count(),
476 };
477 out.push(match elem {
478 TextItem::Char(c) => ds::Char {
479 char: c,
480 font: hyphenation_font,
481 }
482 .into(),
483 TextItem::Kern(scaled) => ds::Kern {
484 width: scaled,
485 kind: ds::KernKind::Normal,
486 }
487 .into(),
488 TextItem::Ligature {
489 c,
490 original,
491 includes_left_boundary,
492 includes_right_boundary,
493 } => ds::Ligature {
494 char: c,
495 font: hyphenation_font,
496 includes_left_boundary,
497 includes_right_boundary,
498 original_chars: original,
499 }
500 .into(),
501 });
502 if main_iter.is_separation_point() {
503 break;
504 }
505 }
506 }
507 }
508 let replace_count = out
509 .len()
510 .checked_sub(disc_insert_point)
511 .expect("have only added to out since calculating disc_insert_point");
512 out.insert(
513 disc_insert_point,
514 ds::Discretionary {
515 pre_break,
516 post_break,
517 replace_count: replace_count.try_into().unwrap(),
518 }
519 .into(),
520 );
521
522 // We increment the hyphen index at least once to account for the hyphen we have just inserted.
523 next_or = indices.next();
524 // When performing synchronization we may have passed over some hyphens.
525 while next_or.unwrap_or(usize::MAX) < chars_pushed {
526 next_or = indices.next();
527 }
528 if next_or.unwrap_or(usize::MAX) > chars_pushed {
529 // We need to consume more characters before trying more hyphens
530 break;
531 }
532 elements_since_separation_point = 0;
533 start_of_separation_point = chars_pushed;
534 }
535 }
536 }
537 out
538}
539
540struct IndexIter<I> {
541 inner: I,
542 min: usize,
543 max: usize,
544}
545
546impl<I: Iterator<Item = usize>> IndexIter<I> {
547 fn new(inner: I, min: usize, max: usize) -> Self {
548 Self { inner, min, max }
549 }
550}
551
552impl<I: Iterator<Item = usize>> Iterator for IndexIter<I> {
553 type Item = usize;
554
555 fn next(&mut self) -> Option<Self::Item> {
556 let n = self.inner.next()?;
557 if n >= self.min && n <= self.max {
558 return Some(n);
559 }
560 self.next()
561 }
562}
563
564#[cfg(test)]
565mod tests {
566 use std::collections::HashMap;
567 use std::path::PathBuf;
568
569 use super::*;
570 use boxworks_testing::assert_box_eq;
571 use boxworks_testing::assert_box_lossy_eq;
572 use boxworks_text as bwt;
573
574 const TFM_CMR10: &'static [u8] = include_bytes!("../../tfm/corpus/computer-modern/cmr10.tfm");
575
576 fn run_hyphenation_test(tc: TestCase, lossy: bool) {
577 let unhyphenated: String = tc.input.chars().filter(|c| *c != '-').collect();
578 let hyphenation_patterns = tc.hyphenation_patterns.unwrap_or(&tc.input);
579
580 // TeX does not hyphenate the first word of a paragraph so we need to put
581 // another word before the word of interest
582 let tex_input = format!["x {unhyphenated}"];
583
584 let mut tfm_file = tfm::File::deserialize(TFM_CMR10).0.unwrap();
585 {
586 let (p, e) = tfm::ligkern::lang::Program::parse_compact(tc.lig_kern_program).unwrap();
587 tfm_file.replace_lig_kern_program(p, e);
588 }
589 if std::env::var("TEXCRAFT_VERIFY").unwrap_or("".to_string()) == "tex" {
590 let tfm_bytes = tfm_file.serialize();
591 let mut auxiliary_files: HashMap<PathBuf, Vec<u8>> = Default::default();
592 auxiliary_files.insert("specialFont.tfm".into(), tfm_bytes);
593
594 let preamble = format![
595 r"
596 \font \customFont specialFont
597
598 \hyphenation{{{}}}
599 \lefthyphenmin={}
600 \righthyphenmin=0
601
602 \customFont
603 ",
604 hyphenation_patterns,
605 tc.left_hyphen_min.unwrap_or(1),
606 ];
607 let mut tex_engine = boxworks::tex::new_tex_engine_binary("tex".to_string()).unwrap();
608 let (_, tex_got) = boxworks::tex::build_horizontal_lists(
609 tex_engine.as_mut(),
610 &auxiliary_files,
611 &preamble,
612 &mut vec![tex_input.clone()].iter(),
613 /*hyphenate=*/ true,
614 );
615 let tex_got: Vec<boxworks::ds::Horizontal> =
616 tex_got[0].list[2..].iter().cloned().collect();
617
618 if lossy {
619 assert_box_lossy_eq!(tc.want, tex_got);
620 } else {
621 assert_box_eq!(tc.want, tex_got);
622 }
623 return;
624 }
625
626 let tfm_font = tfm::Font::build_from_file(tfm_file).0;
627 let mut font_repo: font::Repo<tfm::Font> = Default::default();
628 let font_id = font_repo.register(tfm_font);
629
630 let mut tp = bwt::TextPreprocessor::new(bwt::Params::plain_tex_defaults());
631 tp.activate_font(font_id);
632 let mut list = vec![];
633 for word in tex_input.split_ascii_whitespace() {
634 tp.add_word(&font_repo, word.trim_matches(' '), &mut list);
635 tp.add_space(&font_repo, &mut list);
636 }
637
638 list.pop();
639 let mut state = State::plain_tex_en_us();
640 state.hyphenator.insert_exceptions(hyphenation_patterns);
641 state.left_hyphen_min = tc.left_hyphen_min.unwrap_or(1);
642 state.right_hyphen_min = 1;
643 let hyphenator = Hyphenator {
644 state: &state,
645 font_repo: &font_repo,
646 };
647 {
648 use boxworks::Hyphenator;
649 hyphenator.hyphenate(&mut list)
650 }
651
652 let tex_got: Vec<boxworks::ds::Horizontal> = list[2..].iter().cloned().collect();
653 assert_box_eq!(tc.want, tex_got);
654 }
655
656 macro_rules! hyphenation_tests {
657 ( $( {
658 $name: ident,
659 TestCase {
660 $( $field: ident: $value: expr, )*
661 },
662 $(lossy: $lossy: expr,)?
663 }, )* ) => {
664 $(
665 #[test]
666 fn $name() {
667 let test_case = TestCase {
668 $( $field: $value, )*
669 ..Default::default()
670 };
671 let lossy = false $( || $lossy )?;
672 run_hyphenation_test(test_case, lossy);
673 }
674 )*
675 };
676 }
677
678 #[derive(Default)]
679 struct TestCase {
680 input: &'static str,
681 lig_kern_program: &'static str,
682 want: &'static str,
683 hyphenation_patterns: Option<&'static str>,
684 left_hyphen_min: Option<i32>,
685 }
686
687 hyphenation_tests![
688 {
689 no_hyphens,
690 TestCase {
691 input: "mint",
692 lig_kern_program: "",
693 want: r#"
694 chars("mint")
695 "#,
696 },
697 },
698 {
699 most_simple_case,
700 TestCase {
701 input: "a-b",
702 lig_kern_program: "",
703 want: r#"
704 chars("a")
705 disc(
706 pre_break=[
707 chars("-")
708 ],
709 )
710 chars("b")
711 "#,
712 },
713 },
714 {
715 lig_1,
716 TestCase {
717 input: "a-b",
718 lig_kern_program: "
719 ab -> axb^
720 ",
721 want: r#"
722 disc(
723 pre_break=[
724 chars("a-")
725 ],
726 replace_count=2,
727 )
728 chars("a")
729 lig("x", "")
730 chars("b")
731 "#,
732 },
733 },
734 {
735 lig_with_hyphen,
736 TestCase {
737 input: "a-b",
738 lig_kern_program: "
739 a- -> ax-^
740 ",
741 want: r#"
742 disc(
743 pre_break=[
744 chars("a")
745 lig("x", "")
746 chars("-")
747 ],
748 replace_count=1,
749 )
750 chars("a")
751 chars("b")
752 "#,
753 },
754 },
755 {
756 lig_with_hyphen_and_letters,
757 TestCase {
758 input: "a-b",
759 lig_kern_program: "
760 a- -> ax-^
761 ab -> ac^_
762 ",
763 want: r#"
764 disc(
765 pre_break=[
766 chars("a")
767 lig("x", "")
768 chars("-")
769 ],
770 post_break=[
771 chars("b")
772 ],
773 replace_count=2,
774 )
775 chars("a")
776 lig("c", "b")
777 "#,
778 },
779 },
780 {
781 left_boundary_char_1,
782 TestCase {
783 input: "a-b",
784 lig_kern_program: "
785 |b -> |c^_
786 ",
787 want: r#"
788 chars("a")
789 disc(
790 pre_break=[
791 chars("-")
792 ],
793 post_break=[
794 lig("c", "b", includes_left_boundary="true")
795 ],
796 replace_count=1,
797 )
798 chars("b")
799 "#,
800 },
801 lossy: true,
802 },
803 {
804 left_boundary_char_2,
805 TestCase {
806 input: "a-b",
807 lig_kern_program: "
808 |d -> |c^_
809 ",
810 want: r#"
811 chars("a")
812 disc(
813 pre_break=[
814 chars("-")
815 ],
816 post_break=[],
817 replace_count=0,
818 )
819 chars("b")
820 "#,
821 },
822 },
823 {
824 left_boundary_char_and_pre_break_1,
825 TestCase {
826 input: "a-b",
827 lig_kern_program: "
828 |- -> |c^_
829 ",
830 want: r#"
831 chars("a")
832 disc(
833 pre_break=[
834 chars("-")
835 ],
836 )
837 chars("b")
838 "#,
839 },
840 },
841 {
842 left_boundary_char_and_pre_break_2,
843 TestCase {
844 input: "ab-c",
845 lig_kern_program: "
846 bc -> _z^_
847 |b -> |d^_
848 ",
849 want: r#"
850 chars("a")
851 disc(
852 pre_break=[
853 chars("b")
854 chars("-")
855 ],
856 post_break=[
857 chars("c")
858 ],
859 replace_count=1,
860 )
861 lig("z", "bc")
862 "#,
863 },
864 },
865 {
866 pre_break_lig_kern_starts_from_separation_point,
867 TestCase {
868 input: "abc-d",
869 lig_kern_program: "
870 ab -> ax^_
871 xc -> _y^_
872 yd -> _z^_
873 ",
874 want: r#"
875 disc(
876 pre_break=[
877 chars("a")
878 lig("y", "bc")
879 chars("-")
880 ],
881 post_break=[
882 chars("d")
883 ],
884 replace_count=2,
885 )
886 chars("a")
887 lig("z", "bcd")
888 "#,
889 },
890 },
891 {
892 kern_inside_word,
893 TestCase {
894 input: "ab-c",
895 lig_kern_program: "
896 ab -> a[65536]b
897 ",
898 want: r#"
899 chars("a")
900 kern(0.625pt)
901 chars("b")
902 disc(
903 pre_break=[
904 chars("-")
905 ],
906 )
907 chars("c")
908 "#,
909 },
910 },
911 {
912 right_boundary_char_after_hyphen,
913 TestCase {
914 input: "a-b",
915 lig_kern_program: "
916 -| -> -c^|
917 ",
918 want: r#"
919 chars("a")
920 disc(
921 pre_break=[
922 chars("-")
923 lig("c", "", includes_right_boundary="true")
924 ],
925 )
926 chars("b")
927 "#,
928 },
929 lossy: true,
930 },
931 {
932 right_boundary_char_during_synchronization,
933 TestCase {
934 // The lig/kern program of the post-break text runs ahead of the main
935 // program: it turns all of "bcd" into a single ligature while the main
936 // program has only emitted the "ab" ligature.
937 // The main program then has to catch up in the synchronization loop of
938 // TeX.2021.916, and the last element it emits while catching up is a
939 // ligature built from the right boundary.
940 input: "a-bcd",
941 lig_kern_program: "
942 ab -> _x^_
943 bc -> _z^_
944 zd -> _q^_
945 d| -> _w^|
946 ",
947 want: r#"
948 disc(
949 pre_break=[
950 chars("a")
951 chars("-")
952 ],
953 post_break=[
954 lig("q", "bcd")
955 ],
956 replace_count=3,
957 )
958 lig("x", "ab")
959 chars("c")
960 lig("w", "d", includes_right_boundary="true")
961 "#,
962 },
963 lossy: true,
964 },
965 {
966 big_lig_1,
967 TestCase {
968 input: "a-bc",
969 lig_kern_program: "
970 ab -> _x^_
971 xc -> _y^_
972 ",
973 want: r#"
974 disc(
975 pre_break=[
976 chars("a")
977 chars("-")
978 ],
979 post_break=[
980 chars("b")
981 chars("c")
982 ],
983 replace_count=1,
984 )
985 lig("y", "abc")
986 "#,
987 },
988 },
989 {
990 big_lig_2,
991 TestCase {
992 input: "a-bc",
993 lig_kern_program: "
994 ab -> _x^_
995 xc -> _y^_
996 bc -> _z^_
997 ",
998 want: r#"
999 disc(
1000 pre_break=[
1001 chars("a")
1002 chars("-")
1003 ],
1004 post_break=[
1005 lig("z", "bc")
1006 ],
1007 replace_count=1,
1008 )
1009 lig("y", "abc")
1010 "#,
1011 },
1012 },
1013 {
1014 big_lig_3,
1015 TestCase {
1016 input: "ab-c",
1017 lig_kern_program: "
1018 ab -> _x^_
1019 xc -> _y^_
1020 ",
1021 want: r#"
1022 disc(
1023 pre_break=[
1024 lig("x", "ab")
1025 chars("-")
1026 ],
1027 post_break=[
1028 chars("c")
1029 ],
1030 replace_count=1,
1031 )
1032 lig("y", "abc")
1033 "#,
1034 },
1035 },
1036 {
1037 big_lig_4,
1038 TestCase {
1039 input: "ab-c",
1040 lig_kern_program: "
1041 ab -> ax^_
1042 ",
1043 want: r#"
1044 chars("a")
1045 lig("x", "b")
1046 disc(
1047 pre_break=[
1048 chars("-")
1049 ],
1050 replace_count=0,
1051 )
1052 chars("c")
1053 "#,
1054 },
1055 },
1056 {
1057 big_lig_with_hyphen,
1058 TestCase {
1059 input: "ab-c",
1060 lig_kern_program: "
1061 ab -> ax^_
1062 x- -> xy^-
1063 ",
1064 want: r#"
1065 disc(
1066 pre_break=[
1067 chars("a")
1068 lig("x", "b")
1069 lig("y", "")
1070 chars("-")
1071 ],
1072 replace_count=2,
1073 )
1074 chars("a")
1075 lig("x", "b")
1076 chars("c")
1077 "#,
1078 },
1079 },
1080 {
1081 big_lig_with_hyphen_2,
1082 TestCase {
1083 input: "ab-c",
1084 lig_kern_program: "
1085 ab -> ax^b
1086 x- -> xy^-
1087 ",
1088 want: r#"
1089 chars("a")
1090 lig("x", "")
1091 chars("b")
1092 disc(
1093 pre_break=[
1094 chars("-")
1095 ],
1096 replace_count=0,
1097 )
1098 chars("c")
1099 "#,
1100 },
1101 },
1102 {
1103 empty_lig_before,
1104 TestCase {
1105 input: "a-b",
1106 lig_kern_program: "
1107 ab -> ax^b
1108 ",
1109 want: r#"
1110 disc(
1111 pre_break=[
1112 chars("a")
1113 chars("-")
1114 ],
1115 replace_count=2,
1116 )
1117 chars("a")
1118 lig("x", "")
1119 chars("b")
1120 "#,
1121 },
1122 },
1123 {
1124 simple_kern,
1125 TestCase {
1126 input: "a-b",
1127 lig_kern_program: "
1128 ab -> a[100]b
1129 ",
1130 want: r#"
1131 disc(
1132 pre_break=[
1133 chars("a")
1134 chars("-")
1135 ],
1136 replace_count=2,
1137 )
1138 chars("a")
1139 kern(0.00095pt)
1140 chars("b")
1141 "#,
1142 },
1143 },
1144 {
1145 same_kern,
1146 TestCase {
1147 input: "a-b",
1148 lig_kern_program: "
1149 ab -> a[100]b
1150 a- -> a[100]-
1151 ",
1152 want: r#"
1153 disc(
1154 pre_break=[
1155 chars("a")
1156 kern(0.00095pt)
1157 chars("-")
1158 ],
1159 replace_count=2,
1160 )
1161 chars("a")
1162 kern(0.00095pt)
1163 chars("b")
1164 "#,
1165 },
1166 },
1167 {
1168 synchronization_1,
1169 TestCase {
1170 input: "a-bcdefgh",
1171 lig_kern_program: "
1172 ab -> _x^_
1173 bc -> _y^_
1174 cd -> _z^_
1175 de -> _w^_
1176 ef -> _v^_
1177 ",
1178 want: r#"
1179 disc(
1180 pre_break=[
1181 chars("a-")
1182 ],
1183 post_break=[
1184 lig("y", "bc")
1185 lig("w", "de")
1186 chars("f")
1187 ],
1188 replace_count=3,
1189 )
1190 lig("x", "ab")
1191 lig("z", "cd")
1192 lig("v", "ef")
1193 # synchronization point
1194 chars("gh", font=1)
1195 "#,
1196 },
1197 },
1198 {
1199 synchronization_2,
1200 TestCase {
1201 input: "a-bcd-ef-gh",
1202 lig_kern_program: "
1203 ab -> _x^_
1204 bc -> _y^_
1205 cd -> _z^_
1206 de -> _w^_
1207 ef -> _v^_
1208 ",
1209 want: r#"
1210 disc(
1211 pre_break=[
1212 chars("a-")
1213 ],
1214 post_break=[
1215 lig("y", "bc")
1216 lig("w", "de")
1217 chars("f")
1218 ],
1219 replace_count=3,
1220 )
1221 lig("x", "ab")
1222 lig("z", "cd")
1223 # the hyphen here is skipped
1224 lig("v", "ef")
1225 # synchronization point
1226 disc(
1227 pre_break=[
1228 chars("-")
1229 ],
1230 post_break=[
1231 ],
1232 )
1233 chars("gh", font=1)
1234 "#,
1235 },
1236 },
1237 {
1238 synchronization_3,
1239 TestCase {
1240 input: "a-bcde",
1241 lig_kern_program: "
1242 ab -> _x^_
1243 bc -> _y^_
1244 xc -> _y^_
1245 yd -> yzd^
1246 ",
1247 want: r#"
1248 disc(
1249 pre_break=[
1250 chars("a-")
1251 ],
1252 post_break=[
1253 lig("y", "bc")
1254 lig("z", "")
1255 ],
1256 replace_count=2,
1257 )
1258 lig("y", "abc")
1259 lig("z", "")
1260 chars("de", font=1)
1261 "#,
1262 },
1263 },
1264 {
1265 word_ends_in_comma_1,
1266 TestCase {
1267 input: "baby,",
1268 lig_kern_program: "
1269 y, -> y[100],
1270 y| -> y[200]|
1271 ",
1272 want: r#"
1273 chars("baby")
1274 kern(0.00095pt)
1275 chars(",")
1276 "#,
1277 hyphenation_patterns: Some("baby"),
1278 },
1279 },
1280 {
1281 word_ends_in_comma_2,
1282 TestCase {
1283 input: "baby,",
1284 lig_kern_program: "
1285 y, -> y[100],
1286 y| -> y[200]|
1287 ",
1288 want: r#"
1289 chars("ba")
1290 disc(
1291 pre_break=[
1292 chars("-")
1293 ],
1294 )
1295 chars("by")
1296 kern(0.00095pt)
1297 chars(",")
1298 "#,
1299 hyphenation_patterns: Some("ba-by"),
1300 },
1301 },
1302 {
1303 right_boundary_char_override_1,
1304 TestCase {
1305 input: "ba-by",
1306 lig_kern_program: "
1307 y| -> y.^|
1308 ",
1309 want: r#"
1310 chars("ba")
1311 disc(
1312 pre_break=[
1313 chars("-")
1314 ],
1315 )
1316 chars("by")
1317 lig(".", "", includes_right_boundary="true")
1318 "#,
1319 },
1320 lossy: true,
1321 },
1322 {
1323 right_boundary_char_override_2,
1324 TestCase {
1325 input: "ab.",
1326 lig_kern_program: "
1327 |b -> |c^_
1328 c. -> c,^_
1329 ",
1330 want: r#"
1331 chars("a")
1332 disc(
1333 pre_break=[
1334 chars("-")
1335 ],
1336 post_break=[
1337 lig("c", "b", includes_left_boundary="true")
1338 lig(",", "", includes_right_boundary="true")
1339 ],
1340 replace_count=1,
1341 )
1342 chars("b.")
1343 "#,
1344 hyphenation_patterns: Some("a-b"),
1345 },
1346 lossy: true,
1347 },
1348 {
1349 right_boundary_char_override_3,
1350 TestCase {
1351 input: "journey.",
1352 lig_kern_program: "
1353 y. -> y^,_
1354 ,| -> ,?^|
1355 ",
1356 want: r#"
1357 chars("jour")
1358 disc(
1359 pre_break=[
1360 chars("-")
1361 ],
1362 )
1363 chars("ney")
1364 lig(",", "", includes_right_boundary="true")
1365 lig(",", ".")
1366 lig("?", "", includes_right_boundary="true")
1367 "#,
1368 hyphenation_patterns: Some(""),
1369 },
1370 lossy: true,
1371 },
1372 {
1373 right_boundary_char_override_4,
1374 TestCase {
1375 input: "journey.",
1376 lig_kern_program: "
1377 y. -> y^,_
1378 y, -> y^?_
1379 ",
1380 want: r#"
1381 chars("jour")
1382 disc(
1383 pre_break=[
1384 chars("-")
1385 ],
1386 )
1387 chars("ney")
1388 lig("?", "", includes_right_boundary="true")
1389 lig("?", ".")
1390 "#,
1391 hyphenation_patterns: Some(""),
1392 },
1393 lossy: true,
1394 },
1395 {
1396 right_boundary_char_override_5,
1397 TestCase {
1398 input: "journey.",
1399 lig_kern_program: "
1400 y. -> y,^_
1401 ",
1402 want: r#"
1403 chars("jour")
1404 disc(
1405 pre_break=[
1406 chars("-")
1407 ],
1408 )
1409 chars("ney")
1410 lig(",", "", includes_right_boundary="true")
1411 lig(",", ".")
1412 "#,
1413 hyphenation_patterns: Some(""),
1414 },
1415 lossy: true,
1416 },
1417 {
1418 right_boundary_char_override_6,
1419 TestCase {
1420 input: "journey.",
1421 lig_kern_program: "
1422 y. -> y^,_
1423 y, -> y^?,
1424 ",
1425 want: r#"
1426 chars("jour")
1427 disc(
1428 pre_break=[
1429 chars("-")
1430 ],
1431 )
1432 chars("ney")
1433 lig("?", "")
1434 lig(",", "", includes_right_boundary="true")
1435 lig(",", ".")
1436 "#,
1437 hyphenation_patterns: Some(""),
1438 },
1439 lossy: true,
1440 },
1441 {
1442 sneezing,
1443 TestCase {
1444 input: "sneezing",
1445 lig_kern_program: "
1446 y. -> y^,_
1447 y, -> y^?,
1448 ",
1449 want: r#"
1450 chars("sneez")
1451 disc(
1452 pre_break=[
1453 chars("-")
1454 ],
1455 )
1456 chars("ing")
1457 "#,
1458 hyphenation_patterns: Some(""),
1459 left_hyphen_min: Some(3),
1460 },
1461 },
1462 {
1463 difficult,
1464 TestCase {
1465 input: "d-if-fi-cult",
1466 lig_kern_program: "
1467 ff -> _0^_
1468 0i -> _1^_
1469 ",
1470 want: r#"
1471 chars("di")
1472 disc(
1473 pre_break=[
1474 chars("f-")
1475 ],
1476 post_break=[
1477 chars("fi")
1478 ],
1479 replace_count=1,
1480 )
1481 lig("1", "ffi")
1482 disc(
1483 pre_break=[
1484 chars("-")
1485 ],
1486 )
1487 chars("cult")
1488 "#,
1489 hyphenation_patterns: Some(""),
1490 left_hyphen_min: Some(3),
1491 },
1492 },
1493 ];
1494}