texlang/token/
mod.rs

1//! TeX tokens and category codes.
2
3pub mod lexer;
4pub mod trace;
5use crate::types::CatCode;
6use std::{fmt::Display, num};
7use texcraft_stdext::collections::interner;
8
9/// String type used to represent control sequence names in Texlang.
10///
11/// The implementation of this type is opaque so that it can be performance optimized
12/// without worrying about downstream consumers.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
14#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
15pub struct CsName(num::NonZeroU32);
16
17impl CsName {
18    #[inline]
19    pub fn to_usize(&self) -> usize {
20        self.0.get() as usize
21    }
22
23    pub fn try_from_usize(u: usize) -> Option<CsName> {
24        let u = match u32::try_from(u) {
25            Ok(u) => u,
26            Err(_) => return None,
27        };
28        num::NonZeroU32::new(u).map(CsName)
29    }
30}
31
32/// String interner for control sequence names.
33pub type CsNameInterner = interner::Interner<CsName>;
34
35impl interner::Key for CsName {
36    fn try_from_usize(index: usize) -> Option<Self> {
37        num::NonZeroU32::try_from_usize(index).map(CsName)
38    }
39
40    fn into_usize(self) -> usize {
41        self.0.into_usize()
42    }
43}
44
45/// The value of a token.
46#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)]
47#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
48pub enum Value {
49    BeginGroup(char),
50    EndGroup(char),
51    MathShift(char),
52    AlignmentTab(char),
53    Parameter(char),
54    Superscript(char),
55    Subscript(char),
56    Space(char),
57    Letter(char),
58    Other(char),
59    CommandRef(CommandRef),
60}
61
62/// The value of a token that references a command
63#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)]
64#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
65pub enum CommandRef {
66    ControlSequence(CsName),
67    ActiveCharacter(char),
68}
69
70impl CommandRef {
71    pub fn to_string(&self, cs_name_interner: &CsNameInterner) -> String {
72        match self {
73            CommandRef::ControlSequence(cs_name) => {
74                format!("\\{}", cs_name_interner.resolve(*cs_name).unwrap())
75            }
76            CommandRef::ActiveCharacter(c) => format!("{c}"),
77        }
78    }
79}
80
81impl Value {
82    pub fn new(c: char, cat_code: CatCode) -> Value {
83        match cat_code {
84            CatCode::BeginGroup => Value::BeginGroup(c),
85            CatCode::EndGroup => Value::EndGroup(c),
86            CatCode::MathShift => Value::MathShift(c),
87            CatCode::AlignmentTab => Value::AlignmentTab(c),
88            CatCode::Parameter => Value::Parameter(c),
89            CatCode::Superscript => Value::Superscript(c),
90            CatCode::Subscript => Value::Subscript(c),
91            CatCode::Space => Value::Space(c),
92            CatCode::Letter => Value::Letter(c),
93            CatCode::Other => Value::Other(c),
94            CatCode::Active => Value::CommandRef(CommandRef::ActiveCharacter(c)),
95            _ => panic!("raw cat code not allowed"),
96        }
97    }
98
99    pub fn char(&self) -> Option<char> {
100        Some(self.char_and_cat_code()?.0)
101    }
102
103    pub fn cat_code(&self) -> Option<CatCode> {
104        Some(self.char_and_cat_code()?.1)
105    }
106
107    pub fn char_and_cat_code(&self) -> Option<(char, CatCode)> {
108        let (c, code) = match self {
109            Value::BeginGroup(c) => (c, CatCode::BeginGroup),
110            Value::EndGroup(c) => (c, CatCode::EndGroup),
111            Value::MathShift(c) => (c, CatCode::MathShift),
112            Value::AlignmentTab(c) => (c, CatCode::AlignmentTab),
113            Value::Parameter(c) => (c, CatCode::Parameter),
114            Value::Superscript(c) => (c, CatCode::Superscript),
115            Value::Subscript(c) => (c, CatCode::Subscript),
116            Value::Space(c) => (c, CatCode::Space),
117            Value::Letter(c) => (c, CatCode::Letter),
118            Value::Other(c) => (c, CatCode::Other),
119            Value::CommandRef(command_ref) => match command_ref {
120                CommandRef::ControlSequence(_) => return None,
121                CommandRef::ActiveCharacter(c) => (c, CatCode::Active),
122            },
123        };
124        Some((*c, code))
125    }
126}
127
128/// A TeX token.
129#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
130#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
131pub struct Token {
132    value: Value,
133    trace_key: trace::Key,
134}
135
136impl std::fmt::Display for Token {
137    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
138        match &self.value {
139            Value::CommandRef(_) => {
140                write![f, "todo"] // TODO
141            }
142            _ => {
143                write![f, "{}", self.char().unwrap()]
144            }
145        }
146    }
147}
148
149macro_rules! token_constructor {
150    ($name: ident, $value: expr) => {
151        pub fn $name(c: char, trace_key: trace::Key) -> Token {
152            Token {
153                value: $value(c),
154                trace_key,
155            }
156        }
157    };
158}
159
160impl Token {
161    token_constructor!(new_begin_group, Value::BeginGroup);
162    token_constructor!(new_end_group, Value::EndGroup);
163    token_constructor!(new_math_shift, Value::MathShift);
164    token_constructor!(new_alignment_tab, Value::AlignmentTab);
165    token_constructor!(new_parameter, Value::Parameter);
166    token_constructor!(new_superscript, Value::Superscript);
167    token_constructor!(new_subscript, Value::Subscript);
168    token_constructor!(new_space, Value::Space);
169    token_constructor!(new_letter, Value::Letter);
170    token_constructor!(new_other, Value::Other);
171
172    pub fn new_command_ref(command_ref: CommandRef, trace_key: trace::Key) -> Token {
173        Token {
174            value: Value::CommandRef(command_ref),
175            trace_key,
176        }
177    }
178
179    pub fn new_active_character(c: char, trace_key: trace::Key) -> Token {
180        Token {
181            value: Value::CommandRef(CommandRef::ActiveCharacter(c)),
182            trace_key,
183        }
184    }
185
186    pub fn new_control_sequence(name: CsName, trace_key: trace::Key) -> Token {
187        Token {
188            value: Value::CommandRef(CommandRef::ControlSequence(name)),
189            trace_key,
190        }
191    }
192
193    pub fn new_from_value(value: Value, trace_key: trace::Key) -> Token {
194        Token { value, trace_key }
195    }
196
197    #[inline]
198    pub fn value(&self) -> Value {
199        self.value
200    }
201
202    #[inline]
203    pub fn trace_key(&self) -> trace::Key {
204        self.trace_key
205    }
206
207    pub fn char(&self) -> Option<char> {
208        self.value.char()
209    }
210
211    pub fn cat_code(&self) -> Option<CatCode> {
212        self.value.cat_code()
213    }
214    pub fn char_and_cat_code(&self) -> Option<(char, CatCode)> {
215        self.value.char_and_cat_code()
216    }
217}
218
219#[derive(Default)]
220enum PendingWhitespace {
221    #[default]
222    NotStarted,
223    None,
224    Space,
225    Newlines(usize),
226}
227
228impl PendingWhitespace {
229    fn reset(&mut self) {
230        *self = PendingWhitespace::None;
231    }
232
233    fn add_space(&mut self) {
234        *self = match self {
235            PendingWhitespace::NotStarted => PendingWhitespace::NotStarted,
236            PendingWhitespace::None => PendingWhitespace::Space,
237            PendingWhitespace::Space => PendingWhitespace::Space,
238            PendingWhitespace::Newlines(n) => PendingWhitespace::Newlines(*n),
239        }
240    }
241
242    fn add_newline(&mut self) {
243        *self = match self {
244            PendingWhitespace::NotStarted => PendingWhitespace::NotStarted,
245            PendingWhitespace::None => PendingWhitespace::Newlines(1),
246            PendingWhitespace::Space => PendingWhitespace::Newlines(1),
247            PendingWhitespace::Newlines(n) => PendingWhitespace::Newlines(*n + 1),
248        }
249    }
250
251    fn new_paragraph(&mut self) {
252        *self = match self {
253            PendingWhitespace::NotStarted => PendingWhitespace::NotStarted,
254            PendingWhitespace::None | PendingWhitespace::Space | PendingWhitespace::Newlines(1) => {
255                PendingWhitespace::Newlines(2)
256            }
257            PendingWhitespace::Newlines(n) => PendingWhitespace::Newlines(*n),
258        }
259    }
260}
261
262impl Display for PendingWhitespace {
263    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
264        match self {
265            PendingWhitespace::NotStarted | PendingWhitespace::None => Ok(()),
266            PendingWhitespace::Space => {
267                write!(f, " ")
268            }
269            PendingWhitespace::Newlines(n) => {
270                for _ in 0..*n {
271                    writeln!(f)?;
272                }
273                Ok(())
274            }
275        }
276    }
277}
278
279/// Data structure for writing tokens
280#[derive(Default)]
281pub struct Writer {
282    pending_whitespace: PendingWhitespace,
283}
284
285impl Writer {
286    /// Write a token.
287    pub fn write(
288        &mut self,
289        io_writer: &mut dyn std::io::Write,
290        interner: &CsNameInterner,
291        value: Value,
292    ) -> Result<(), std::io::Error> {
293        match &value {
294            Value::CommandRef(CommandRef::ControlSequence(s)) => {
295                write!(
296                    io_writer,
297                    "{}\\{}",
298                    self.pending_whitespace,
299                    interner.resolve(*s).unwrap()
300                )?;
301                self.pending_whitespace.reset();
302            }
303            Value::Space(_) => self.pending_whitespace.add_space(),
304            _ => {
305                write!(
306                    io_writer,
307                    "{}{}",
308                    self.pending_whitespace,
309                    value.char().unwrap()
310                )?;
311                self.pending_whitespace.reset();
312            }
313        }
314        io_writer.flush()
315    }
316
317    pub fn add_newline(&mut self) {
318        self.pending_whitespace.add_newline();
319    }
320
321    pub fn start_paragraph(&mut self) {
322        self.pending_whitespace.new_paragraph();
323    }
324}
325
326/// Write a collection of tokens to a string.
327pub fn write_tokens<'a, T>(tokens: T, interner: &CsNameInterner) -> String
328where
329    T: IntoIterator<Item = &'a Token>,
330{
331    let mut buffer: Vec<u8> = Default::default();
332    let mut writer: Writer = Default::default();
333    for token in tokens.into_iter() {
334        writer.write(&mut buffer, interner, token.value()).unwrap();
335    }
336    std::str::from_utf8(&buffer).unwrap().into()
337}
338
339/// Write a collection of token values to a string.
340pub fn write_token_values<'a, T>(values: T, interner: &CsNameInterner) -> String
341where
342    T: IntoIterator<Item = &'a Value>,
343{
344    let mut buffer: Vec<u8> = Default::default();
345    let mut writer: Writer = Default::default();
346    for value in values.into_iter() {
347        writer.write(&mut buffer, interner, *value).unwrap();
348    }
349    std::str::from_utf8(&buffer).unwrap().into()
350}
351
352#[cfg(test)]
353mod tests {
354    use super::*;
355
356    enum Instruction {
357        ControlSequence(&'static str),
358        Character(char, CatCode),
359        Newline,
360        NewParagraph,
361    }
362
363    fn writer_test(input: Vec<Instruction>, want: &str) {
364        let mut buffer: Vec<u8> = Default::default();
365        let mut writer: Writer = Default::default();
366        let mut interner = CsNameInterner::default();
367        for pre_interned_token in input {
368            match pre_interned_token {
369                Instruction::ControlSequence(name) => {
370                    let cs_name = interner.get_or_intern(name);
371                    let token = Token::new_control_sequence(cs_name, trace::Key::dummy());
372                    writer.write(&mut buffer, &interner, token.value()).unwrap();
373                }
374                Instruction::Character(c, code) => {
375                    let token = Token::new_from_value(Value::new(c, code), trace::Key::dummy());
376                    writer.write(&mut buffer, &interner, token.value()).unwrap();
377                }
378                Instruction::Newline => {
379                    writer.add_newline();
380                }
381                Instruction::NewParagraph => {
382                    writer.start_paragraph();
383                }
384            };
385        }
386        let got: String = std::str::from_utf8(&buffer).unwrap().into();
387        let want = want.to_string();
388
389        if got != want {
390            println!("Output is different:");
391            println!("------[got]-------");
392            println!("{}", got);
393            println!("------[want]------");
394            println!("{}", want);
395            println!("-----------------");
396            panic!("write_tokens test failed");
397        }
398    }
399
400    macro_rules! write_tokens_tests {
401        ($( ($name: ident, $input: expr, $want: expr), )+) => {
402            $(
403            #[test]
404            fn $name() {
405                writer_test($input, $want);
406            }
407            )+
408        };
409    }
410
411    write_tokens_tests!(
412        (blank, vec!(), ""),
413        (
414            trim_whitespace_from_start,
415            vec![
416                Instruction::Character('\n', CatCode::Space),
417                Instruction::Character('\n', CatCode::Space),
418                Instruction::Character('\n', CatCode::Space),
419                Instruction::Character('H', CatCode::Letter),
420            ],
421            "H"
422        ),
423        (
424            trim_whitespace_from_end,
425            vec![
426                Instruction::Character('H', CatCode::Letter),
427                Instruction::Character('\n', CatCode::Space),
428                Instruction::Character('\n', CatCode::Space),
429                Instruction::Character('\n', CatCode::Space),
430            ],
431            "H"
432        ),
433        (
434            trim_whitespace_from_middle_1,
435            vec![
436                Instruction::Character('H', CatCode::Letter),
437                Instruction::Character(' ', CatCode::Space),
438                Instruction::Character(' ', CatCode::Space),
439                Instruction::Character('W', CatCode::Letter),
440            ],
441            "H W"
442        ),
443        (
444            trim_whitespace_from_middle_2,
445            vec![
446                Instruction::Character('H', CatCode::Letter),
447                Instruction::Character('\n', CatCode::Space),
448                Instruction::Character(' ', CatCode::Space),
449                Instruction::Character('\n', CatCode::Space),
450                Instruction::Character('W', CatCode::Letter),
451            ],
452            "H W"
453        ),
454        (
455            trim_whitespace_from_middle_3,
456            vec![
457                Instruction::Character('H', CatCode::Letter),
458                Instruction::Character('\n', CatCode::Space),
459                Instruction::Character('\n', CatCode::Space),
460                Instruction::Character('\n', CatCode::Space),
461                Instruction::Character('W', CatCode::Letter),
462            ],
463            "H W"
464        ),
465        (
466            control_sequence,
467            vec![Instruction::ControlSequence("HelloWorld"),],
468            "\\HelloWorld"
469        ),
470        (
471            newline_1,
472            vec![
473                Instruction::Character('H', CatCode::Letter),
474                Instruction::Newline,
475                Instruction::Character('W', CatCode::Letter),
476            ],
477            "H\nW"
478        ),
479        (
480            newline_2,
481            vec![
482                Instruction::Character('H', CatCode::Letter),
483                Instruction::Newline,
484                Instruction::Character(' ', CatCode::Space),
485                Instruction::Newline,
486                Instruction::Character('W', CatCode::Letter),
487            ],
488            "H\n\nW"
489        ),
490        (
491            newline_3,
492            vec![
493                Instruction::Character('H', CatCode::Letter),
494                Instruction::Newline,
495                Instruction::Character(' ', CatCode::Space),
496                Instruction::Newline,
497                Instruction::Character(' ', CatCode::Space),
498                Instruction::Newline,
499                Instruction::Character('W', CatCode::Letter),
500            ],
501            "H\n\n\nW"
502        ),
503        (
504            par_1,
505            vec![
506                Instruction::Character('H', CatCode::Letter),
507                Instruction::NewParagraph,
508                Instruction::NewParagraph,
509                Instruction::Character('W', CatCode::Letter),
510            ],
511            "H\n\nW"
512        ),
513        (
514            par_2,
515            vec![
516                Instruction::Character('H', CatCode::Letter),
517                Instruction::NewParagraph,
518                Instruction::NewParagraph,
519                Instruction::NewParagraph,
520                Instruction::Character('W', CatCode::Letter),
521            ],
522            "H\n\nW"
523        ),
524    );
525
526    #[test]
527    fn token_size() {
528        assert_eq!(std::mem::size_of::<CommandRef>(), 8);
529        assert_eq!(std::mem::size_of::<Value>(), 8);
530        assert_eq!(std::mem::size_of::<Token>(), 12);
531        assert_eq!(std::mem::size_of::<Result<Token, ()>>(), 12);
532        assert_eq!(std::mem::size_of::<Result<Option<Token>, ()>>(), 12);
533        assert_eq!(std::mem::size_of::<crate::prelude::Result<Token>>(), 12);
534        assert_eq!(
535            std::mem::size_of::<crate::prelude::Result<Option<Token>>>(),
536            12
537        );
538    }
539}
texlang/token/mod.rs

texlang/token/
mod.rs
