texlang_stdlib/
input.rs

1//! Primitives for inputting source code from files and the terminal
2
3use std::cell::RefCell;
4use std::rc::Rc;
5use texlang::parse::{FileLocation, OptionalEquals};
6use texlang::prelude as txl;
7use texlang::token::lexer;
8use texlang::token::trace;
9use texlang::traits::*;
10use texlang::*;
11use texlang_common as common;
12
13use crate::conditional::{self, Condition};
14
15/// Get the `\input` expansion primitive.
16pub fn get_input<S: TexlangState + common::HasFileSystem>() -> command::BuiltIn<S> {
17    command::BuiltIn::new_expansion(input_fn)
18}
19
20fn input_fn<S: TexlangState + common::HasFileSystem>(
21    input_token: token::Token,
22    input: &mut vm::ExpansionInput<S>,
23) -> txl::Result<()> {
24    let file_location = FileLocation::parse(input)?;
25    let (file_path, source_code) =
26        match texlang_common::read_file_to_string(input.vm(), file_location, "tex") {
27            Ok(ok) => ok,
28            Err(err) => {
29                return Err(input.fatal_error(err));
30            }
31        };
32    if input.vm().num_current_sources() > 100 {
33        return Err(input.fatal_error(TooManyInputs {}));
34    }
35    input.push_source(input_token, file_path, source_code)?;
36    Ok(())
37}
38
39#[derive(Debug)]
40struct TooManyInputs {}
41
42impl error::TexError for TooManyInputs {
43    fn kind(&self) -> error::Kind {
44        error::Kind::FailedPrecondition
45    }
46
47    fn title(&self) -> String {
48        "too many input levels (100)".into()
49    }
50}
51
52/// Get the `\endinput` expansion primitive.
53pub fn get_endinput<S: TexlangState>() -> command::BuiltIn<S> {
54    command::BuiltIn::new_expansion(endinput_fn)
55}
56
57fn endinput_fn<S: TexlangState>(
58    _: token::Token,
59    input: &mut vm::ExpansionInput<S>,
60) -> txl::Result<()> {
61    input.end_current_file();
62    Ok(())
63}
64
65#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
66pub struct Component<const N: usize> {
67    #[cfg_attr(feature = "serde", serde(with = "texcraft_stdext::serde_tools::array"))]
68    files: [Option<Box<lexer::Lexer>>; N],
69}
70
71impl<const N: usize> Component<N> {
72    fn take_file(&mut self, index: i32) -> Option<Box<lexer::Lexer>> {
73        let u: usize = match index.try_into() {
74            Ok(u) => u,
75            Err(_) => return None,
76        };
77        match self.files.get_mut(u) {
78            None => None,
79            Some(file_or) => file_or.take(),
80        }
81    }
82
83    fn return_file(&mut self, index: i32, file: Box<lexer::Lexer>) {
84        let u: usize = index.try_into().unwrap();
85        *self.files.get_mut(u).unwrap() = Some(file);
86    }
87}
88
89impl<const N: usize> Default for Component<N> {
90    fn default() -> Self {
91        // We construct the array as a vector first because the element type
92        // cannot be cloned and so we can't use the standard array constructor.
93        // But note that this isn't inefficient, because the array will
94        // simply use the vector's buffer.
95        let v: Vec<Option<Box<lexer::Lexer>>> = (0..N).map(|_| None).collect();
96        Self {
97            files: v.try_into().unwrap(),
98        }
99    }
100}
101
102/// Get the `\openin` execution primitive.
103pub fn get_openin<const N: usize, S: HasComponent<Component<N>> + common::HasFileSystem>(
104) -> command::BuiltIn<S> {
105    command::BuiltIn::new_execution(openin_fn)
106}
107
108fn openin_fn<const N: usize, S: HasComponent<Component<N>> + common::HasFileSystem>(
109    openin_token: token::Token,
110    input: &mut vm::ExecutionInput<S>,
111) -> txl::Result<()> {
112    let (u, _, file_location) = <(parse::Uint<N>, OptionalEquals, FileLocation)>::parse(input)?;
113    let lexer_or = match texlang_common::read_file_to_string(input.vm(), file_location, "tex") {
114        // If the file fails to open TeX does not error out.
115        // Instead, TeX users are expected to test the result with \ifeof.
116        Err(_) => None,
117        Ok((file_path, source_code)) => {
118            let source_code = ensure_ends_in_newline(source_code);
119            let trace_key_range = input.tracer_mut().register_source_code(
120                Some(openin_token),
121                trace::Origin::File(file_path),
122                &source_code,
123            );
124            Some(Box::new(lexer::Lexer::new(source_code, trace_key_range)))
125        }
126    };
127    *input
128        .state_mut()
129        .component_mut()
130        .files
131        .get_mut(u.0)
132        .unwrap() = lexer_or;
133    Ok(())
134}
135
136fn ensure_ends_in_newline(mut s: String) -> String {
137    if !s.ends_with('\n') {
138        s.push('\n')
139    }
140    s
141}
142
143/// Get the `\closein` execution primitive.
144pub fn get_closein<const N: usize, S: HasComponent<Component<N>>>() -> command::BuiltIn<S> {
145    command::BuiltIn::new_execution(closein_fn)
146}
147
148fn closein_fn<const N: usize, S: HasComponent<Component<N>>>(
149    _: token::Token,
150    input: &mut vm::ExecutionInput<S>,
151) -> txl::Result<()> {
152    let u = parse::Uint::<N>::parse(input)?;
153    *input
154        .state_mut()
155        .component_mut()
156        .files
157        .get_mut(u.0)
158        .unwrap() = None;
159    Ok(())
160}
161
162/// Get the `\read` execution primitive.
163pub fn get_read<const N: usize, S: HasComponent<Component<N>> + common::HasTerminalIn>(
164) -> command::BuiltIn<S> {
165    command::BuiltIn::new_execution(read_fn)
166}
167
168fn read_fn<const N: usize, S: HasComponent<Component<N>> + common::HasTerminalIn>(
169    _: token::Token,
170    input: &mut vm::ExecutionInput<S>,
171) -> txl::Result<()> {
172    let scope = TexlangState::variable_assignment_scope_hook(input.state_mut());
173    let (index, _, cmd_ref_or) = <(i32, To, Option<token::CommandRef>)>::parse(input)?;
174
175    #[derive(Copy, Clone, PartialEq, Eq)]
176    enum Mode {
177        File,
178        Terminal,
179    }
180
181    let terminal_in = input.state().terminal_in().clone();
182    let (mut lexer, mode, prompt) = match input.state_mut().component_mut().take_file(index) {
183        Some(file) => (file, Mode::File, None),
184        None => {
185            let prompt = if index < 0 {
186                None
187            } else {
188                Some(format!(
189                    r"{}=",
190                    match cmd_ref_or {
191                        None => "unreachable".to_string(),
192                        Some(cmd_ref) => cmd_ref.to_string(input.vm().cs_name_interner()),
193                    }
194                ))
195            };
196            (
197                read_from_terminal(&terminal_in, input, &prompt)?,
198                Mode::Terminal,
199                prompt,
200            )
201        }
202    };
203
204    let mut tokens = vec![];
205    let mut more_lines_exist = true;
206    let mut braces: Vec<token::Token> = vec![];
207    loop {
208        let vm::Parts {
209            state,
210            cs_name_interner,
211            ..
212        } = input.vm_parts();
213        match (lexer.next(state, cs_name_interner, true), mode) {
214            (lexer::Result::Token(token), _) => {
215                match token.cat_code() {
216                    Some(types::CatCode::BeginGroup) => {
217                        braces.push(token);
218                    }
219                    Some(types::CatCode::EndGroup) => {
220                        if braces.pop().is_none() {
221                            more_lines_exist = drain_line(&mut lexer, state, cs_name_interner);
222                            break;
223                        }
224                    }
225                    _ => (),
226                };
227                tokens.push(token);
228            }
229            (lexer::Result::InvalidCharacter(c, trace_key), _) => {
230                return Err(input.fatal_error(lexer::InvalidCharacterError::new(
231                    input.vm(),
232                    c,
233                    trace_key,
234                )))
235            }
236            (lexer::Result::EndOfLine, Mode::File) => {
237                if braces.is_empty() {
238                    break;
239                }
240            }
241            (lexer::Result::EndOfInput, Mode::File) => {
242                if let Some(unmatched_brace) = braces.pop() {
243                    return Err(input.fatal_error(UnmatchedBracesError { unmatched_brace }));
244                }
245                more_lines_exist = false;
246                break;
247            }
248            (lexer::Result::EndOfLine | lexer::Result::EndOfInput, Mode::Terminal) => {
249                if !braces.is_empty() {
250                    lexer = read_from_terminal(&terminal_in, input, &prompt)?;
251                    continue;
252                }
253                break;
254            }
255        }
256    }
257    if mode == Mode::File && more_lines_exist {
258        input.state_mut().component_mut().return_file(index, lexer);
259    }
260    tokens.reverse();
261    let user_defined_macro =
262        texmacro::Macro::new(vec![], vec![], vec![texmacro::Replacement::Tokens(tokens)]);
263    if let Some(cmd_ref) = cmd_ref_or {
264        input
265            .commands_map_mut()
266            .insert_macro(cmd_ref, user_defined_macro, scope);
267    }
268    Ok(())
269}
270
271fn drain_line<S: TexlangState>(
272    file: &mut lexer::Lexer,
273    state: &S,
274    cs_name_interner: &mut token::CsNameInterner,
275) -> bool {
276    loop {
277        match file.next(state, cs_name_interner, true) {
278            lexer::Result::Token(_) | lexer::Result::InvalidCharacter(_, _) => {}
279            lexer::Result::EndOfLine => {
280                return true;
281            }
282            lexer::Result::EndOfInput => {
283                return false;
284            }
285        }
286    }
287}
288
289fn read_from_terminal<S: TexlangState>(
290    terminal_in: &Rc<RefCell<dyn common::TerminalIn>>,
291    input: &mut vm::ExecutionInput<S>,
292    prompt: &Option<String>,
293) -> txl::Result<Box<lexer::Lexer>> {
294    let mut buffer = String::new();
295    if let Err(err) = terminal_in
296        .borrow_mut()
297        .read_line(prompt.as_deref(), &mut buffer)
298    {
299        return Err(input.fatal_error(IoError {
300            title: "failed to read from the terminal".into(),
301            underlying_error: err,
302        }));
303    }
304    let trace_key_range =
305        input
306            .tracer_mut()
307            .register_source_code(None, trace::Origin::Terminal, &buffer);
308    Ok(Box::new(lexer::Lexer::new(buffer, trace_key_range)))
309}
310
311/// Get the `\ifeof` conditional expansion primitive.
312pub fn get_ifeof<const N: usize, S>() -> command::BuiltIn<S>
313where
314    S: HasComponent<Component<N>> + HasComponent<conditional::Component>,
315{
316    IsEof::build_if_command()
317}
318
319struct IsEof<const N: usize>;
320
321impl<const N: usize, S> conditional::Condition<S> for IsEof<N>
322where
323    S: HasComponent<Component<N>> + HasComponent<conditional::Component>,
324{
325    fn evaluate(input: &mut vm::ExpansionInput<S>) -> txl::Result<bool> {
326        let u = parse::Uint::<N>::parse(input)?;
327        Ok(HasComponent::<Component<N>>::component(input.state())
328            .files
329            .get(u.0)
330            .unwrap()
331            .is_none())
332    }
333}
334
335#[derive(Debug)]
336struct IoError {
337    title: String,
338    underlying_error: std::io::Error,
339}
340
341impl error::TexError for IoError {
342    fn kind(&self) -> error::Kind {
343        error::Kind::FailedPrecondition
344    }
345
346    fn title(&self) -> String {
347        self.title.clone()
348    }
349
350    fn notes(&self) -> Vec<error::display::Note> {
351        vec![format!("underlying filesystem error: {}", self.underlying_error).into()]
352    }
353}
354
355#[derive(Debug)]
356struct UnmatchedBracesError {
357    unmatched_brace: token::Token,
358}
359
360impl error::TexError for UnmatchedBracesError {
361    fn kind(&self) -> error::Kind {
362        error::Kind::Token(self.unmatched_brace)
363    }
364
365    fn title(&self) -> String {
366        "file has an unmatched opening brace".into()
367    }
368
369    fn notes(&self) -> Vec<error::display::Note> {
370        vec![r"files being read with the \read primitive must match all opening braces with closing braces".into()]
371    }
372}
373
374/// When parsed, this type consumes a required `to` keyword from the input stream.
375struct To;
376
377impl Parsable for To {
378    fn parse_impl<S: TexlangState>(input: &mut vm::ExpandedStream<S>) -> txl::Result<Self> {
379        texlang::parse::parse_keyword(input, "to")?;
380        Ok(To {})
381    }
382}
383
384#[cfg(test)]
385mod tests {
386    use super::*;
387    use crate::{def, expansion, prefix};
388    use std::collections::HashMap;
389    use texlang_testing::*;
390
391    #[derive(Default)]
392    #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
393    struct State {
394        conditional: conditional::Component,
395        input: Component<16>,
396        prefix: prefix::Component,
397        testing: TestingComponent,
398        #[cfg_attr(feature = "serde", serde(skip))]
399        file_system: Rc<RefCell<common::InMemoryFileSystem>>,
400        #[cfg_attr(feature = "serde", serde(skip))]
401        terminal_in: Rc<RefCell<common::MockTerminalIn>>,
402    }
403
404    impl TexlangState for State {}
405
406    impl common::HasFileSystem for State {
407        fn file_system(&self) -> Rc<RefCell<dyn common::FileSystem>> {
408            self.file_system.clone()
409        }
410    }
411
412    impl common::HasTerminalIn for State {
413        fn terminal_in(&self) -> Rc<RefCell<dyn common::TerminalIn>> {
414            self.terminal_in.clone()
415        }
416    }
417
418    implement_has_component![State{
419        conditional: conditional::Component,
420        input: Component<16>,
421        prefix: prefix::Component,
422        testing: TestingComponent,
423    }];
424
425    fn built_in_commands() -> HashMap<&'static str, command::BuiltIn<State>> {
426        HashMap::from([
427            ("closein", get_closein()),
428            ("def", def::get_def()),
429            ("else", conditional::get_else()),
430            ("endinput", get_endinput()),
431            ("fi", conditional::get_fi()),
432            ("ifeof", get_ifeof()),
433            ("input", get_input()),
434            ("openin", get_openin()),
435            ("read", get_read()),
436            ("relax", expansion::get_relax()),
437        ])
438    }
439
440    fn custom_vm_initialization(vm: &mut vm::VM<State>) {
441        let mut fs = common::InMemoryFileSystem::new(&vm.working_directory.as_ref().unwrap());
442        fs.add_string_file("file1.tex", "content1\n");
443        fs.add_string_file("file2.tex", "content2%\n");
444        fs.add_string_file("file3.tex", r"\input nested/file4");
445        fs.add_string_file("nested/file4.tex", "content4");
446        fs.add_string_file("file5.tex", "file1.tex");
447        fs.add_string_file("recursive.tex", r"\input recursive.tex content");
448        vm.state.file_system = Rc::new(RefCell::new(fs));
449    }
450
451    test_suite!(
452        @options(
453            TestOption::BuiltInCommands(built_in_commands),
454            TestOption::CustomVMInitialization(custom_vm_initialization),
455        ),
456        expansion_equality_tests(
457            (basic_case, r"\input file1 hello", "content1 hello"),
458            (input_together, r"\input file2 hello", r"content2hello"),
459            (basic_case_with_ext, r"\input file1.tex", r"content1 "),
460            (nested, r"\input file3", r"content4"),
461            (nested_2, r"\input \input file5", r"content1 "),
462        ),
463        fatal_error_tests(
464            (file_does_not_exist, r"\input doesNotExist"),
465            (recursive_input, r"\input recursive s"),
466        ),
467    );
468
469    fn end_input_vm_initialization(vm: &mut vm::VM<State>) {
470        let mut fs = common::InMemoryFileSystem::new(&vm.working_directory.as_ref().unwrap());
471        fs.add_string_file(
472            "file1.tex",
473            "Hello\\def\\Macro{Hola\\endinput Mundo}\\Macro World\n",
474        );
475        vm.state.file_system = Rc::new(RefCell::new(fs));
476    }
477
478    test_suite!(
479        @option(TestOption::BuiltInCommands(built_in_commands)),
480        @option(TestOption::CustomVMInitialization(end_input_vm_initialization)),
481        expansion_equality_tests(
482            (end_input_simple, r"Hello\endinput World", "Hello",),
483            (
484                end_input_in_second_file,
485                r"Before\input file1 After",
486                "BeforeHelloHolaMundoAfter"
487            ),
488        ),
489    );
490
491    fn read_vm_initialization(vm: &mut vm::VM<State>) {
492        let mut fs = common::InMemoryFileSystem::new(&vm.working_directory.as_ref().unwrap());
493        fs.add_string_file("file1.tex", "1\n2%\n3");
494        fs.add_string_file("file2.tex", "1{\n2\n3}");
495        fs.add_string_file("file3.tex", "1}1\n2");
496        fs.add_string_file("file4.tex", "");
497        fs.add_string_file("file5.tex", "hello { world");
498        vm.state.file_system = Rc::new(RefCell::new(fs));
499
500        let mut terminal_in: common::MockTerminalIn = Default::default();
501        terminal_in.add_line("first-line");
502        terminal_in.add_line("second-line {");
503        terminal_in.add_line("third-line }");
504        terminal_in.add_line("fourth}line");
505        vm.state.terminal_in = Rc::new(RefCell::new(terminal_in));
506    }
507
508    test_suite!(
509        @options(
510            TestOption::BuiltInCommands(built_in_commands),
511            TestOption::CustomVMInitialization(read_vm_initialization),
512        ),
513        expansion_equality_tests(
514            (
515                ifeof_nothing_open,
516                r"\ifeof 0 Closed\else Open\fi",
517                "Closed",
518            ),
519            (
520                ifeof_non_existent_file,
521                r"\openin 0 doesNotExist \ifeof 0 Closed\else Open\fi",
522                "Closed",
523            ),
524            (
525                ifeof_file_exists,
526                r"\openin 0 file1 \ifeof 0 Closed\else Open\fi",
527                "Open",
528            ),
529            (
530                ifeof_non_existent_file_2,
531                r"\openin 0 file1 \openin 0 doesNotExist \ifeof 0 Closed\else Open\fi",
532                "Closed",
533            ),
534            (
535                ifeof_file_closed,
536                r"\openin 0 file1 \closein 0 \ifeof 0 Closed\else Open\fi",
537                "Closed",
538            ),
539            (
540                read_1,
541                r"\openin 0 file1\read 0 to \line line1='\line'\read 0 to \line line2='\line'\read 0 to \line line3='\line'\ifeof 0 Closed\else Open\fi",
542                "line1='1 'line2='2'line3='3 'Closed",
543            ),
544            (
545                read_2,
546                r"\openin 0 file2\read 0 to ~line1='~'\ifeof 0 Closed\else Open\fi",
547                "line1='1{ 2 3} 'Closed",
548            ),
549            (
550                read_3,
551                r"\openin 0 file3\read 0 to \line line1='\line'\read 0 to \line line2='\line'",
552                "line1='1'line2='2 '",
553            ),
554            (
555                read_4,
556                r"\def\par{par}\openin 0 file4\read 0 to \line line1='\line'\ifeof 0 Closed\else Open\fi",
557                "line1='par'Closed",
558            ),
559            (
560                read_from_terminal,
561                r"\read 0 to \line line1='\line'\read 0 to \line line2='\line'\read 0 to \line line3='\line'",
562                "line1='first-line 'line2='second-line { third-line } 'line3='fourth'",
563            ),
564        ),
565        serde_tests((
566            ifeof_file_exists_serde,
567            r"\openin 0 file1 ",
568            r"\ifeof 0 Closed\else Open\fi"
569        ),),
570        fatal_error_tests(
571            (
572                file_has_unmatched_braces,
573                r"\openin 0 file5 \read 0 to \X (\X)",
574            ),
575            (
576                failed_to_read_from_terminal,
577                r"\read 0 to \X \read 0 to \X \read 0 to \X \read 0 to \X",
578            ),
579        ),
580    );
581}
texlang_stdlib/input.rs

texlang_stdlib/
input.rs
