diff --git a/lrlex/src/lib/codegen.rs b/lrlex/src/lib/codegen.rs new file mode 100644 index 000000000..6ceaec517 --- /dev/null +++ b/lrlex/src/lib/codegen.rs @@ -0,0 +1,607 @@ +use crate::{LRNonStreamingLexerDef, LexFlags, LexerDef, Visibility, ctbuilder::LexerKind}; +use cfgrammar::{ + header::{GrmtoolsSectionParser, HeaderValue}, + markmap::MarkMap, + span::{Location, Span}, +}; +use lrpar::{ + LexerTypes, + diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, +}; +use proc_macro2::{Ident, TokenStream}; +use quote::{ToTokens, TokenStreamExt, format_ident, quote}; +use regex::Regex; +use std::{ + any::type_name, + borrow::Borrow, + collections::HashMap, + error::Error, + fmt::{self, Display, Write as _}, + marker::PhantomData, + path::Path, + sync::LazyLock, +}; + +static RE_TOKEN_ID: LazyLock = + LazyLock::new(|| Regex::new(r"^[a-zA-Z_][a-zA-Z_0-9]*$").unwrap()); + +use crate::ctbuilder::ERROR; +/// The quote impl of `ToTokens` for `Option` prints an empty string for `None` +/// and the inner value for `Some(inner_value)`. +/// +/// This wrapper instead emits both `Some` and `None` variants. +/// See: [quote #20](https://github.com/dtolnay/quote/issues/20) +struct QuoteOption(Option); + +impl ToTokens for QuoteOption { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.append_all(match self.0 { + Some(ref t) => quote! { ::std::option::Option::Some(#t) }, + None => quote! { ::std::option::Option::None }, + }); + } +} + +/// The wrapped `&str` value will be emitted with a call to `to_string()` +struct QuoteToString<'a>(&'a str); + +impl ToTokens for QuoteToString<'_> { + fn to_tokens(&self, tokens: &mut TokenStream) { + let x = &self.0; + tokens.append_all(quote! { #x.to_string() }); + } +} + +/// This wrapper adds a missing impl of `ToTokens` for tuples. +/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())` +struct QuoteTuple(T); + +impl ToTokens for QuoteTuple<(A, B)> { + fn to_tokens(&self, tokens: &mut TokenStream) { + let (a, b) = &self.0; + tokens.append_all(quote!((#a, #b))); + } +} + +/// Currently this is kind of a hodge podge of everything between parsing/validation +/// and code generation, from lex source input strings, to rust source output strings. +/// +/// This probably needs a better name, as note that the self variable only gets used +/// for the former parsering/validation stages, and the latter codegen phases are all +/// implemented through associated functions. +pub(crate) struct LexCodegenBuilder<'a> { + src: &'a str, + // We store the path here so we can generate a module name from it if needed. + // But should never use it for filesystem interaction within this module. + path: &'a Path, + diagnostics: SpannedDiagnosticFormatter<'a>, + header: MarkMap>, +} + +pub(crate) struct LexCodegen +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + kind: LexerKind, + lexerdef: LRNonStreamingLexerDef, + lex_flags: LexFlags, + mod_name: Ident, + rule_ids_map: Option>, + visibility: Visibility, +} + +impl<'a> LexCodegenBuilder<'a> { + pub(crate) fn new( + src: &'a str, + path: &'a Path, + header: MarkMap>, + ) -> LexCodegenBuilder<'a> { + let diagnostics = SpannedDiagnosticFormatter::new(src, path); + LexCodegenBuilder { + src, + path, + header, + diagnostics, + } + } + + pub(crate) fn lex_diag(&self) -> &SpannedDiagnosticFormatter<'a> { + &self.diagnostics + } + + fn merge_headers(&mut self) -> Result<(), Box> { + let (parsed_header, _) = self.parse_header()?; + Ok(self.header.merge_from(parsed_header)?) + } + + fn parse_header(&self) -> Result<(MarkMap>, usize), Box> { + GrmtoolsSectionParser::new(self.src, false) + .parse() + .map_err(|es| { + let mut out = String::new(); + out.push_str(&format!( + "\n{ERROR}{}\n", + self.lex_diag() + .file_location_msg(" parsing the `%grmtools` section", None) + )); + for e in es { + out.push_str(&indent( + " ", + &self.lex_diag().format_error(e).to_string(), + )); + out.push('\n'); + } + ErrorString(out).into() + }) + } + + fn extract_lexerkind( + &mut self, + lexerkind: Option, + ) -> Result> { + self.header.mark_used(&"lexerkind".to_string()); + match lexerkind { + Some(lexerkind) => Ok(lexerkind), + None => { + if let Some(HeaderValue(_, lk_val)) = self.header.get("lexerkind") { + Ok(LexerKind::try_from(lk_val)?) + } else { + Ok(LexerKind::LRNonStreamingLexer) + } + } + } + } + + fn extract_lexerdef( + &mut self, + lexerkind: &LexerKind, + ) -> Result<(LRNonStreamingLexerDef, LexFlags), Box> + where + LexerTypesT: LexerTypes, + LexerTypesT::StorageT: TryFrom, + usize: num_traits::AsPrimitive, + { + let (lexerdef, lex_flags): (LRNonStreamingLexerDef, LexFlags) = match lexerkind + { + LexerKind::LRNonStreamingLexer => { + let lex_flags = LexFlags::try_from(&mut self.header)?; + let lexerdef = + LRNonStreamingLexerDef::::new_with_options(self.src, lex_flags) + .map_err(|errs| { + let mut out = String::new(); + out.push_str(&format!( + "\n{ERROR}{}\n", + self.lex_diag().file_location_msg("", None) + )); + for e in errs { + out.push_str(&indent( + " ", + &self.lex_diag().format_error(e).to_string(), + )); + out.push('\n'); + } + ErrorString(out) + })?; + let lex_flags = lexerdef.lex_flags().cloned(); + (lexerdef, lex_flags.unwrap()) + } + }; + Ok((lexerdef, lex_flags)) + } + + fn check_unused_header_values(&self) -> Result<(), Box> { + let unused_header_values = self.header.unused(); + if !unused_header_values.is_empty() { + Err(format!("Unused header values: {}", unused_header_values.join(", ")).into()) + } else { + Ok(()) + } + } + + fn mod_name_tokens(&self, mod_name: Option<&str>) -> Result> { + let mod_name = match mod_name { + Some(s) => s.to_owned(), + None => { + // The user hasn't specified a module name, so we create one automatically: what we + // do is strip off all the filename extensions (note that it's likely that inp ends + // with `l.rs`, so we potentially have to strip off more than one extension) and + // then add `_l` to the end. + let mut stem = self.path.to_str().unwrap(); + loop { + let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap(); + if stem == new_stem { + break; + } + stem = new_stem; + } + format!("{}_l", stem) + } + }; + let mod_name = + match syn::parse_str::(&mod_name) { + Ok(s) => s, + Err(e) => return Err(format!( + "CTLexerBuilder::mod_name(\"{}\") is not a valid rust identifier due to '{}'", + mod_name, e + ) + .into()), + }; + Ok(mod_name) + } + + pub(crate) fn build( + &mut self, + lexerkind_specified: Option, + mod_name_specified: Option<&str>, + visibility: Visibility, + ) -> Result, Box> + where + LexerTypesT: LexerTypes, + LexerTypesT::StorageT: TryFrom, + usize: num_traits::AsPrimitive, + { + self.merge_headers()?; + let kind = self.extract_lexerkind(lexerkind_specified)?; + let (lexerdef, lex_flags) = self.extract_lexerdef::(&kind)?; + let mod_name = self.mod_name_tokens(mod_name_specified)?; + self.check_unused_header_values()?; + Ok(LexCodegen { + kind, + lexerdef, + lex_flags, + mod_name, + rule_ids_map: None, + visibility, + }) + } +} + +impl LexCodegen +where + LexerTypesT: LexerTypes, + usize: num_traits::AsPrimitive, +{ + #[cfg(test)] + pub(crate) fn lexerkind(&self) -> &LexerKind { + &self.kind + } + + pub(crate) fn lexerdef(&self) -> &LRNonStreamingLexerDef { + &self.lexerdef + } + + pub(crate) fn lexerdef_mut(&mut self) -> &mut LRNonStreamingLexerDef { + &mut self.lexerdef + } + + pub(crate) fn set_rule_ids_map( + &mut self, + rule_ids_map: Option>, + ) { + self.rule_ids_map = rule_ids_map; + } + + fn gen_lex_flags(&self) -> TokenStream { + let LexFlags { + allow_wholeline_comments, + dot_matches_new_line, + multi_line, + octal, + posix_escapes, + case_insensitive, + unicode, + swap_greed, + ignore_whitespace, + size_limit, + dfa_size_limit, + nest_limit, + } = self.lex_flags; + let allow_wholeline_comments = QuoteOption(allow_wholeline_comments); + let dot_matches_new_line = QuoteOption(dot_matches_new_line); + let multi_line = QuoteOption(multi_line); + let octal = QuoteOption(octal); + let posix_escapes = QuoteOption(posix_escapes); + let case_insensitive = QuoteOption(case_insensitive); + let unicode = QuoteOption(unicode); + let swap_greed = QuoteOption(swap_greed); + let ignore_whitespace = QuoteOption(ignore_whitespace); + let size_limit = QuoteOption(size_limit); + let dfa_size_limit = QuoteOption(dfa_size_limit); + let nest_limit = QuoteOption(nest_limit); + + // Code gen for the lexerdef() `lex_flags` variable. + quote! { + let mut lex_flags = ::lrlex::DEFAULT_LEX_FLAGS; + lex_flags.allow_wholeline_comments = #allow_wholeline_comments.or(::lrlex::DEFAULT_LEX_FLAGS.allow_wholeline_comments); + lex_flags.dot_matches_new_line = #dot_matches_new_line.or(::lrlex::DEFAULT_LEX_FLAGS.dot_matches_new_line); + lex_flags.multi_line = #multi_line.or(::lrlex::DEFAULT_LEX_FLAGS.multi_line); + lex_flags.octal = #octal.or(::lrlex::DEFAULT_LEX_FLAGS.octal); + lex_flags.posix_escapes = #posix_escapes.or(::lrlex::DEFAULT_LEX_FLAGS.posix_escapes); + lex_flags.case_insensitive = #case_insensitive.or(::lrlex::DEFAULT_LEX_FLAGS.case_insensitive); + lex_flags.unicode = #unicode.or(::lrlex::DEFAULT_LEX_FLAGS.unicode); + lex_flags.swap_greed = #swap_greed.or(::lrlex::DEFAULT_LEX_FLAGS.swap_greed); + lex_flags.ignore_whitespace = #ignore_whitespace.or(::lrlex::DEFAULT_LEX_FLAGS.ignore_whitespace); + lex_flags.size_limit = #size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.size_limit); + lex_flags.dfa_size_limit = #dfa_size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.dfa_size_limit); + lex_flags.nest_limit = #nest_limit.or(::lrlex::DEFAULT_LEX_FLAGS.nest_limit); + let lex_flags = lex_flags; + } + } + + fn gen_lexerdef(&self, token_stream: &mut TokenStream) + where + LexerTypesT: LexerTypes, + LexerTypesT::StorageT: TryFrom + ToTokens, + usize: num_traits::AsPrimitive, + { + let start_states = self.lexerdef.iter_start_states(); + let rules = self.lexerdef.iter_rules().map(|r| { + let tok_id = QuoteOption(r.tok_id); + let n = QuoteOption(r.name().map(QuoteToString)); + let target_state = QuoteOption(r.target_state().map(|(x, y)| QuoteTuple((x, y)))); + let n_span = r.name_span(); + let regex = QuoteToString(&r.re_str); + let start_states = r.start_states(); + // Code gen to construct a rule. + // + // We cannot `impl ToToken for Rule` because `Rule` never stores `lex_flags`, + // Thus we reference the local lex_flags variable bound earlier. + quote! { + Rule::new(::lrlex::unstable_api::InternalPublicApi, #tok_id, #n, #n_span, #regex, + vec![#(#start_states),*], #target_state, &lex_flags).unwrap() + } + }); + // Code gen for `lexerdef()`s rules and the stack of `start_states`. + token_stream.append_all(quote! { + let start_states: Vec = vec![#(#start_states),*]; + let rules = vec![#(#rules),*]; + }); + } + + fn gen_lexerkind(&self, token_stream: &mut TokenStream) -> TokenStream { + let lexerdef_ty = match self.kind { + LexerKind::LRNonStreamingLexer => { + quote!(::lrlex::LRNonStreamingLexerDef) + } + }; + token_stream.append_all(quote! { + #lexerdef_ty::from_rules(start_states, rules) + }); + lexerdef_ty + } + + fn gen_module(&self, lexerdef_func_impl: TokenStream, lexerdef_ty: TokenStream) -> TokenStream + where + LexerTypesT: LexerTypes, + LexerTypesT::StorageT: ToTokens, + usize: num_traits::AsPrimitive, + { + let mut token_consts = TokenStream::new(); + if let Some(rim) = &self.rule_ids_map { + let mut rim_sorted = Vec::from_iter(rim.iter()); + rim_sorted.sort_by_key(|(k, _)| *k); + for (name, id) in rim_sorted { + if RE_TOKEN_ID.is_match(name) { + let tok_ident = format_ident!("N_{}", name.to_ascii_uppercase()); + let storaget = + str::parse::(type_name::()).unwrap(); + // Code gen for the constant token values. + let tok_const = quote! { + #[allow(dead_code)] + pub const #tok_ident: #storaget = #id; + }; + token_consts.extend(tok_const) + } + } + } + let token_consts = token_consts.into_iter(); + let lexerdef_param = str::parse::(type_name::()).unwrap(); + let mod_vis = &self.visibility; + let mod_name = &self.mod_name; + // Code gen for the generated module. + quote! { + #mod_vis mod #mod_name { + use ::lrlex::{LexerDef, Rule, StartState}; + #[allow(dead_code)] + pub fn lexerdef() -> #lexerdef_ty<#lexerdef_param> { + #lexerdef_func_impl + } + + #(#token_consts)* + } + } + } + + fn gen_unformatted_source(&self, token_stream: TokenStream) -> String { + token_stream.to_string() + } + + fn gen_formatted_source(&self, unformatted: String, timestamp: &str) -> String { + let mut outs = String::new(); + write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok(); + outs.push_str( + &syn::parse_str(&unformatted) + .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) + .unwrap_or(unformatted), + ); + outs + } + + pub(crate) fn generate(&self, timestamp: &str) -> String + where + LexerTypesT::StorageT: ToTokens + TryFrom, + { + let mut lexerdef_func_impl = self.gen_lex_flags(); + self.gen_lexerdef(&mut lexerdef_func_impl); + let lexerdef_ty = self.gen_lexerkind(&mut lexerdef_func_impl); + let module_impl = self.gen_module(lexerdef_func_impl, lexerdef_ty); + self.gen_formatted_source(self.gen_unformatted_source(module_impl), timestamp) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct TokenMapCodegen { + mod_name: String, + token_map: Vec<(String, TokenStream)>, + rename_map: Option>, + allow_dead_code: bool, + _marker: PhantomData, +} + +impl TokenMapCodegen { + pub(crate) fn new( + mod_name: impl Into, + token_map: impl Borrow>, + ) -> Self { + Self { + mod_name: mod_name.into(), + token_map: token_map + .borrow() + .iter() + .map(|(tok_name, tok_value)| (tok_name.clone(), tok_value.to_token_stream())) + .collect(), + rename_map: None, + allow_dead_code: false, + _marker: PhantomData, + } + } + pub(crate) fn rename_map(mut self, rename_map: Option) -> Self + where + M: IntoIterator, + I: Borrow<(K, V)>, + K: AsRef, + V: AsRef, + { + self.rename_map = rename_map.map(|rename_map| { + rename_map + .into_iter() + .map(|it| { + let (k, v) = it.borrow(); + let k = k.as_ref().into(); + let v = v.as_ref().into(); + (k, v) + }) + .collect() + }); + self + } + + pub(crate) fn allow_dead_code(mut self, allow_dead_code: bool) -> Self { + self.allow_dead_code = allow_dead_code; + self + } + + pub(crate) fn generate(&self) -> Result> { + // Record the time that this version of lrlex was built. If the source code changes and rustc + // forces a recompile, this will change this value, causing anything which depends on this + // build of lrlex to be recompiled too. + let mut outs = String::new(); + let timestamp = env!("VERGEN_BUILD_TIMESTAMP"); + let mod_ident = format_ident!("{}", self.mod_name); + write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok(); + let storaget = str::parse::(type_name::()).unwrap(); + // Sort the tokens so that they're always in the same order. + // This will prevent unneeded rebuilds. + let mut token_map_sorted = self.token_map.clone(); + token_map_sorted.sort_by(|(l, _), (r, _)| l.cmp(r)); + let (token_array, tokens) = token_map_sorted + .iter() + .map(|(k, id)| { + let name = match &self.rename_map { + Some(rmap) => rmap.get(k).unwrap_or(k), + _ => k, + }; + let tok_ident: Ident = syn::parse_str(&format!("T_{}", name.to_ascii_uppercase())) + .map_err(|e| { + format!( + "token name {:?} is not a valid Rust identifier: {}; \ + consider renaming it via `CTTokenMapBuilder::rename_map`.", + name, e + ) + })?; + Ok(( + // Note: the array of all tokens can't use `tok_ident` because + // it will confuse the dead code checker. For this reason, + // we use `id` here. + quote! { + #id, + }, + quote! { + pub const #tok_ident: #storaget = #id; + }, + )) + }) + .collect::>>()?; + let unused_annotation = if self.allow_dead_code { + quote! {#[allow(dead_code)]} + } else { + quote! {} + }; + // Since the formatter doesn't preserve comments and we don't want to lose build time, + // just format the module contents. + let unformatted = quote! { + #unused_annotation + mod #mod_ident { + #tokens + #[allow(dead_code)] + pub const TOK_IDS: &[#storaget] = &[#token_array]; + } + } + .to_string(); + let out_mod = syn::parse_str(&unformatted) + .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) + .unwrap_or(unformatted); + outs.push_str(&out_mod); + Ok(outs) + } + + pub(crate) fn mod_name(&self) -> &str { + self.mod_name.as_str() + } +} +/// Indents a multi-line string and trims any trailing newline. +/// This currently assumes that indentation on blank lines does not matter. +/// +/// The algorithm used by this function is: +/// 1. Prefix `s` with the indentation, indenting the first line. +/// 2. Trim any trailing newlines. +/// 3. Replace all newlines with `\n{indent}`` to indent all lines after the first. +/// +/// It is plausible that we should a step 4, but currently do not: +/// 4. Replace all `\n{indent}\n` with `\n\n` +fn indent(indent: &str, s: &str) -> String { + format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent)) +} + +/// A string which uses `Display` for it's `Debug` impl. +struct ErrorString(String); +impl fmt::Display for ErrorString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let ErrorString(s) = self; + write!(f, "{}", s) + } +} +impl fmt::Debug for ErrorString { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let ErrorString(s) = self; + write!(f, "{}", s) + } +} +impl Error for ErrorString {} + +impl ToTokens for Visibility { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(match self { + Visibility::Private => quote!(), + Visibility::Public => quote! {pub}, + Visibility::PublicSuper => quote! {pub(super)}, + Visibility::PublicSelf => quote! {pub(self)}, + Visibility::PublicCrate => quote! {pub(crate)}, + Visibility::PublicIn(data) => { + let other = str::parse::(data).unwrap(); + quote! {pub(in #other)} + } + }) + } +} diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 325e6bab4..11436e3a4 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -1,25 +1,15 @@ //! Build grammars at run-time. use cfgrammar::{ - header::{ - GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, - Setting, Value, - }, + header::{Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, Setting, Value}, markmap::MergeBehavior, span::{Location, Span}, }; use glob::glob; -use lrpar::{ - CTParserBuilder, LexerTypes, - diagnostics::{DiagnosticFormatter, SpannedDiagnosticFormatter}, -}; +use lrpar::{CTParserBuilder, LexerTypes, diagnostics::SpannedDiagnosticFormatter}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; -use proc_macro2::{Ident, TokenStream}; -use quote::{ToTokens, TokenStreamExt, format_ident, quote}; -use regex::Regex; -use std::marker::PhantomData; +use quote::ToTokens; use std::{ - any::type_name, borrow::Borrow, collections::{HashMap, HashSet}, env::{current_dir, var}, @@ -33,16 +23,13 @@ use std::{ }; use wincode::SchemaWrite; -use crate::{DefaultLexerTypes, LRNonStreamingLexer, LRNonStreamingLexerDef, LexFlags, LexerDef}; +use crate::{DefaultLexerTypes, LRNonStreamingLexer, LexCodegenBuilder, LexerDef, TokenMapCodegen}; const RUST_FILE_EXT: &str = "rs"; -const ERROR: &str = "[Error]"; +pub(crate) const ERROR: &str = "[Error]"; const WARNING: &str = "[Warning]"; -static RE_TOKEN_ID: LazyLock = - LazyLock::new(|| Regex::new(r"^[a-zA-Z_][a-zA-Z_0-9]*$").unwrap()); - static GENERATED_PATHS: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); @@ -146,22 +133,6 @@ pub enum Visibility { PublicIn(String), } -impl ToTokens for Visibility { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.extend(match self { - Visibility::Private => quote!(), - Visibility::Public => quote! {pub}, - Visibility::PublicSuper => quote! {pub(super)}, - Visibility::PublicSelf => quote! {pub(self)}, - Visibility::PublicCrate => quote! {pub(crate)}, - Visibility::PublicIn(data) => { - let other = str::parse::(data).unwrap(); - quote! {pub(in #other)} - } - }) - } -} - /// Specifies the [Rust Edition] that will be emitted during code generation. /// /// [Rust Edition]: https://doc.rust-lang.org/edition-guide/rust-2021/index.html @@ -173,43 +144,6 @@ pub enum RustEdition { Rust2021, } -/// The quote impl of `ToTokens` for `Option` prints an empty string for `None` -/// and the inner value for `Some(inner_value)`. -/// -/// This wrapper instead emits both `Some` and `None` variants. -/// See: [quote #20](https://github.com/dtolnay/quote/issues/20) -struct QuoteOption(Option); - -impl ToTokens for QuoteOption { - fn to_tokens(&self, tokens: &mut TokenStream) { - tokens.append_all(match self.0 { - Some(ref t) => quote! { ::std::option::Option::Some(#t) }, - None => quote! { ::std::option::Option::None }, - }); - } -} - -/// This wrapper adds a missing impl of `ToTokens` for tuples. -/// For a tuple `(a, b)` emits `(a.to_tokens(), b.to_tokens())` -struct QuoteTuple(T); - -impl ToTokens for QuoteTuple<(A, B)> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let (a, b) = &self.0; - tokens.append_all(quote!((#a, #b))); - } -} - -/// The wrapped `&str` value will be emitted with a call to `to_string()` -struct QuoteToString<'a>(&'a str); - -impl ToTokens for QuoteToString<'_> { - fn to_tokens(&self, tokens: &mut TokenStream) { - let x = &self.0; - tokens.append_all(quote! { #x.to_string() }); - } -} - /// A string which uses `Display` for it's `Debug` impl. struct ErrorString(String); impl fmt::Display for ErrorString { @@ -495,69 +429,21 @@ where } let lex_src = read_to_string(lexerp) .map_err(|e| format!("When reading '{}': {e}", lexerp.display()))?; - let lex_diag = SpannedDiagnosticFormatter::new(&lex_src, lexerp); - let mut header = self.header; - let (parsed_header, _) = GrmtoolsSectionParser::new(&lex_src, false) - .parse() - .map_err(|es| { - let mut out = String::new(); - out.push_str(&format!( - "\n{ERROR}{}\n", - lex_diag.file_location_msg(" parsing the `%grmtools` section", None) - )); - for e in es { - out.push_str(&indent(" ", &lex_diag.format_error(e).to_string())); - out.push('\n'); - } - ErrorString(out) - })?; - header.merge_from(parsed_header)?; - header.mark_used(&"lexerkind".to_string()); - let lexerkind = match self.lexerkind { - Some(lexerkind) => lexerkind, - None => { - if let Some(HeaderValue(_, lk_val)) = header.get("lexerkind") { - LexerKind::try_from(lk_val)? - } else { - LexerKind::LRNonStreamingLexer - } - } - }; + let mut cgb = LexCodegenBuilder::new(&lex_src, lexerp, self.header); + let mut codegen = + cgb.build::(self.lexerkind, self.mod_name, self.visibility)?; #[cfg(test)] if let Some(inspect_lexerkind_cb) = self.inspect_lexerkind_cb { - inspect_lexerkind_cb(&lexerkind)? + inspect_lexerkind_cb(codegen.lexerkind())? } - let (lexerdef, lex_flags): (LRNonStreamingLexerDef, LexFlags) = - match lexerkind { - LexerKind::LRNonStreamingLexer => { - let lex_flags = LexFlags::try_from(&mut header)?; - let lexerdef = LRNonStreamingLexerDef::::new_with_options( - &lex_src, lex_flags, - ) - .map_err(|errs| { - let mut out = String::new(); - out.push_str(&format!( - "\n{ERROR}{}\n", - lex_diag.file_location_msg("", None) - )); - for e in errs { - out.push_str(&indent(" ", &lex_diag.format_error(e).to_string())); - out.push('\n'); - } - ErrorString(out) - })?; - let lex_flags = lexerdef.lex_flags().cloned(); - (lexerdef, lex_flags.unwrap()) - } - }; let ct_parser = if let Some(ref lrcfg) = self.lrpar_config { - let mut closure_lexerdef = lexerdef.clone(); + let mut closure_lexerdef = codegen.lexerdef().clone(); let mut ctp = CTParserBuilder::::new().inspect_rt(Box::new( move |yacc_header, rtpb, rule_ids_map, grm_path| { let owned_map = rule_ids_map .iter() - .map(|(x, y)| (&**x, *y)) + .map(|(rule_id, tok)| (&**rule_id, *tok)) .collect::>(); closure_lexerdef.set_rule_ids(&owned_map); yacc_header.mark_used(&"test_files".to_string()); @@ -631,21 +517,13 @@ where } else { None }; - - let mut lexerdef = Box::new(lexerdef); - let unused_header_values = header.unused(); - if !unused_header_values.is_empty() { - return Err( - format!("Unused header values: {}", unused_header_values.join(", ")).into(), - ); - } - - let (mut missing_from_lexer, missing_from_parser) = match self.rule_ids_map { - Some(ref rim) => { + let lexerdef = Box::new(codegen.lexerdef_mut()); + let (mut missing_from_lexer, missing_from_parser) = match &self.rule_ids_map { + Some(rim) => { // Convert from HashMap to HashMap<&str, _> let owned_map = rim .iter() - .map(|(x, y)| (&**x, *y)) + .map(|(rule_id, tok)| (&**rule_id, *tok)) .collect::>(); let (x, y) = lexerdef.set_rule_ids_spanned(&owned_map); ( @@ -660,6 +538,8 @@ where None => (None, None), }; + codegen.set_rule_ids_map(self.rule_ids_map); + let lexerdef = codegen.lexerdef(); if let Some(mut mfl) = missing_from_lexer.take() { for tok in &lexerdef.expected_missing_tokens { mfl.remove(tok.as_str()); @@ -735,10 +615,10 @@ where outs.push(format!("{error_prefix} these tokens are not referenced in the grammar but defined as follows")); outs.push(format!( "{err_indent} {}", - lex_diag.file_location_msg("in the lexer", None) + cgb.lex_diag().file_location_msg("in the lexer", None) )); for (_, span) in mfp { - let error_contents = lex_diag.underline_span_with_text( + let error_contents = cgb.lex_diag().underline_span_with_text( *span, "Missing from parser".to_string(), '^', @@ -761,162 +641,11 @@ where panic!(); } - let mod_name = match self.mod_name { - Some(s) => s.to_owned(), - None => { - // The user hasn't specified a module name, so we create one automatically: what we - // do is strip off all the filename extensions (note that it's likely that inp ends - // with `l.rs`, so we potentially have to strip off more than one extension) and - // then add `_l` to the end. - let mut stem = lexerp.to_str().unwrap(); - loop { - let new_stem = Path::new(stem).file_stem().unwrap().to_str().unwrap(); - if stem == new_stem { - break; - } - stem = new_stem; - } - format!("{}_l", stem) - } - }; - let mod_name = - match syn::parse_str::(&mod_name) { - Ok(s) => s, - Err(e) => return Err(format!( - "CTLexerBuilder::mod_name(\"{}\") is not a valid rust identifier due to '{}'", - mod_name, e - ) - .into()), - }; - let mut lexerdef_func_impl = { - let LexFlags { - allow_wholeline_comments, - dot_matches_new_line, - multi_line, - octal, - posix_escapes, - case_insensitive, - unicode, - swap_greed, - ignore_whitespace, - size_limit, - dfa_size_limit, - nest_limit, - } = lex_flags; - let allow_wholeline_comments = QuoteOption(allow_wholeline_comments); - let dot_matches_new_line = QuoteOption(dot_matches_new_line); - let multi_line = QuoteOption(multi_line); - let octal = QuoteOption(octal); - let posix_escapes = QuoteOption(posix_escapes); - let case_insensitive = QuoteOption(case_insensitive); - let unicode = QuoteOption(unicode); - let swap_greed = QuoteOption(swap_greed); - let ignore_whitespace = QuoteOption(ignore_whitespace); - let size_limit = QuoteOption(size_limit); - let dfa_size_limit = QuoteOption(dfa_size_limit); - let nest_limit = QuoteOption(nest_limit); - - // Code gen for the lexerdef() `lex_flags` variable. - quote! { - let mut lex_flags = ::lrlex::DEFAULT_LEX_FLAGS; - lex_flags.allow_wholeline_comments = #allow_wholeline_comments.or(::lrlex::DEFAULT_LEX_FLAGS.allow_wholeline_comments); - lex_flags.dot_matches_new_line = #dot_matches_new_line.or(::lrlex::DEFAULT_LEX_FLAGS.dot_matches_new_line); - lex_flags.multi_line = #multi_line.or(::lrlex::DEFAULT_LEX_FLAGS.multi_line); - lex_flags.octal = #octal.or(::lrlex::DEFAULT_LEX_FLAGS.octal); - lex_flags.posix_escapes = #posix_escapes.or(::lrlex::DEFAULT_LEX_FLAGS.posix_escapes); - lex_flags.case_insensitive = #case_insensitive.or(::lrlex::DEFAULT_LEX_FLAGS.case_insensitive); - lex_flags.unicode = #unicode.or(::lrlex::DEFAULT_LEX_FLAGS.unicode); - lex_flags.swap_greed = #swap_greed.or(::lrlex::DEFAULT_LEX_FLAGS.swap_greed); - lex_flags.ignore_whitespace = #ignore_whitespace.or(::lrlex::DEFAULT_LEX_FLAGS.ignore_whitespace); - lex_flags.size_limit = #size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.size_limit); - lex_flags.dfa_size_limit = #dfa_size_limit.or(::lrlex::DEFAULT_LEX_FLAGS.dfa_size_limit); - lex_flags.nest_limit = #nest_limit.or(::lrlex::DEFAULT_LEX_FLAGS.nest_limit); - let lex_flags = lex_flags; - } - }; - { - let start_states = lexerdef.iter_start_states(); - let rules = lexerdef.iter_rules().map(|r| { - let tok_id = QuoteOption(r.tok_id); - let n = QuoteOption(r.name().map(QuoteToString)); - let target_state = - QuoteOption(r.target_state().map(|(x, y)| QuoteTuple((x, y)))); - let n_span = r.name_span(); - let regex = QuoteToString(&r.re_str); - let start_states = r.start_states(); - // Code gen to construct a rule. - // - // We cannot `impl ToToken for Rule` because `Rule` never stores `lex_flags`, - // Thus we reference the local lex_flags variable bound earlier. - quote! { - Rule::new(::lrlex::unstable_api::InternalPublicApi, #tok_id, #n, #n_span, #regex, - vec![#(#start_states),*], #target_state, &lex_flags).unwrap() - } - }); - // Code gen for `lexerdef()`s rules and the stack of `start_states`. - lexerdef_func_impl.append_all(quote! { - let start_states: Vec = vec![#(#start_states),*]; - let rules = vec![#(#rules),*]; - }); - } - let lexerdef_ty = match lexerkind { - LexerKind::LRNonStreamingLexer => { - quote!(::lrlex::LRNonStreamingLexerDef) - } - }; - // Code gen for the lexerdef() return value referencing variables bound earlier. - lexerdef_func_impl.append_all(quote! { - #lexerdef_ty::from_rules(start_states, rules) - }); - - let mut token_consts = TokenStream::new(); - if let Some(rim) = self.rule_ids_map { - let mut rim_sorted = Vec::from_iter(rim.iter()); - rim_sorted.sort_by_key(|(k, _)| *k); - for (name, id) in rim_sorted { - if RE_TOKEN_ID.is_match(name) { - let tok_ident = format_ident!("N_{}", name.to_ascii_uppercase()); - let storaget = - str::parse::(type_name::()).unwrap(); - // Code gen for the constant token values. - let tok_const = quote! { - #[allow(dead_code)] - pub const #tok_ident: #storaget = #id; - }; - token_consts.extend(tok_const) - } - } - } - let token_consts = token_consts.into_iter(); - let out_tokens = { - let lexerdef_param = str::parse::(type_name::()).unwrap(); - let mod_vis = self.visibility; - // Code gen for the generated module. - quote! { - #mod_vis mod #mod_name { - use ::lrlex::{LexerDef, Rule, StartState}; - #[allow(dead_code)] - pub fn lexerdef() -> #lexerdef_ty<#lexerdef_param> { - #lexerdef_func_impl - } - - #(#token_consts)* - } - } - }; - // Try and run a code formatter on the generated code. - let unformatted = out_tokens.to_string(); - let mut outs = String::new(); // Record the time that this version of lrlex was built. If the source code changes and rustc // forces a recompile, this will change this value, causing anything which depends on this // build of lrlex to be recompiled too. - let timestamp = env!("VERGEN_BUILD_TIMESTAMP"); - write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok(); - outs.push_str( - &syn::parse_str(&unformatted) - .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) - .unwrap_or(unformatted), - ); + let outs = codegen.generate(env!("VERGEN_BUILD_TIMESTAMP")); + // If the file we're about to write out already exists with the same contents, then we // don't overwrite it (since that will force a recompile of the file, and relinking of the // binary etc). @@ -1292,11 +1021,7 @@ impl CTLexer { /// [custom lexer example]: https://github.com/softdevteam/grmtools/tree/master/lrlex/examples/calc_manual_lex #[derive(Debug, Clone)] pub struct CTTokenMapBuilder { - mod_name: String, - token_map: Vec<(String, TokenStream)>, - rename_map: Option>, - allow_dead_code: bool, - _marker: PhantomData, + codegen: TokenMapCodegen, } impl CTTokenMapBuilder { @@ -1310,15 +1035,7 @@ impl CTTokenMapBuilder { token_map: impl Borrow>, ) -> Self { Self { - mod_name: mod_name.into(), - token_map: token_map - .borrow() - .iter() - .map(|(tok_name, tok_value)| (tok_name.clone(), tok_value.to_token_stream())) - .collect(), - rename_map: None, - allow_dead_code: false, - _marker: PhantomData, + codegen: TokenMapCodegen::new(mod_name, token_map), } } @@ -1342,17 +1059,7 @@ impl CTTokenMapBuilder { K: AsRef, V: AsRef, { - self.rename_map = rename_map.map(|rename_map| { - rename_map - .into_iter() - .map(|it| { - let (k, v) = it.borrow(); - let k = k.as_ref().into(); - let v = v.as_ref().into(); - (k, v) - }) - .collect() - }); + self.codegen = self.codegen.rename_map(rename_map); self } @@ -1363,74 +1070,15 @@ impl CTTokenMapBuilder { /// get a warning if your custom lexer doesn't use any of them. /// This function can be used to disable this behavior. pub fn allow_dead_code(mut self, allow_dead_code: bool) -> Self { - self.allow_dead_code = allow_dead_code; + self.codegen = self.codegen.allow_dead_code(allow_dead_code); self } /// Build the token map module. pub fn build(&self) -> Result<(), Box> { - // Record the time that this version of lrlex was built. If the source code changes and rustc - // forces a recompile, this will change this value, causing anything which depends on this - // build of lrlex to be recompiled too. - let mut outs = String::new(); - let timestamp = env!("VERGEN_BUILD_TIMESTAMP"); - let mod_ident = format_ident!("{}", self.mod_name); - write!(outs, "// lrlex build time: {}\n\n", quote!(#timestamp),).ok(); - let storaget = str::parse::(type_name::()).unwrap(); - // Sort the tokens so that they're always in the same order. - // This will prevent unneeded rebuilds. - let mut token_map_sorted = self.token_map.clone(); - token_map_sorted.sort_by(|(l, _), (r, _)| l.cmp(r)); - let (token_array, tokens) = token_map_sorted - .iter() - .map(|(k, id)| { - let name = match &self.rename_map { - Some(rmap) => rmap.get(k).unwrap_or(k), - _ => k, - }; - let tok_ident: Ident = syn::parse_str(&format!("T_{}", name.to_ascii_uppercase())) - .map_err(|e| { - format!( - "token name {:?} is not a valid Rust identifier: {}; \ - consider renaming it via `CTTokenMapBuilder::rename_map`.", - name, e - ) - })?; - Ok(( - // Note: the array of all tokens can't use `tok_ident` because - // it will confuse the dead code checker. For this reason, - // we use `id` here. - quote! { - #id, - }, - quote! { - pub const #tok_ident: #storaget = #id; - }, - )) - }) - .collect::>>()?; - let unused_annotation = if self.allow_dead_code { - quote! {#[allow(dead_code)]} - } else { - quote! {} - }; - // Since the formatter doesn't preserve comments and we don't want to lose build time, - // just format the module contents. - let unformatted = quote! { - #unused_annotation - mod #mod_ident { - #tokens - #[allow(dead_code)] - pub const TOK_IDS: &[#storaget] = &[#token_array]; - } - } - .to_string(); - let out_mod = syn::parse_str(&unformatted) - .map(|syntax_tree| prettyplease::unparse(&syntax_tree)) - .unwrap_or(unformatted); - outs.push_str(&out_mod); + let outs = self.codegen.generate()?; let mut outp = PathBuf::from(var("OUT_DIR")?); - outp.push(&self.mod_name); + outp.push(self.codegen.mod_name()); outp.set_extension("rs"); // If the file we're about to write out already exists with the same contents, then we @@ -1464,20 +1112,6 @@ pub fn ct_token_map( .build() } -/// Indents a multi-line string and trims any trailing newline. -/// This currently assumes that indentation on blank lines does not matter. -/// -/// The algorithm used by this function is: -/// 1. Prefix `s` with the indentation, indenting the first line. -/// 2. Trim any trailing newlines. -/// 3. Replace all newlines with `\n{indent}`` to indent all lines after the first. -/// -/// It is plausible that we should a step 4, but currently do not: -/// 4. Replace all `\n{indent}\n` with `\n\n` -fn indent(indent: &str, s: &str) -> String { - format!("{indent}{}\n", s.trim_end_matches('\n')).replace('\n', &format!("\n{}", indent)) -} - // It isn't clear to me why this test isn't working on wasm32, // as the `workspace_runner` should allow access to `OUT_DIR` // perhaps it is related to absolute paths diff --git a/lrlex/src/lib/mod.rs b/lrlex/src/lib/mod.rs index f32cf641a..e7451a3ff 100644 --- a/lrlex/src/lib/mod.rs +++ b/lrlex/src/lib/mod.rs @@ -14,6 +14,7 @@ use std::{error::Error, fmt}; +mod codegen; mod ctbuilder; #[doc(hidden)] pub mod defaults; @@ -38,6 +39,7 @@ pub use crate::{ use cfgrammar::header::{HeaderError, HeaderErrorKind}; use cfgrammar::yacc::parser::SpansKind; use cfgrammar::{Span, Spanned}; +use codegen::{LexCodegenBuilder, TokenMapCodegen}; pub type LexBuildResult = Result>;