From a6d5fb318afd453a01257b1463f84a86c5a7e561 Mon Sep 17 00:00:00 2001 From: Wojtek Figat Date: Tue, 12 Jan 2021 14:14:08 +0100 Subject: [PATCH] Fix default field value parsing to skip whitespaces --- .../Bindings/BindingsGenerator.Parsing.cs | 2 +- Source/Tools/Flax.Build/Utilities/Tokenizer.cs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs index 916916995..1c4cadb4a 100644 --- a/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs +++ b/Source/Tools/Flax.Build/Bindings/BindingsGenerator.Parsing.cs @@ -1038,7 +1038,7 @@ namespace Flax.Build.Bindings var token = context.Tokenizer.ExpectAnyTokens(new[] { TokenType.SemiColon, TokenType.Equal, TokenType.LeftBracket, TokenType.Colon }); if (token.Type == TokenType.Equal) { - context.Tokenizer.SkipUntil(TokenType.SemiColon, out desc.DefaultValue); + context.Tokenizer.SkipUntil(TokenType.SemiColon, out desc.DefaultValue, false); } else if (token.Type == TokenType.LeftBracket) { diff --git a/Source/Tools/Flax.Build/Utilities/Tokenizer.cs b/Source/Tools/Flax.Build/Utilities/Tokenizer.cs index 6913ec31a..e73ddcbca 100644 --- a/Source/Tools/Flax.Build/Utilities/Tokenizer.cs +++ b/Source/Tools/Flax.Build/Utilities/Tokenizer.cs @@ -460,6 +460,24 @@ namespace Flax.Build } } + /// + /// Skips all tokens until the tokenizer steps into token of given type (and it is also skipped, so, NextToken will give the next token). + /// + /// The expected token type. + /// The output contents of the skipped tokens. + /// When false, all white-space tokens will be ignored. + public void SkipUntil(TokenType tokenType, out string context, bool includeWhitespaces) + { + context = string.Empty; + while (NextToken(true).Type != tokenType) + { + var token = CurrentToken; + if (!includeWhitespaces && (token.Type == TokenType.Newline || token.Type == TokenType.Whitespace)) + continue; + context += token.Value; + } + } + /// /// Disposes the . ///