glsl/glcpp: Drop extra, final newline from most output

The glcpp parser is line-based, so it needs to see a NEWLINE token at the end
of each line. This causes a trick for files that end without a final newline.

Previously, the lexer for glcpp punted in this case by unconditionally
returning a NEWLINE token at end-of-file, (causing most files to have an extra
blank line at the end). Here, we refine this by lexing end-of-file as a
NEWLINE token only if the immediately preceding token was not a NEWLINE token.

The patch is a minor change that only looks huge for two reasons:

	1. Almost all glcpp test result ".expected" files are updated to drop
	   the extra newline.

	2. All return statements from the lexer are adjusted to use a new
	   RETURN_TOKEN macro that tracks the last-token-was-a-newline state.

Reviewed-by: Jordan Justen <jordan.l.justen@intel.com>
This commit is contained in:
Carl Worth 2014-06-20 14:58:48 -07:00
parent 5dbdc341e8
commit 828686d4eb
130 changed files with 51 additions and 165 deletions

View file

@ -60,6 +60,16 @@ void glcpp_set_column (int column_no , yyscan_t yyscanner);
yycolumn = 1; \ yycolumn = 1; \
yylloc->source = 0; \ yylloc->source = 0; \
} while(0) } while(0)
#define RETURN_TOKEN(token) \
do { \
if (token == NEWLINE) \
parser->last_token_was_newline = 1; \
else \
parser->last_token_was_newline = 0; \
return (token); \
} while(0)
%} %}
%option bison-bridge bison-locations reentrant noyywrap %option bison-bridge bison-locations reentrant noyywrap
@ -111,7 +121,7 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
parser->commented_newlines--; parser->commented_newlines--;
if (parser->commented_newlines == 0) if (parser->commented_newlines == 0)
BEGIN INITIAL; BEGIN INITIAL;
return NEWLINE; RETURN_TOKEN (NEWLINE);
} }
/* The handling of the SKIP vs INITIAL start states requires /* The handling of the SKIP vs INITIAL start states requires
@ -169,13 +179,13 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
<COMMENT>"*"+"/" { <COMMENT>"*"+"/" {
yy_pop_state(yyscanner); yy_pop_state(yyscanner);
if (yyextra->space_tokens) if (yyextra->space_tokens)
return SPACE; RETURN_TOKEN (SPACE);
} }
{HASH}version{HSPACE}+ { {HASH}version{HSPACE}+ {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_VERSION; RETURN_TOKEN (HASH_VERSION);
} }
/* glcpp doesn't handle #extension, #version, or #pragma directives. /* glcpp doesn't handle #extension, #version, or #pragma directives.
@ -184,46 +194,46 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
yylineno++; yylineno++;
yycolumn = 0; yycolumn = 0;
return OTHER; RETURN_TOKEN (OTHER);
} }
{HASH}line{HSPACE}+ { {HASH}line{HSPACE}+ {
return HASH_LINE; RETURN_TOKEN (HASH_LINE);
} }
<SKIP,INITIAL>{ <SKIP,INITIAL>{
{HASH}ifdef { {HASH}ifdef {
yyextra->lexing_directive = 1; yyextra->lexing_directive = 1;
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_IFDEF; RETURN_TOKEN (HASH_IFDEF);
} }
{HASH}ifndef { {HASH}ifndef {
yyextra->lexing_directive = 1; yyextra->lexing_directive = 1;
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_IFNDEF; RETURN_TOKEN (HASH_IFNDEF);
} }
{HASH}if/[^_a-zA-Z0-9] { {HASH}if/[^_a-zA-Z0-9] {
yyextra->lexing_directive = 1; yyextra->lexing_directive = 1;
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_IF; RETURN_TOKEN (HASH_IF);
} }
{HASH}elif/[^_a-zA-Z0-9] { {HASH}elif/[^_a-zA-Z0-9] {
yyextra->lexing_directive = 1; yyextra->lexing_directive = 1;
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_ELIF; RETURN_TOKEN (HASH_ELIF);
} }
{HASH}else { {HASH}else {
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_ELSE; RETURN_TOKEN (HASH_ELSE);
} }
{HASH}endif { {HASH}endif {
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_ENDIF; RETURN_TOKEN (HASH_ENDIF);
} }
} }
@ -258,21 +268,21 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
{HASH}define{HSPACE}+ { {HASH}define{HSPACE}+ {
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
yy_push_state(DEFINE, yyscanner); yy_push_state(DEFINE, yyscanner);
return HASH_DEFINE; RETURN_TOKEN (HASH_DEFINE);
} }
/* An identifier immediately followed by '(' */ /* An identifier immediately followed by '(' */
<DEFINE>{IDENTIFIER}/"(" { <DEFINE>{IDENTIFIER}/"(" {
yy_pop_state(yyscanner); yy_pop_state(yyscanner);
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return FUNC_IDENTIFIER; RETURN_TOKEN (FUNC_IDENTIFIER);
} }
/* An identifier not immediately followed by '(' */ /* An identifier not immediately followed by '(' */
<DEFINE>{IDENTIFIER} { <DEFINE>{IDENTIFIER} {
yy_pop_state(yyscanner); yy_pop_state(yyscanner);
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return OBJ_IDENTIFIER; RETURN_TOKEN (OBJ_IDENTIFIER);
} }
/* Whitespace */ /* Whitespace */
@ -284,7 +294,7 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
<DEFINE>[/][^*]{NONSPACE}* { <DEFINE>[/][^*]{NONSPACE}* {
BEGIN INITIAL; BEGIN INITIAL;
glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext); glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
return INTEGER_STRING; RETURN_TOKEN (INTEGER_STRING);
} }
/* A character that can't start an identifier, comment, or /* A character that can't start an identifier, comment, or
@ -292,98 +302,98 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
<DEFINE>[^_a-zA-Z/[:space:]]{NONSPACE}* { <DEFINE>[^_a-zA-Z/[:space:]]{NONSPACE}* {
BEGIN INITIAL; BEGIN INITIAL;
glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext); glcpp_error(yylloc, yyextra, "#define followed by a non-identifier: %s", yytext);
return INTEGER_STRING; RETURN_TOKEN (INTEGER_STRING);
} }
{HASH}undef { {HASH}undef {
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH_UNDEF; RETURN_TOKEN (HASH_UNDEF);
} }
{HASH} { {HASH} {
yyextra->space_tokens = 0; yyextra->space_tokens = 0;
return HASH; RETURN_TOKEN (HASH);
} }
{DECIMAL_INTEGER} { {DECIMAL_INTEGER} {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return INTEGER_STRING; RETURN_TOKEN (INTEGER_STRING);
} }
{OCTAL_INTEGER} { {OCTAL_INTEGER} {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return INTEGER_STRING; RETURN_TOKEN (INTEGER_STRING);
} }
{HEXADECIMAL_INTEGER} { {HEXADECIMAL_INTEGER} {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return INTEGER_STRING; RETURN_TOKEN (INTEGER_STRING);
} }
"<<" { "<<" {
return LEFT_SHIFT; RETURN_TOKEN (LEFT_SHIFT);
} }
">>" { ">>" {
return RIGHT_SHIFT; RETURN_TOKEN (RIGHT_SHIFT);
} }
"<=" { "<=" {
return LESS_OR_EQUAL; RETURN_TOKEN (LESS_OR_EQUAL);
} }
">=" { ">=" {
return GREATER_OR_EQUAL; RETURN_TOKEN (GREATER_OR_EQUAL);
} }
"==" { "==" {
return EQUAL; RETURN_TOKEN (EQUAL);
} }
"!=" { "!=" {
return NOT_EQUAL; RETURN_TOKEN (NOT_EQUAL);
} }
"&&" { "&&" {
return AND; RETURN_TOKEN (AND);
} }
"||" { "||" {
return OR; RETURN_TOKEN (OR);
} }
"##" { "##" {
if (parser->is_gles) if (parser->is_gles)
glcpp_error(yylloc, yyextra, "Token pasting (##) is illegal in GLES"); glcpp_error(yylloc, yyextra, "Token pasting (##) is illegal in GLES");
return PASTE; RETURN_TOKEN (PASTE);
} }
"defined" { "defined" {
return DEFINED; RETURN_TOKEN (DEFINED);
} }
{IDENTIFIER} { {IDENTIFIER} {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return IDENTIFIER; RETURN_TOKEN (IDENTIFIER);
} }
{PP_NUMBER} { {PP_NUMBER} {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return OTHER; RETURN_TOKEN (OTHER);
} }
{PUNCTUATION} { {PUNCTUATION} {
return yytext[0]; RETURN_TOKEN (yytext[0]);
} }
{OTHER}+ { {OTHER}+ {
yylval->str = ralloc_strdup (yyextra, yytext); yylval->str = ralloc_strdup (yyextra, yytext);
return OTHER; RETURN_TOKEN (OTHER);
} }
{HSPACE} { {HSPACE} {
if (yyextra->space_tokens) { if (yyextra->space_tokens) {
return SPACE; RETURN_TOKEN (SPACE);
} }
} }
@ -395,7 +405,7 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
yyextra->lexing_directive = 0; yyextra->lexing_directive = 0;
yylineno++; yylineno++;
yycolumn = 0; yycolumn = 0;
return NEWLINE; RETURN_TOKEN (NEWLINE);
} }
<INITIAL,COMMENT,DEFINE><<EOF>> { <INITIAL,COMMENT,DEFINE><<EOF>> {
@ -405,7 +415,8 @@ HEXADECIMAL_INTEGER 0[xX][0-9a-fA-F]+[uU]?
glcpp_error(yylloc, yyextra, "#define without macro name"); glcpp_error(yylloc, yyextra, "#define without macro name");
BEGIN DONE; /* Don't keep matching this rule forever. */ BEGIN DONE; /* Don't keep matching this rule forever. */
yyextra->lexing_directive = 0; yyextra->lexing_directive = 0;
return NEWLINE; if (! parser->last_token_was_newline)
RETURN_TOKEN (NEWLINE);
} }
/* We don't actually use the UNREACHABLE start condition. We /* We don't actually use the UNREACHABLE start condition. We

View file

@ -1308,6 +1308,7 @@ glcpp_parser_create (const struct gl_extensions *extensions, gl_api api)
parser->active = NULL; parser->active = NULL;
parser->lexing_directive = 0; parser->lexing_directive = 0;
parser->space_tokens = 1; parser->space_tokens = 1;
parser->last_token_was_newline = 0;
parser->newline_as_space = 0; parser->newline_as_space = 0;
parser->in_control_line = 0; parser->in_control_line = 0;
parser->paren_count = 0; parser->paren_count = 0;

View file

@ -177,6 +177,7 @@ struct glcpp_parser {
active_list_t *active; active_list_t *active;
int lexing_directive; int lexing_directive;
int space_tokens; int space_tokens;
int last_token_was_newline;
int newline_as_space; int newline_as_space;
int in_control_line; int in_control_line;
int paren_count; int paren_count;

View file

@ -1,2 +1 @@
this is four tokens with spaces this is four tokens with spaces

View file

@ -1,3 +1,2 @@
1 1

View file

@ -1,4 +1,3 @@
1 1

View file

@ -4,4 +4,3 @@
foo foo
bar bar
baz baz

View file

@ -4,4 +4,3 @@
a b c foo a b c foo
b c a bar b c a bar
c a b baz c a b baz

View file

@ -2,4 +2,3 @@
1 1
foo foo

View file

@ -4,4 +4,3 @@
foo foo
2 2

View file

@ -1,3 +1,2 @@
bar bar

View file

@ -2,4 +2,3 @@
()1() ()1()
()2() ()2()

View file

@ -1,3 +1,2 @@
((bar)+1) ((bar)+1)

View file

@ -1,3 +1,2 @@
((bar)*(baz)) ((bar)*(baz))

View file

@ -1,3 +1,2 @@
(this is more than one word) (this is more than one word)

View file

@ -1,3 +1,2 @@
one fish,two fish,red fish,blue fish one fish,two fish,red fish,blue fish

View file

@ -1,4 +1,3 @@
(2*((1+(3)))) (2*((1+(3))))

View file

@ -1,3 +1,2 @@
(argument(including parens)for the win) (argument(including parens)for the win)

View file

@ -6,4 +6,3 @@
2 2
3 4 3 4
5 6 7 5 6 7

View file

@ -1,3 +1,2 @@
foo bar foo bar

View file

@ -1,4 +1,3 @@
success success

View file

@ -1,4 +1,3 @@
success success

View file

@ -2,4 +2,3 @@
success success

View file

@ -2,4 +2,3 @@
success success

View file

@ -1,3 +1,2 @@
foo(2*(3)) foo(2*(3))

View file

@ -1,3 +1,2 @@
foo(2*(foo(2*(3)))) foo(2*(foo(2*(3))))

View file

@ -1,4 +1,3 @@
more success more success

View file

@ -1,4 +1,3 @@
expand(just once) expand(just once)

View file

@ -1,3 +1,2 @@
success success

View file

@ -1,4 +1,3 @@
(two,words) (two,words)

View file

@ -1,3 +1,2 @@
onetoken onetoken

View file

@ -3,4 +3,3 @@ success_1
success_2 success_2

View file

@ -3,4 +3,3 @@ success_1
success_2 success_2
success_3 success_3

View file

@ -5,4 +5,3 @@ success_1
success_2 success_2
success_3 success_3

View file

@ -5,4 +5,3 @@ success_2
success_3 success_3

View file

@ -9,4 +9,3 @@ success_3
success_4 success_4

View file

@ -9,4 +9,3 @@ success_2
success_3 success_3

View file

@ -9,4 +9,3 @@ success_1
success_2 success_2
success_3 success_3

View file

@ -9,4 +9,3 @@ success_1
success_2 success_2

View file

@ -3,4 +3,3 @@
success success

View file

@ -15,4 +15,3 @@ success_2
success_3 success_3

View file

@ -33,4 +33,3 @@ success_5

View file

@ -18,4 +18,3 @@ success_4

View file

@ -13,4 +13,3 @@ success_2
success_3 success_3

View file

@ -32,4 +32,3 @@ success_6

View file

@ -2,4 +2,3 @@
success success

View file

@ -4,4 +4,3 @@ success
success success
success success

View file

@ -2,4 +2,3 @@
12 12
1000 1000
identifier2 identifier2

View file

@ -3,4 +3,3 @@
success success

View file

@ -18,4 +18,3 @@ more code here
are not treated like comments. are not treated like comments.

View file

@ -1,3 +1,2 @@
#version 130 #version 130

View file

@ -15,4 +15,3 @@ success_2
success_3 success_3

View file

@ -1,4 +1,3 @@
success success

View file

@ -38,4 +38,3 @@ success
success success

View file

@ -9,4 +9,3 @@
| | | |
+ + + +
- - - -

View file

@ -1,3 +1,2 @@
1 1 1 1

View file

@ -3,4 +3,3 @@
Success Success

View file

@ -1,2 +1 @@
a = b a = b

View file

@ -1,3 +1,2 @@
success_1 success_2 success_3 success_1 success_2 success_3

View file

@ -1,4 +1,3 @@
0:1(2): preprocessor error: else without #if 0:1(2): preprocessor error: else without #if

View file

@ -1,4 +1,3 @@
0:1(2): preprocessor error: elif without #if 0:1(2): preprocessor error: elif without #if

View file

@ -1,4 +1,3 @@
0:1(2): preprocessor error: #endif without #if 0:1(2): preprocessor error: #endif without #if

View file

@ -17,4 +17,3 @@ Pasting "4" and "+" does not give a valid preprocessing token.
2 2
34 34
45.2 45.2

View file

@ -9,4 +9,3 @@ MULT()
MULT(1) MULT(1)
MULT(1,2,3) MULT(1,2,3)

View file

@ -3,4 +3,3 @@
win win

View file

@ -1,3 +1,2 @@
0:1(2): preprocessor error: #error human error 0:1(2): preprocessor error: #error human error

View file

@ -16,4 +16,3 @@
#line 90 2 #line 90 2
#line 180 2 #line 180 2

View file

@ -1,4 +1,3 @@
0:1(13): preprocessor error: division by 0 in preprocessor directive 0:1(13): preprocessor error: division by 0 in preprocessor directive

View file

@ -1,4 +1,3 @@
B(0, C) B(0, C)

View file

@ -1,4 +1,3 @@
justonetoken justonetoken

View file

@ -1,4 +1,3 @@
onetoken onetoken

Some files were not shown because too many files have changed in this diff Show more