commit 386cf95ca17d28ae4bc7db92a784ab8f6dde9f8b
Author: Lukas Fittl <lukas@fittl.com>
Date:   Sun Jan 3 15:59:40 2021 -0800

    pg_query: Track yyllocend in lexer
    
    This is helpful for tracking the extent of tokens in the scan output,
    as this is made available by pg_query for uses such as syntax highlighting.

diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index a0dfcf3b99..768170bb30 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -513,6 +513,7 @@ other			.
 					{
 						/* If NCHAR isn't a keyword, just return "n" */
 						yylval->str = pstrdup("n");
+						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 						return IDENT;
 					}
 				}
@@ -581,9 +582,11 @@ other			.
 					{
 						case xb:
 							yylval->str = litbufdup(yyscanner);
+							yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 							return BCONST;
 						case xh:
 							yylval->str = litbufdup(yyscanner);
+							yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 							return XCONST;
 						case xq:
 						case xe:
@@ -596,9 +599,11 @@ other			.
 											   yyextra->literallen,
 											   false);
 							yylval->str = litbufdup(yyscanner);
+							yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 							return SCONST;
 						case xus:
 							yylval->str = litbufdup(yyscanner);
+							yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 							return USCONST;
 						default:
 							yyerror("unhandled previous state in xqs");
@@ -736,6 +741,7 @@ other			.
 						yyextra->dolqstart = NULL;
 						BEGIN(INITIAL);
 						yylval->str = litbufdup(yyscanner);
+						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 						return SCONST;
 					}
 					else
@@ -781,6 +787,7 @@ other			.
 					if (yyextra->literallen >= NAMEDATALEN)
 						truncate_identifier(ident, yyextra->literallen, true);
 					yylval->str = ident;
+					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 					return IDENT;
 				}
 <xui>{dquote}	{
@@ -789,6 +796,7 @@ other			.
 						yyerror("zero-length delimited identifier");
 					/* can't truncate till after we de-escape the ident */
 					yylval->str = litbufdup(yyscanner);
+					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 					return UIDENT;
 				}
 <xd,xui>{xddouble}	{
@@ -808,6 +816,7 @@ other			.
 					/* and treat it as {identifier} */
 					ident = downcase_truncate_identifier(yytext, yyleng, true);
 					yylval->str = ident;
+					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 					return IDENT;
 				}
 
@@ -1068,6 +1077,7 @@ other			.
 					 */
 					ident = downcase_truncate_identifier(yytext, yyleng, true);
 					yylval->str = ident;
+					yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
 					return IDENT;
 				}
 
diff --git a/src/include/parser/scanner.h b/src/include/parser/scanner.h
index a27352afc1..29852fbd86 100644
--- a/src/include/parser/scanner.h
+++ b/src/include/parser/scanner.h
@@ -113,6 +113,8 @@ typedef struct core_yy_extra_type
 	/* state variables for literal-lexing warnings */
 	bool		warn_on_first_escape;
 	bool		saw_non_ascii;
+
+	int yyllocend;
 } core_yy_extra_type;
 
 /*