* [gentoo-commits] proj/libbash:master commit in: bashast/, test/, bashast/gunit/
@ 2011-04-09 13:08 Petteri Räty
0 siblings, 0 replies; 5+ messages in thread
From: Petteri Räty @ 2011-04-09 13:08 UTC (permalink / raw
To: gentoo-commits
commit: 842c390d16c128f5271f193b26bd35a0a5c97ef2
Author: Mu Qiao <qiaomuf <AT> gentoo <DOT> org>
AuthorDate: Fri Apr 8 13:53:15 2011 +0000
Commit: Petteri Räty <betelgeuse <AT> gentoo <DOT> org>
CommitDate: Sat Apr 9 13:02:52 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/libbash.git;a=commit;h=842c390d
Transform ESC_CHAR token into a parser rule
ESC_CHAR results in huge lexer file size. After the transformation,
the size of lexer is reduced by 1/3.
---
bashast/bashast.g | 16 +++++++++-------
bashast/gunit/fname.gunit | 10 +++++-----
test/post_check.cpp | 2 +-
3 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/bashast/bashast.g b/bashast/bashast.g
index 6230b66..356142a 100644
--- a/bashast/bashast.g
+++ b/bashast/bashast.g
@@ -363,10 +363,11 @@ ns_str_part
ns_str_part_no_res
: num
| name
+ | esc_char
|OTHER|EQUALS|PCT|PCTPCT|MINUS|DOT|DOTDOT|COLON|TEST_EXPR|'_'
|TILDE|MUL_ASSIGN|DIVIDE_ASSIGN|MOD_ASSIGN|PLUS_ASSIGN|MINUS_ASSIGN
|TIME_POSIX|LSHIFT_ASSIGN|RSHIFT_ASSIGN|AND_ASSIGN|XOR_ASSIGN
- |OR_ASSIGN|ESC_CHAR|CARET;
+ |OR_ASSIGN|CARET;
//strings with no slashes, used in certain variable expansions
ns_str : ns_str_part* -> ^(STRING ns_str_part*);
//Generic strings/filenames.
@@ -493,6 +494,7 @@ parens : LPAREN BLANK* RPAREN;
name : NAME
| LETTER
| '_';
+esc_char: ESC (DIGIT DIGIT? DIGIT?|LETTER ALPHANUM ALPHANUM?|.);
//****************
// TOKENS/LEXER RULES
@@ -602,17 +604,17 @@ LOGICOR : '||';
EXPORT : 'export';
//Tokens for strings
CONTINUE_LINE
- : ('\\' EOL)+{$channel=HIDDEN;};
+ : (ESC EOL)+{$channel=HIDDEN;};
ESC_RPAREN
- : '\\' RPAREN;
+ : ESC RPAREN;
ESC_LPAREN
- : '\\' LPAREN;
-ESC_LT : '\\''<';
-ESC_GT : '\\''>';
+ : ESC LPAREN;
+ESC_LT : ESC'<';
+ESC_GT : ESC'>';
//For pipeline
TIME_POSIX
: '-p';
//Handle ANSI C escaped characters: escaped octal, escaped hex, escaped ctrl+ chars, then all others
-ESC_CHAR: '\\' (('0'..'7')('0'..'7')('0'..'7')?|'x'('0'..'9'|'a'..'f'|'A'..'F')('0'..'9'|'a'..'f'|'A'..'F')?|'c'.|.);
+ESC : '\\';
NAME : (LETTER|'_')(ALPHANUM|'_')+;
OTHER : .;
diff --git a/bashast/gunit/fname.gunit b/bashast/gunit/fname.gunit
index 0272c37..e783504 100644
--- a/bashast/gunit/fname.gunit
+++ b/bashast/gunit/fname.gunit
@@ -42,11 +42,11 @@ fname:
"ab@(gh|i)" -> (STRING ab (MATCH_EXACTLY_ONE (STRING gh) (STRING i)))
"ab!(gh|i)" -> (STRING ab (MATCH_NONE (STRING gh) (STRING i)))
"\"abc\"\'\"\'\"def\"" -> (STRING (DOUBLE_QUOTED_STRING abc) (SINGLE_QUOTED_STRING ") (DOUBLE_QUOTED_STRING def))
-"my\ name\ is" -> (STRING my \ name \ is)
-"octal\007" -> (STRING octal \007)
-"hex\xaF" -> (STRING hex \xaF)
-"ctrlx\cx" -> (STRING ctrlx \cx)
-"tab\\ttab" -> "(STRING tab \\\t tab)"
+"my\ name\ is" -> (STRING my \ name \ is)
+"octal\007" -> (STRING octal \ 007)
+"hex\xaF" -> (STRING hex \ xaF)
+"ctrlx\cx" -> (STRING ctrlx \ cx)
+"tab\\ttab" -> "(STRING tab \\ \t tab)"
"abc[def]" -> (STRING abc (MATCH_PATTERN def))
"a[]" -> (STRING a [ ])
"ab[d-h]" -> (STRING ab (MATCH_PATTERN d - h))
diff --git a/test/post_check.cpp b/test/post_check.cpp
index 1385290..ebe0409 100644
--- a/test/post_check.cpp
+++ b/test/post_check.cpp
@@ -36,7 +36,7 @@ static void check_file_size(const char *path, uintmax_t size_max)
TEST(post_check, lexer_size)
{
- check_file_size("libbashLexer.c", 1024 * 1024);
+ check_file_size("libbashLexer.c", 512 * 1024);
}
TEST(post_check, parser_size)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/libbash:master commit in: bashast/, test/, bashast/gunit/
@ 2011-04-11 6:50 Petteri Räty
0 siblings, 0 replies; 5+ messages in thread
From: Petteri Räty @ 2011-04-11 6:50 UTC (permalink / raw
To: gentoo-commits
commit: 27bdf540572c482bb0e693f480149999069450fb
Author: Petteri Räty <petsku <AT> petteriraty <DOT> eu>
AuthorDate: Sun Apr 10 18:52:51 2011 +0000
Commit: Petteri Räty <betelgeuse <AT> gentoo <DOT> org>
CommitDate: Mon Apr 11 06:45:53 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/libbash.git;a=commit;h=27bdf540
Simplify case expression rules
With the new implementation the parser is under 70k lines.
---
bashast/bashast.g | 26 ++++++++++----------------
bashast/gunit/compound.gunit | 1 +
test/post_check.cpp | 2 +-
3 files changed, 12 insertions(+), 17 deletions(-)
diff --git a/bashast/bashast.g b/bashast/bashast.g
index a9ded45..985f929 100644
--- a/bashast/bashast.g
+++ b/bashast/bashast.g
@@ -187,21 +187,18 @@ while_expr
until_expr
: UNTIL wspace? istrue=clist semiel DO wspace dothis=clist semiel DONE -> ^(UNTIL $istrue $dothis)
;
+// double semicolon is optional for the last alternative
case_expr
- : CASE^ BLANK!+ word wspace! IN! wspace! (case_stmt wspace!)* last_case? ESAC!;
+ : CASE BLANK+ word wspace IN wspace case_body? ESAC -> ^(CASE word case_body?);
+case_body
+ : case_stmt (wspace* DOUBLE_SEMIC case_stmt)* wspace* DOUBLE_SEMIC? wspace* -> case_stmt*;
case_stmt
-options{greedy=false;}
- : wspace* (LPAREN BLANK*)? pattern (BLANK* PIPE BLANK? pattern)* BLANK* RPAREN wspace* clist wspace* DOUBLE_SEMIC
- -> ^(CASE_PATTERN pattern+ clist)
- | wspace* (LPAREN BLANK*)? pattern (BLANK? PIPE BLANK* pattern)* BLANK* RPAREN wspace* DOUBLE_SEMIC
- -> ^(CASE_PATTERN pattern+)
- ;
-//the last case can have a slightly different structure than the rest; this accounts for that
-last_case
-options{greedy=false;}
- : wspace* (LPAREN BLANK*)? pattern (BLANK* PIPE BLANK? pattern)* BLANK* RPAREN wspace* clist? (wspace* DOUBLE_SEMIC|(BLANK* EOL)+)
- -> ^(CASE_PATTERN pattern+ clist?)
- ;
+ : wspace* (LPAREN BLANK*)? case_pattern (BLANK* PIPE BLANK? case_pattern)* BLANK* RPAREN (wspace* clist)?
+ -> ^(CASE_PATTERN case_pattern+ clist?);
+case_pattern
+ : command_sub
+ | fname
+ | TIMES;
//A grouping of commands executed in a subshell
subshell: LPAREN wspace? clist (BLANK* SEMIC)? (BLANK* EOL)* BLANK* RPAREN -> ^(SUBSHELL clist);
//A grouping of commands executed in the current shell
@@ -336,9 +333,6 @@ word : (brace_expansion) => brace_expansion
| (arithmetic_expansion) => arithmetic_expansion
| fname;
-pattern : command_sub
- | fname
- | TIMES;
num
options{k=1;backtrack=false;}
: DIGIT|NUMBER;
diff --git a/bashast/gunit/compound.gunit b/bashast/gunit/compound.gunit
index 0814978..4e931fd 100644
--- a/bashast/gunit/compound.gunit
+++ b/bashast/gunit/compound.gunit
@@ -47,6 +47,7 @@ subshell:
)" -> (SUBSHELL (LIST (COMMAND (STRING cat) time)))
case_expr:
+"case a in esac" -> (case (STRING a))
"case `echo asdf` in
esac" -> (case (COMMAND_SUB (COMMAND (STRING echo) (STRING asdf))))
diff --git a/test/post_check.cpp b/test/post_check.cpp
index ebe0409..5fe39b2 100644
--- a/test/post_check.cpp
+++ b/test/post_check.cpp
@@ -41,7 +41,7 @@ TEST(post_check, lexer_size)
TEST(post_check, parser_size)
{
- check_file_size("libbashParser.c", 4096 * 1024);
+ check_file_size("libbashParser.c", 2*1024*1024+500*1024);
}
TEST(post_check, walker_size)
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/libbash:master commit in: bashast/, test/, bashast/gunit/
@ 2011-06-01 12:03 Petteri Räty
0 siblings, 0 replies; 5+ messages in thread
From: Petteri Räty @ 2011-06-01 12:03 UTC (permalink / raw
To: gentoo-commits
commit: 5f561a36bfefb24ad61f2848536317e62328301c
Author: Mu Qiao <qiaomuf <AT> gentoo <DOT> org>
AuthorDate: Mon May 30 08:42:14 2011 +0000
Commit: Petteri Räty <betelgeuse <AT> gentoo <DOT> org>
CommitDate: Wed Jun 1 06:12:20 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/libbash.git;a=commit;h=5f561a36
Parser: support nested function definitions
---
bashast/bashast.g | 3 ++-
bashast/gunit/function.gunit | 2 ++
test/verify_error_output_test.sh | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/bashast/bashast.g b/bashast/bashast.g
index c560644..88a44cf 100644
--- a/bashast/bashast.g
+++ b/bashast/bashast.g
@@ -111,7 +111,7 @@ flcomment
clist
: list_level_2 -> ^(LIST list_level_2);
list_level_1
- : (function|pipeline) (BLANK!*(LOGICAND^|LOGICOR^)BLANK!* (function|pipeline))*;
+ : pipeline (BLANK!*(LOGICAND^|LOGICOR^)BLANK!* pipeline)*;
// ';' '&' and EOL have lower operator precedence than '&&' and '||' so we need level2 here
list_level_2
: list_level_1 (BLANK!? command_separator (BLANK!? EOL!)* BLANK!? list_level_1)*;
@@ -127,6 +127,7 @@ time_posix
//The structure of a command in bash
command
: compound_command
+ | function
| simple_command;
//Simple bash commands
simple_command
diff --git a/bashast/gunit/function.gunit b/bashast/gunit/function.gunit
index 6ee6c68..612ff4a 100644
--- a/bashast/gunit/function.gunit
+++ b/bashast/gunit/function.gunit
@@ -40,6 +40,8 @@ function:
"function help { echo 3; } 2> /dev/null > output" OK
"xorg-2_reconf_source() { :; }" -> (function (STRING xorg - 2 _reconf_source) (CURRENT_SHELL (LIST (COMMAND (STRING :)))))
+"function out() { function inner() { :; }; }" -> (function (STRING out) (CURRENT_SHELL (LIST (function (STRING inner) (CURRENT_SHELL (LIST (COMMAND (STRING :))))))))
+
function_name:
"xemacs-packages_src_unpack" OK
"while" OK
diff --git a/test/verify_error_output_test.sh b/test/verify_error_output_test.sh
index b3c2eed..e50252f 100755
--- a/test/verify_error_output_test.sh
+++ b/test/verify_error_output_test.sh
@@ -2,4 +2,4 @@
illegal="${srcdir}/scripts/illegal_script.sh"
output=$(./variable_printer "$illegal" 2>&1)
-[[ $output == "${illegal}(1) : error 5 : Unexpected token, at offset 3"* ]]
+[[ $output == "${illegal}(1) : error 3 : 128:1: command : ( compound_command | function | simple_command );, at offset 3"* ]]
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/libbash:master commit in: bashast/, test/, bashast/gunit/
@ 2011-06-16 16:53 Petteri Räty
0 siblings, 0 replies; 5+ messages in thread
From: Petteri Räty @ 2011-06-16 16:53 UTC (permalink / raw
To: gentoo-commits
commit: 63ba78213c1ed411fedf92d99c44fd827871af3b
Author: Mu Qiao <qiaomuf <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 16 08:58:36 2011 +0000
Commit: Petteri Räty <betelgeuse <AT> gentoo <DOT> org>
CommitDate: Thu Jun 16 08:58:36 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/libbash.git;a=commit;h=63ba7821
Parser: support regular expression in keyword test
---
bashast/bashast.g | 6 +++++-
bashast/gunit/cond_main.gunit | 1 +
test/verify_error_output_test.sh | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/bashast/bashast.g b/bashast/bashast.g
index fd626b7..ca3abc9 100644
--- a/bashast/bashast.g
+++ b/bashast/bashast.g
@@ -69,6 +69,7 @@ tokens{
EXTENDED_MATCH_ANY;
EXTENDED_MATCH_AT_LEAST_ONE;
MATCH_PATTERN;
+ MATCH_REGULAR_EXPRESSION;
NOT_MATCH_PATTERN;
MATCH_ANY;
MATCH_ANY_EXCEPT;
@@ -345,7 +346,10 @@ cond_primary
| keyword_cond_unary
| fname;
keyword_cond_binary
- : cond_part BLANK!* binary_str_op_keyword^ BLANK!? cond_part;
+ : cond_part BLANK* EQUALS TILDE BLANK? bash_pattern_part -> ^(MATCH_REGULAR_EXPRESSION cond_part ^(STRING bash_pattern_part))
+ | cond_part BLANK!* binary_str_op_keyword^ BLANK!? cond_part;
+bash_pattern_part
+ :( ESC BLANK |ESC RSQUARE | (~(BLANK|RSQUARE)))+;
keyword_cond_unary
: uop^ BLANK!+ cond_part;
builtin_cond_primary
diff --git a/bashast/gunit/cond_main.gunit b/bashast/gunit/cond_main.gunit
index eeef052..2c1eee7 100644
--- a/bashast/gunit/cond_main.gunit
+++ b/bashast/gunit/cond_main.gunit
@@ -30,3 +30,4 @@ cond_expr:
"[ a = b ]" -> (BUILTIN_TEST (= (STRING a) (STRING b)))
"[ a == b ]" -> (BUILTIN_TEST (= (STRING a) (STRING b)))
"[ a != b ]" -> (BUILTIN_TEST (NOT_EQUALS (STRING a) (STRING b)))
+"[[ \"${DISTUTILS_SRC_TEST}\" =~ ^(setup\.py|nosetests|py\.test|trial(\ .*)?)$ ]]" -> (KEYWORD_TEST (MATCH_REGULAR_EXPRESSION (STRING (DOUBLE_QUOTED_STRING (VAR_REF DISTUTILS_SRC_TEST))) (STRING ^ ( setup \ . py | nosetests | py \ . test | trial ( \ . * ) ? ) $)))
diff --git a/test/verify_error_output_test.sh b/test/verify_error_output_test.sh
index ebc5b3c..f8a2a7b 100755
--- a/test/verify_error_output_test.sh
+++ b/test/verify_error_output_test.sh
@@ -2,4 +2,4 @@
illegal="${srcdir}/scripts/illegal_script.sh"
output=$(./variable_printer "$illegal" 2>&1)
-[[ $output == "${illegal}(1) : error 3 : 130:1: command_atom : ( compound_command | function | simple_command );, at offset 3"* ]]
+[[ $output == "${illegal}(1) : error 3 : 131:1: command_atom : ( compound_command | function | simple_command );, at offset 3"* ]]
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [gentoo-commits] proj/libbash:master commit in: bashast/, test/, bashast/gunit/
@ 2011-08-04 13:53 Petteri Räty
0 siblings, 0 replies; 5+ messages in thread
From: Petteri Räty @ 2011-08-04 13:53 UTC (permalink / raw
To: gentoo-commits
commit: 8d0126787eaf4ee648ebe7f47a8048dc3c15c88e
Author: Mu Qiao <qiaomuf <AT> gentoo <DOT> org>
AuthorDate: Thu Jul 21 09:00:05 2011 +0000
Commit: Petteri Räty <betelgeuse <AT> gentoo <DOT> org>
CommitDate: Tue Aug 2 07:46:29 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=proj/libbash.git;a=commit;h=8d012678
Parser: allow braces in command arguments
---
bashast/bashast.g | 8 +++++++-
bashast/gunit/simp_command.gunit | 2 +-
test/walker_test.cpp | 3 +--
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/bashast/bashast.g b/bashast/bashast.g
index 65cc493..14d7d10 100644
--- a/bashast/bashast.g
+++ b/bashast/bashast.g
@@ -404,7 +404,13 @@ bash_command_arguments
: bash_command_argument_atom+ -> ^(STRING bash_command_argument_atom+);
// TODO support brace expansion and braces
bash_command_argument_atom
- : string_expr_part;
+ : (LBRACE) =>
+ (
+ (brace_expansion) => brace_expansion
+ |LBRACE
+ )
+ | RBRACE
+ | string_expr_part;
parens
: LPAREN BLANK? RPAREN;
diff --git a/bashast/gunit/simp_command.gunit b/bashast/gunit/simp_command.gunit
index d7b25e9..965b351 100644
--- a/bashast/gunit/simp_command.gunit
+++ b/bashast/gunit/simp_command.gunit
@@ -28,7 +28,7 @@ command_atom:
"cat ~/Documents/todo.txt" -> (STRING cat) (STRING ~ / Documents / todo . txt)
"dodir ${foo}/${bar}" -> (STRING dodir) (STRING (VAR_REF foo) / (VAR_REF bar))
"local a=123 b=(1 2 3) c" -> (VARIABLE_DEFINITIONS local (= a (STRING 123)) (= b (ARRAY (STRING 1) (STRING 2) (STRING 3))) (EQUALS c))
-//"echo {}{}}{{{}}{{}" -> (STRING echo) (STRING { } { } } { { { } } { { })
+"echo {}{}}{{{}}{{}" -> (STRING echo) (STRING { } { } } { { { } } { { })
"echo \"ab#af ###\" #abc" -> (STRING echo) (STRING (DOUBLE_QUOTED_STRING ab # af ## #))
command:
diff --git a/test/walker_test.cpp b/test/walker_test.cpp
index 4a69e9f..4f87c26 100644
--- a/test/walker_test.cpp
+++ b/test/walker_test.cpp
@@ -91,7 +91,6 @@ TEST(extglob, used_when_disabled)
}
}
-/*
TEST(brace_expansion, not_in_raw_string)
{
interpreter walker;
@@ -100,4 +99,4 @@ TEST(brace_expansion, not_in_raw_string)
std::istringstream input(script);
bash_ast ast(input);
EXPECT_THROW(ast.interpret_with(walker), libbash::unsupported_exception);
-} */
+}
^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2011-08-04 13:58 UTC | newest]
Thread overview: 5+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2011-08-04 13:53 [gentoo-commits] proj/libbash:master commit in: bashast/, test/, bashast/gunit/ Petteri Räty
-- strict thread matches above, loose matches on Subject: below --
2011-06-16 16:53 Petteri Räty
2011-06-01 12:03 Petteri Räty
2011-04-11 6:50 Petteri Räty
2011-04-09 13:08 Petteri Räty
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox