Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add test and fix for buffer over-read in pcre2grep #594

Merged
merged 2 commits into from
Dec 6, 2024
+193 −71
Merged

Add test and fix for buffer over-read in pcre2grep #594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02701bc
Add test for buffer over-read in pcre2grep
NWilson Dec 3, 2024
501593d
Fix UTF-8 decoding to be safer
NWilson Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 45 additions & 6 deletions RunGrepTest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,17 @@ fi
valgrind=
while [ $# -gt 0 ] ; do
case $1 in
valgrind) valgrind="valgrind -q --leak-check=no --smc-check=all-non-file";;
valgrind|-valgrind) valgrind="valgrind -q --leak-check=no --smc-check=all-non-file --error-exitcode=70";;
*) echo "RunGrepTest: Unknown argument $1"; exit 1;;
esac
shift
done

vjs=
pcre2grep_version=`$pcre2grep -V`
if [ "$valgrind" = "" ] ; then
echo "Testing $pcre2grep_version"
else
echo "Testing $pcre2grep_version using valgrind"
$pcre2test -C jit >/dev/null
if [ $? -ne 0 ]; then
vjs="--suppressions=./testdata/valgrind-jit.supp"
fi
fi

# Set up a suitable "diff" command for comparison. Some systems have a diff
Expand Down Expand Up @@ -105,6 +100,16 @@ if [ -z "$srcdir" -o ! -d "$srcdir/testdata" ] ; then
fi
fi

# Set up the path to the valgrind JIT suppressions

vjs=
if [ "$valgrind" != "" ] ; then
$pcre2test -C jit >/dev/null
if [ $? -ne 0 ]; then
vjs="--suppressions=`realpath "$srcdir"`/testdata/valgrind-jit.supp"
fi
fi

# Check for the availability of UTF-8 support

$pcre2test -C unicode >/dev/null
Expand Down Expand Up @@ -318,8 +323,11 @@ echo "RC=$?" >>testtrygrep

echo "---------------------------- Test 46 ------------------------------" >>testtrygrep
(cd $srcdir; $valgrind $vjs $pcre2grep -e 'unopened)' -e abc ./testdata/grepinput) >>testtrygrep 2>&1
echo "RC=$?" >>testtrygrep
(cd $srcdir; $valgrind $vjs $pcre2grep -eabc -e '(unclosed' ./testdata/grepinput) >>testtrygrep 2>&1
echo "RC=$?" >>testtrygrep
(cd $srcdir; $valgrind $vjs $pcre2grep -eabc -e xyz -e '[unclosed' ./testdata/grepinput) >>testtrygrep 2>&1
echo "RC=$?" >>testtrygrep
(cd $srcdir; $valgrind $vjs $pcre2grep --regex=123 -eabc -e xyz -e '[unclosed' ./testdata/grepinput) >>testtrygrep 2>&1
echo "RC=$?" >>testtrygrep

Expand Down Expand Up @@ -975,31 +983,48 @@ printf 'abc\rdef\r\nghi\njkl' >testNinputgrep

printf '%c--------------------------- Test N1 ------------------------------\r\n' - >testtrygrep
$valgrind $vjs $pcre2grep -n -N CR "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep
$valgrind $vjs $pcre2grep -B1 -n -N CR "^def" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N2 ------------------------------\r\n' - >>testtrygrep
$valgrind $vjs $pcre2grep -n --newline=crlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep
$valgrind $vjs $pcre2grep -B1 -n -N CRLF "^ghi" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N3 ------------------------------\r\n' - >>testtrygrep
pattern=`printf 'def\rjkl'`
$valgrind $vjs $pcre2grep -n --newline=cr -F "$pattern" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N4 ------------------------------\r\n' - >>testtrygrep
$valgrind $vjs $pcre2grep -n --newline=crlf -F -f $srcdir/testdata/greppatN4 testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N5 ------------------------------\r\n' - >>testtrygrep
$valgrind $vjs $pcre2grep -n --newline=any "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep
$valgrind $vjs $pcre2grep -B1 -n --newline=any "^def" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N6 ------------------------------\r\n' - >>testtrygrep
$valgrind $vjs $pcre2grep -n --newline=anycrlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep
$valgrind $vjs $pcre2grep -B1 -n --newline=anycrlf "^jkl" testNinputgrep >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N7 ------------------------------\r\n' - >>testtrygrep
printf 'xyz\0abc\0def' >testNinputgrep
$valgrind $vjs $pcre2grep -na --newline=nul "^(abc|def)" testNinputgrep | $tr '\000' '@' >>testtrygrep
echo "RC=$?" >>testtrygrep
$valgrind $vjs $pcre2grep -B1 -na --newline=nul "^(abc|def)" testNinputgrep | $tr '\000' '@' >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test N8 ------------------------------\r\n' - >>testtrygrep
$valgrind $vjs $pcre2grep -na --newline=anycrlf "^a" $srcdir/testdata/grepinputBad8_Trail >>testtrygrep
echo "RC=$?" >>testtrygrep

echo "" >>testtrygrep

$cf $srcdir/testdata/grepoutputN testtrygrep
Expand All @@ -1013,6 +1038,12 @@ if [ $utf8 -ne 0 ] ; then

printf '%c--------------------------- Test UN1 ------------------------------\r\n' - >testtrygrep
$valgrind $vjs $pcre2grep -nau --newline=anycrlf "^(abc|def)" $srcdir/testdata/grepinputUN >>testtrygrep
echo "RC=$?" >>testtrygrep

printf '%c--------------------------- Test UN2 ------------------------------\r\n' - >testtrygrep
$valgrind $vjs $pcre2grep -nauU --newline=anycrlf "^a" $srcdir/testdata/grepinputBad8_Trail >>testtrygrep
echo "RC=$?" >>testtrygrep

echo "" >>testtrygrep

$cf $srcdir/testdata/grepoutputUN testtrygrep
Expand All @@ -1031,16 +1062,22 @@ if $valgrind $vjs $pcre2grep --help | $valgrind $vjs $pcre2grep -q 'callout scri
echo "Testing pcre2grep script callouts"
echo "--- Test 1 ---" >testtrygrep
$valgrind $vjs $pcre2grep '(T)(..(.))(?C"/bin/echo|Arg1: [$1] [$2] [$3]|Arg2: $|${1}$| ($4) ($14) ($0)")()' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep
echo "--- Test 2 ---" >>testtrygrep
$valgrind $vjs $pcre2grep '(T)(..(.))()()()()()()()(..)(?C"/bin/echo|Arg1: [$11] [${11}]")' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep
echo "--- Test 3 ---" >>testtrygrep
$valgrind $vjs $pcre2grep '(T)(?C"|$0:$1$n")' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep
echo "--- Test 4 ---" >>testtrygrep
$valgrind $vjs $pcre2grep '(T)(?C"/bin/echo|$0:$1$n")' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep
echo "--- Test 5 ---" >>testtrygrep
$valgrind $vjs $pcre2grep '(T)(?C"|$1$n")(*F)' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep
echo "--- Test 6 ---" >>testtrygrep
$valgrind $vjs $pcre2grep -m1 '(T)(?C"|$0:$1:$x{41}$o{101}$n")' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep

if $valgrind $vjs $pcre2grep --help | $valgrind $vjs $pcre2grep -q 'Non-fork callout scripts in patterns are supported'; then
nonfork=1
Expand All @@ -1057,8 +1094,10 @@ if $valgrind $vjs $pcre2grep --help | $valgrind $vjs $pcre2grep -q 'callout scri
echo "Testing pcre2grep script callout with UTF-8 features"
echo "--- Test 1 ---" >testtrygrep
$valgrind $vjs $pcre2grep -u '(T)(?C"|$0:$x{a6}$n")' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep
echo "--- Test 2 ---" >>testtrygrep
$valgrind $vjs $pcre2grep -u '(T)(?C"/bin/echo|$0:$x{a6}$n")' $srcdir/testdata/grepinputv >>testtrygrep
echo "RC=$?" >>testtrygrep

if [ $nonfork = 1 ] ; then
$cf $srcdir/testdata/grepoutputCNU testtrygrep
Expand Down
34 changes: 34 additions & 0 deletions RunGrepTest.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,8 +281,11 @@ echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test 46 ------------------------------>>testtrygrep
(pushd %srcdir% & %pcre2grep% -e "unopened)" -e abc ./testdata/grepinput & popd) >>testtrygrep 2>&1
echo RC=^%ERRORLEVEL%>>testtrygrep
(pushd %srcdir% & %pcre2grep% -eabc -e "(unclosed" ./testdata/grepinput & popd) >>testtrygrep 2>&1
echo RC=^%ERRORLEVEL%>>testtrygrep
(pushd %srcdir% & %pcre2grep% -eabc -e xyz -e "[unclosed" ./testdata/grepinput & popd) >>testtrygrep 2>&1
echo RC=^%ERRORLEVEL%>>testtrygrep
(pushd %srcdir% & %pcre2grep% --regex=123 -eabc -e xyz -e "[unclosed" ./testdata/grepinput & popd) >>testtrygrep 2>&1
echo RC=^%ERRORLEVEL%>>testtrygrep

Expand Down Expand Up @@ -937,31 +940,48 @@ echo Testing pcre2grep newline settings

echo ---------------------------- Test N1 ------------------------------>testtrygrep
%pcre2grep% -n -N CR "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep
%pcre2grep% -B1 -n -N CR "^def" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N2 ------------------------------>>testtrygrep
%pcre2grep% -n --newline=crlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep
%pcre2grep% -B1 -n -N CRLF "^ghi" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N3 ------------------------------>>testtrygrep
for /f %%a in ('%printf% "def\rjkl"') do set pattern=%%a
%pcre2grep% -n --newline=cr -F "!pattern!" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N4 ------------------------------>>testtrygrep
%pcre2grep% -n --newline=crlf -F -f %srcdir%\testdata\greppatN4 testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N5 ------------------------------>>testtrygrep
%pcre2grep% -n --newline=any "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep
%pcre2grep% -B1 -n --newline=any "^def" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N6 ------------------------------>>testtrygrep
%pcre2grep% -n --newline=anycrlf "^(abc|def|ghi|jkl)" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep
%pcre2grep% -B1 -n --newline=anycrlf "^jkl" testNinputgrep >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N7 ------------------------------>>testtrygrep
%printf% "xyz\0abc\0def" >testNinputgrep
%pcre2grep% -na --newline=nul "^(abc|def)" testNinputgrep | %trnull% >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep
%pcre2grep% -B1 -na --newline=nul "^(abc|def)" testNinputgrep | %trnull% >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

echo ---------------------------- Test N8 ------------------------------>>testtrygrep
%pcre2grep% -na --newline=anycrlf "^a" %srcdir%\testdata\grepinputBad8_Trail >>testtrygrep
echo RC=^%ERRORLEVEL%>>testtrygrep

%printf% "\n" >>testtrygrep

%cf% %srcdir%\testdata\grepoutputN testtrygrep %cfout%
Expand All @@ -975,6 +995,12 @@ if %utf8% neq 0 (

echo ---------------------------- Test UN1 ------------------------------>testtrygrep
%pcre2grep% -nau --newline=anycrlf "^(abc|def)" %srcdir%\testdata\grepinputUN >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep

echo ---------------------------- Test UN2 ------------------------------>testtrygrep
%pcre2grep% -nauU --newline=anycrlf "^a" %srcdir%\testdata\grepinputBad8_Trail >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep

%printf% "\n" >>testtrygrep

%cf% %srcdir%\testdata\grepoutputUN testtrygrep %cfout%
Expand All @@ -996,16 +1022,22 @@ if %ERRORLEVEL% equ 0 (

echo --- Test 1 --->testtrygrep
%pcre2grep% "(T)(..(.))(?C'cmd|/c echo|Arg1: [$1] [$2] [$3]|Arg2: ^$|${1}^$| ($4) ($14) ($0)')()" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep
echo --- Test 2 --->>testtrygrep
%pcre2grep% "(T)(..(.))()()()()()()()(..)(?C'cmd|/c echo|Arg1: [$11] [${11}]')" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep
echo --- Test 3 --->>testtrygrep
%pcre2grep% "(T)(?C'|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep
echo --- Test 4 --->>testtrygrep
%pcre2grep% "(T)(?C'cscript|//nologo|printf.js|%%s\r\n|$0:$1$n')" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep
echo --- Test 5 --->>testtrygrep
%pcre2grep% "(T)(?C'|$1$n')(*F)" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep
echo --- Test 6 --->>testtrygrep
%pcre2grep% -m1 "(T)(?C'|$0:$1:$x{41}$o{101}$n')" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep

%pcre2grep% --help | %pcre2grep% -q "Non-fork callout scripts in patterns are supported"
if ^!ERRORLEVEL! equ 0 (
Expand All @@ -1024,8 +1056,10 @@ if %ERRORLEVEL% equ 0 (

echo --- Test 1 --->testtrygrep
%pcre2grep% -u "(T)(?C'|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep
echo --- Test 2 --->>testtrygrep
%pcre2grep% -u "(T)(?C'cscript|//nologo|printf.js|%%s\r\n|$0:$x{a6}$n')" %srcdir%\testdata\grepinputv >>testtrygrep
echo RC=^!ERRORLEVEL!>>testtrygrep

if ^!nonfork! equ 1 (
%cf% %srcdir%\testdata\grepoutputCNU testtrygrep %cfout%
Expand Down
2 changes: 1 addition & 1 deletion RunTest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ while [ $# -gt 0 ] ; do
bigstack|-bigstack) bigstack=yes;;
nojit|-nojit) nojit=yes;;
sim|-sim) shift; sim=$1;;
valgrind|-valgrind) valgrind="valgrind --tool=memcheck -q --smc-check=all-non-file";;
valgrind|-valgrind) valgrind="valgrind --tool=memcheck -q --smc-check=all-non-file --error-exitcode=70";;
valgrind-log|-valgrind-log) valgrind="valgrind --tool=memcheck --num-callers=30 --leak-check=no --error-limit=no --smc-check=all-non-file --log-file=report.%p ";;
~*)
if expr "$1" : '~[0-9][0-9]*$' >/dev/null; then
Expand Down
Loading
Loading