awk: fix handling of literal backslashes in replacement

Follow actual gawk behavior.
    https://www.gnu.org/software/gawk/manual/html_node/Gory-Details.html
Erm... wait. Follow *actual* gawk behavior - as seen in testing,
because even Gory-Details.html do not tell the whole awful truth.

function                                             old     new   delta
awk_sub                                              550     658    +108

Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>
This commit is contained in:
Denys Vlasenko committed 2026-10-11 23:15:35 +02:00
1 parent 538499b65a
commit f0e5be9fbc
2 files changed
+174 -16

No files matched your search

+57 -16
View File
@@ -2758,9 +2758,8 @@ static char *awk_printf(node *n, size_t *len)
* store result into (dest), return number of substitutions.
* If nm = 0, replace all matches.
* If src or dst is NULL, use $0.
* If subexp != 0, enable subexpression matching (\0-\9).
*/
static int awk_sub(node *rn, const char *repl, int nm, var *src, var *dest /*,int subexp*/)
static int awk_sub(node *rn, const char *repl, int nm, var *src, var *dest /*,int in_gensub*/)
{
char *resbuf;
const char *sp;
@@ -2769,7 +2768,7 @@ static int awk_sub(node *rn, const char *repl, int nm, var *src, var *dest /*,in
regmatch_t pmatch[10];
regex_t sreg, *regex;
/* True only if called to implement gensub(): */
int subexp = (src != dest);
int in_gensub = (src != dest);
#if defined(REG_STARTEND)
const char *src_string;
size_t src_strlen;
@@ -2817,32 +2816,74 @@ static int awk_sub(node *rn, const char *repl, int nm, var *src, var *dest /*,in
residx += eo;
if (++match_no >= nm) {
const char *s;
int bslash;
/* replace */
residx -= (eo - so);
bslash = 0;
for (s = repl; *s; s++) {
char c = *s;
if (c == '\\' && s[1]) {
bslash ^= 1;
if (bslash)
continue;
s = repl;
while (*s) {
// https://www.gnu.org/software/gawk/manual/html_node/Gory-Details.html
// Table 9.2: gawk rules for sub() and backslash:
// You type sub() sees sub() generates
// \\q \q A literal ‘\q’ <<-- THIS IS DIFFERENT
// & & The matched text
// \\& \& A literal ‘&’
// \\\\ \\ Two literal ‘\’s <<-- THIS IS DIFFERENT
// \\\\& \\& A literal ‘\’, then the matched text
// \\\\\\& \\\& A literal ‘\&’
// \\\\\\q \\\q Two literal ‘\’s and literal ‘\q’ <<-- THIS IS DIFFERENT
// (^^^ non-POSIX special-casing of two backslashes by gawk)
// Table 9.4: Escape sequence processing for gensub():
// You type gensub() sees gensub() generates
// \\q \q A literal ‘q’ <<-- THIS IS DIFFERENT
// & & The matched text
// \\& \& A literal ‘&’
// \\\\ \\ A literal ‘\’ <<-- THIS IS DIFFERENT
// \\\\& \\& A literal ‘\’, then the matched text
// \\\\\\& \\\& A literal ‘\&’
// \\\\\\q \\\q A literal ‘\’ and literal ‘q’ <<-- THIS IS DIFFERENT
char c;
unsigned bslash = 0;
while ((c = *s++) == '\\' && *s) {
bslash++;
}
if ((!bslash && c == '&')
|| (subexp && bslash && c >= '0' && c <= '9')
if ((!(bslash & 1) && c == '&')
|| (in_gensub && (bslash & 1) && c >= '0' && c <= '9')
) {
int n, j = 0;
if (c != '&') {
j = c - '0';
}
bslash /= 2;
while (bslash) {
resbuf[residx++] = '\\';
bslash--;
}
n = pmatch[j].rm_eo - pmatch[j].rm_so;
resbuf = qrealloc(resbuf, residx + replen + n, &resbufsize);
memcpy(resbuf + residx, sp + pmatch[j].rm_so - start_ofs, n);
residx += n;
} else
resbuf[residx++] = c;
bslash = 0;
continue;
}
// We saw N backslashes, and we are at a "normal char", not special one.
// The "normal char" can be a "&" or backslash, examples:
// "\\\\\\&..." (in awk program) -> (what we process here) "\\\&...": escaped "&" become "normal".
// "\\\\\\" (in awk program) -> (what we process here) "\\\": last unpaired backslash is "normal".
//bb_error_msg("in_gensub:%d bslash:%d c:'%c'",in_gensub,bslash,c);
if (!in_gensub) {
// for non-gensub:
// special-case "two backslashes" case, including "\\\q" case which is logically "\\"+"\q"
// if (c != '&' && (bslash == 2 || bslash == 3)) // what doc claims
if (c != '&' && (bslash & 2)) // the actual logic in gawk-5.3.0 special-cases NOT ONLY 2,3 - but every two out of four!
bslash += 2;
if (c != '&') // gensub removes backslash from \q, sub does not:
bslash++; // if odd no.of backslashes, print one more, if even: no change
}
bslash /= 2;
while (bslash) {
resbuf[residx++] = '\\';
bslash--;
}
resbuf[residx++] = c;
}
}
+117
View File
@@ -711,4 +711,121 @@ test x"$SKIP_KNOWN_BUG_AWK_WORDMATCH" = x"" \
'abc\n' \
'' ''
# gawk 5.3.0 warns:
# awk: cmd. line:1: warning: escape sequence `\z' treated as plain `z'
testing 'awk sub backslash + z in replacement' \
'awk '$sq'sub(/b/, "_\z_")'$sq \
'a_z_c\n' \
'' 'abc\n'
# no warning
testing 'awk sub backslash*2 + z in replacement' \
'awk '$sq'sub(/b/, "_\\z_")'$sq \
'a_\\z_c\n' \
'' 'abc\n'
# awk: cmd. line:1: warning: escape sequence `\z' treated as plain `z'
testing 'awk sub backslash*3 + z in replacement' \
'awk '$sq'sub(/b/, "_\\\z_")'$sq \
'a_\\z_c\n' \
'' 'abc\n'
# (skipping odd numbers from now on) no warning
testing 'awk sub backslash*4 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\z_")'$sq 'a_\\\\z_c\n' '' 'abc\n' # +1
testing 'awk sub backslash*6 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\z_")'$sq 'a_\\\\\\z_c\n' '' 'abc\n' # +1
testing 'awk sub backslash*8 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\z_")'$sq 'a_\\\\z_c\n' '' 'abc\n'
testing 'awk sub backslash*10 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\z_")'$sq 'a_\\\\\\z_c\n' '' 'abc\n'
testing 'awk sub backslash*12 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\z_c\n' '' 'abc\n' # +1
testing 'awk sub backslash*14 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\\\z_c\n' '' 'abc\n' # +1
testing 'awk sub backslash*16 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\z_c\n' '' 'abc\n'
testing 'awk sub backslash*18 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\\\z_c\n' '' 'abc\n'
testing 'awk sub backslash*20 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\\\\\z_c\n' '' 'abc\n' # +1
testing 'awk sub backslash*22 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\\\\\\\z_c\n' '' 'abc\n' # +1
testing 'awk sub backslash*24 + z in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\\\\\\\\\\\\\z_")'$sq 'a_\\\\\\\\\\\\z_c\n' '' 'abc\n'
# Keep in mind that results as seen above are shell-escaped (doubled \\).
# Looks crazy isn't it?
# See https://www.gnu.org/software/gawk/manual/html_node/Gory-Details.html
# Allegedly, only "two bslashes" case is special-cased, but actual (YES, REALLY, I TESTED IT) gawk-5.3.0 behavior is:
# if number of bslashes is (2 mod 4) or (3 mod 4) [!!!], emit extra bslash (marked as +1 above).
# This causes absurd "sawtooth" pattern in the results.
# If you remove extra bslash (extra \\ due to shell-escaping) in +1 columns, it looks much more sane.
# awk: cmd. line:1: warning: escape sequence `\z' treated as plain `z'
testing 'awk gensub backslash + z in replacement' \
'awk '$sq'{ print gensub(/b/, "_\z_", 1) }'$sq \
'a_z_c\n' \
'' 'abc\n'
# no warning
testing 'awk gensub backslash*2 + z in replacement' \
'awk '$sq'{ print gensub(/b/, "_\\z_", 1) }'$sq \
'a_z_c\n' \
'' 'abc\n'
# awk: cmd. line:1: warning: escape sequence `\z' treated as plain `z'
testing 'awk gensub backslash*3 + z in replacement' \
'awk '$sq'{ print gensub(/b/, "_\\\z_", 1) }'$sq \
'a_z_c\n' \
'' 'abc\n'
# no warning
testing 'awk gensub backslash*4 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\z_", 1) }'$sq 'a_\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*6 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\z_", 1) }'$sq 'a_\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*8 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\z_", 1) }'$sq 'a_\\\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*10 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\z_", 1) }'$sq 'a_\\\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*12 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\\\z_", 1) }'$sq 'a_\\\\\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*14 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\\\\\z_", 1) }'$sq 'a_\\\\\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*16 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\\\\\\\z_", 1) }'$sq 'a_\\\\\\\\z_c\n' '' 'abc\n'
testing 'awk gensub backslash*18 + z in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\\\\\\\\\z_", 1) }'$sq 'a_\\\\\\\\z_c\n' '' 'abc\n'
# In comparison to the above, this looks sane
# no warning
testing 'awk sub & in replacement' \
'awk '$sq'sub(/b/, "_&_")'$sq \
'a_b_c\n' \
'' 'abc\n'
# awk: cmd. line:1: warning: escape sequence `\&' treated as plain `&'
# (HOW IS THIS A "PLAIN" &, GNU AWK??? IT *WAS* SUBST'ED!)
testing 'awk sub backslash + & in replacement' \
'awk '$sq'sub(/b/, "_\&_")'$sq \
'a_b_c\n' \
'' 'abc\n'
# no warning
testing 'awk sub backslash*2 + & in replacement' \
'awk '$sq'sub(/b/, "_\\&_")'$sq \
'a_&_c\n' \
'' 'abc\n'
# awk: cmd. line:1: warning: escape sequence `\&' treated as plain `&'
testing 'awk sub backslash*3 + & in replacement' \
'awk '$sq'sub(/b/, "_\\\&_")'$sq \
'a_&_c\n' \
'' 'abc\n'
# no warning
testing 'awk sub backslash*4 + & in replacement' 'awk '$sq'sub(/b/, "_\\\\&_")'$sq 'a_\\b_c\n' '' 'abc\n'
testing 'awk sub backslash*6 + & in replacement' 'awk '$sq'sub(/b/, "_\\\\\\&_")'$sq 'a_\\&_c\n' '' 'abc\n'
testing 'awk sub backslash*8 + & in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\&_")'$sq 'a_\\\\b_c\n' '' 'abc\n'
testing 'awk sub backslash*10 + & in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\&_")'$sq 'a_\\\\&_c\n' '' 'abc\n'
testing 'awk sub backslash*12 + & in replacement' 'awk '$sq'sub(/b/, "_\\\\\\\\\\\\&_")'$sq 'a_\\\\\\b_c\n' '' 'abc\n'
# no warning
testing 'awk gensub & in replacement' \
'awk '$sq'{ print gensub(/b/, "_&_", 1) }'$sq \
'a_b_c\n' \
'' 'abc\n'
# awk: cmd. line:1: warning: escape sequence `\&' treated as plain `&'
testing 'awk gensub backslash + & in replacement' \
'awk '$sq'{ print gensub(/b/, "_\&_", 1) }'$sq \
'a_b_c\n' \
'' 'abc\n'
# no warning
testing 'awk gensub backslash*2 + & in replacement' \
'awk '$sq'{ print gensub(/b/, "_\\&_", 1) }'$sq \
'a_&_c\n' \
'' 'abc\n'
# awk: cmd. line:1: warning: escape sequence `\&' treated as plain `&'
testing 'awk gensub backslash*3 + & in replacement' \
'awk '$sq'{ print gensub(/b/, "_\\\&_", 1) }'$sq \
'a_&_c\n' \
'' 'abc\n'
# no warning
testing 'awk gensub backslash*4 + & in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\&_", 1) }'$sq 'a_\\b_c\n' '' 'abc\n'
testing 'awk gensub backslash*6 + & in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\&_", 1) }'$sq 'a_\\&_c\n' '' 'abc\n'
testing 'awk gensub backslash*8 + & in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\&_", 1) }'$sq 'a_\\\\b_c\n' '' 'abc\n'
testing 'awk gensub backslash*10 + & in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\&_", 1) }'$sq 'a_\\\\&_c\n' '' 'abc\n'
testing 'awk gensub backslash*12 + & in replacement' 'awk '$sq'{ print gensub(/b/, "_\\\\\\\\\\\\&_", 1) }'$sq 'a_\\\\\\b_c\n' '' 'abc\n'
exit $FAILCOUNT