Merge branch 'es/chainlint'

The chainlint test script linter in the test suite has been updated.

* es/chainlint:
  chainlint.sed: stop splitting "(..." into separate lines "(" and "..."
  chainlint.sed: swallow comments consistently
  chainlint.sed: stop throwing away here-doc tags
  chainlint.sed: don't mistake `<< word` in string as here-doc operator
  chainlint.sed: make here-doc "<<-" operator recognition more POSIX-like
  chainlint.sed: drop subshell-closing ">" annotation
  chainlint.sed: drop unnecessary distinction between ?!AMP?! and ?!SEMI?!
  chainlint.sed: tolerate harmless ";" at end of last line in block
  chainlint.sed: improve ?!SEMI?! placement accuracy
  chainlint.sed: improve ?!AMP?! placement accuracy
  t/Makefile: optimize chainlint self-test
  t/chainlint/one-liner: avoid overly intimate chainlint.sed knowledge
  t/chainlint/*.test: generalize self-test commentary
  t/chainlint/*.test: fix invalid test cases due to mixing quote types
  t/chainlint/*.test: don't use invalid shell syntax

Author: gitster

t/Makefile

@@ -71,12 +71,10 @@ clean-chainlint:
 
 check-chainlint:
 	@mkdir -p '$(CHAINLINTTMP_SQ)' && \
-	err=0 && \
-	for i in $(CHAINLINTTESTS); do \
-		$(CHAINLINT) <chainlint/$$i.test | \
-		sed -e '/^# LINT: /d' >'$(CHAINLINTTMP_SQ)'/$$i.actual && \
-		diff -u chainlint/$$i.expect '$(CHAINLINTTMP_SQ)'/$$i.actual || err=1; \
-	done && exit $$err
+	sed -e '/^# LINT: /d' $(patsubst %,chainlint/%.test,$(CHAINLINTTESTS)) >'$(CHAINLINTTMP_SQ)'/tests && \
+	sed -e '/^[ 	]*$$/d' $(patsubst %,chainlint/%.expect,$(CHAINLINTTESTS)) >'$(CHAINLINTTMP_SQ)'/expect && \
+	$(CHAINLINT) '$(CHAINLINTTMP_SQ)'/tests | grep -v '^[	]*$$' >'$(CHAINLINTTMP_SQ)'/actual && \
+	diff -u '$(CHAINLINTTMP_SQ)'/expect '$(CHAINLINTTMP_SQ)'/actual
 
 test-lint: test-lint-duplicates test-lint-executable test-lint-shell-syntax \
 	test-lint-filenames

t/chainlint.sed

@@ -24,9 +24,9 @@
 # in order to avoid misinterpreting the ")" in constructs such as "x=$(...)"
 # and "case $x in *)" as ending the subshell.
 #
-# Lines missing a final "&&" are flagged with "?!AMP?!", and lines which chain
-# commands with ";" internally rather than "&&" are flagged "?!SEMI?!". A line
-# may be flagged for both violations.
+# Lines missing a final "&&" are flagged with "?!AMP?!", as are lines which
+# chain commands with ";" internally rather than "&&". A line may be flagged
+# for both violations.
 #
 # Detection of a missing &&-link in a multi-line subshell is complicated by the
 # fact that the last statement before the closing ")" must not end with "&&".
@@ -47,8 +47,8 @@
 # "?!AMP?!" violation is removed from the "bar" line (retrieved from the "hold"
 # area) since the final statement of a subshell must not end with "&&". The
 # final line of a subshell may still break the &&-chain by using ";" internally
-# to chain commands together rather than "&&", so "?!SEMI?!" is never removed
-# from a line (even though "?!AMP?!" might be).
+# to chain commands together rather than "&&", but an internal "?!AMP?!" is
+# never removed from a line even though a line-ending "?!AMP?!" might be.
 #
 # Care is taken to recognize the last _statement_ of a multi-line subshell, not
 # necessarily the last textual _line_ within the subshell, since &&-chaining
@@ -62,26 +62,20 @@
 # receives similar treatment.
 #
 # Swallowing here-docs with arbitrary tags requires a bit of finesse. When a
-# line such as "cat <<EOF >out" is seen, the here-doc tag is moved to the front
-# of the line enclosed in angle brackets as a sentinel, giving "<EOF>cat >out".
+# line such as "cat <<EOF" is seen, the here-doc tag is copied to the front of
+# the line enclosed in angle brackets as a sentinel, giving "<EOF>cat <<EOF".
 # As each subsequent line is read, it is appended to the target line and a
 # (whitespace-loose) back-reference match /^<(.*)>\n\1$/ is attempted to see if
 # the content inside "<...>" matches the entirety of the newly-read line. For
 # instance, if the next line read is "some data", when concatenated with the
-# target line, it becomes "<EOF>cat >out\nsome data", and a match is attempted
+# target line, it becomes "<EOF>cat <<EOF\nsome data", and a match is attempted
 # to see if "EOF" matches "some data". Since it doesn't, the next line is
 # attempted. When a line consisting of only "EOF" (and possible whitespace) is
-# encountered, it is appended to the target line giving "<EOF>cat >out\nEOF",
+# encountered, it is appended to the target line giving "<EOF>cat <<EOF\nEOF",
 # in which case the "EOF" inside "<...>" does match the text following the
 # newline, thus the closing here-doc tag has been found. The closing tag line
 # and the "<...>" prefix on the target line are then discarded, leaving just
-# the target line "cat >out".
-#
-# To facilitate regression testing (and manual debugging), a ">" annotation is
-# applied to the line containing ")" which closes a subshell, ">>" to a line
-# closing a nested subshell, and ">>>" to a line closing both at once. This
-# makes it easy to detect whether the heuristics correctly identify
-# end-of-subshell.
+# the target line "cat <<EOF".
 #------------------------------------------------------------------------------
 
 # incomplete line -- slurp up next line
@@ -94,9 +88,9 @@
 
 # here-doc -- swallow it to avoid false hits within its body (but keep the
 # command to which it was attached)
-/<<[ 	]*[-\\'"]*[A-Za-z0-9_]/ {
-	s/^\(.*\)<<[ 	]*[-\\'"]*\([A-Za-z0-9_][A-Za-z0-9_]*\)['"]*/<\2>\1<</
-	s/[ 	]*<<//
+/<<-*[ 	]*[\\'"]*[A-Za-z0-9_]/ {
+	/"[^"]*<<[^"]*"/bnotdoc
+	s/^\(.*<<-*[ 	]*\)[\\'"]*\([A-Za-z0-9_][A-Za-z0-9_]*\)['"]*/<\2>\1\2/
 	:hered
 	N
 	/^<\([^>]*\)>.*\n[ 	]*\1[ 	]*$/!{
@@ -106,6 +100,7 @@
 	s/^<[^>]*>//
 	s/\n.*$//
 }
+:notdoc
 
 # one-liner "(...) &&"
 /^[ 	]*!*[ 	]*(..*)[ 	]*&&[ 	]*$/boneline
@@ -126,7 +121,7 @@ b
 # "&&" (but not ";" in a string)
 :oneline
 /;/{
-	/"[^"]*;[^"]*"/!s/^/?!SEMI?!/
+	/"[^"]*;[^"]*"/!s/;/; ?!AMP?!/
 }
 b
 
@@ -136,11 +131,15 @@ b
 	h
 	bnextln
 }
-# "(..." line -- split off and stash "(", then process "..." as its own line
+# "(..." line -- "(" opening subshell cuddled with command; temporarily replace
+# "(" with sentinel "^" and process the line as if "(" had been seen solo on
+# the preceding line; this temporary replacement prevents several rules from
+# accidentally thinking "(" introduces a nested subshell; "^" is changed back
+# to "(" at output time
 x
-s/.*/(/
+s/.*//
 x
-s/(//
+s/(/^/
 bslurp
 
 :nextln
@@ -157,8 +156,10 @@ s/.*\n//
 	/"[^'"]*'[^'"]*"/!bsqstr
 }
 :folded
-# here-doc -- swallow it
-/<<[ 	]*[-\\'"]*[A-Za-z0-9_]/bheredoc
+# here-doc -- swallow it (but not "<<" in a string)
+/<<-*[ 	]*[\\'"]*[A-Za-z0-9_]/{
+	/"[^"]*<<[^"]*"/!bheredoc
+}
 # comment or empty line -- discard since final non-comment, non-empty line
 # before closing ")", "done", "elsif", "else", or "fi" will need to be
 # re-visited to drop "suspect" marking since final line of those constructs
@@ -171,12 +172,12 @@ s/.*\n//
 	/"[^"]*#[^"]*"/!s/[ 	]#.*$//
 }
 # one-liner "case ... esac"
-/^[ 	]*case[ 	]*..*esac/bchkchn
+/^[ 	^]*case[ 	]*..*esac/bchkchn
 # multi-line "case ... esac"
-/^[ 	]*case[ 	]..*[ 	]in/bcase
+/^[ 	^]*case[ 	]..*[ 	]in/bcase
 # multi-line "for ... done" or "while ... done"
-/^[ 	]*for[ 	]..*[ 	]in/bcont
-/^[ 	]*while[ 	]/bcont
+/^[ 	^]*for[ 	]..*[ 	]in/bcont
+/^[ 	^]*while[ 	]/bcont
 /^[ 	]*do[ 	]/bcont
 /^[ 	]*do[ 	]*$/bcont
 /;[ 	]*do/bcont
@@ -187,7 +188,7 @@ s/.*\n//
 /||[ 	]*exit[ 	]/bcont
 /||[ 	]*exit[ 	]*$/bcont
 # multi-line "if...elsif...else...fi"
-/^[ 	]*if[ 	]/bcont
+/^[ 	^]*if[ 	]/bcont
 /^[ 	]*then[ 	]/bcont
 /^[ 	]*then[ 	]*$/bcont
 /;[ 	]*then/bcont
@@ -200,15 +201,15 @@ s/.*\n//
 /^[ 	]*fi[ 	]*[<>|]/bdone
 /^[ 	]*fi[ 	]*)/bdone
 # nested one-liner "(...) &&"
-/^[ 	]*(.*)[ 	]*&&[ 	]*$/bchkchn
+/^[ 	^]*(.*)[ 	]*&&[ 	]*$/bchkchn
 # nested one-liner "(...)"
-/^[ 	]*(.*)[ 	]*$/bchkchn
+/^[ 	^]*(.*)[ 	]*$/bchkchn
 # nested one-liner "(...) >x" (or "2>x" or "<x" or "|x")
-/^[ 	]*(.*)[ 	]*[0-9]*[<>|]/bchkchn
+/^[ 	^]*(.*)[ 	]*[0-9]*[<>|]/bchkchn
 # nested multi-line "(...\n...)"
-/^[ 	]*(/bnest
+/^[ 	^]*(/bnest
 # multi-line "{...\n...}"
-/^[ 	]*{/bblock
+/^[ 	^]*{/bblock
 # closing ")" on own line -- exit subshell
 /^[ 	]*)/bclssolo
 # "$((...))" -- arithmetic expansion; not closing ")"
@@ -230,16 +231,18 @@ s/.*\n//
 # string and not ";;" in one-liner "case...esac")
 /;/{
 	/;;/!{
-		/"[^"]*;[^"]*"/!s/^/?!SEMI?!/
+		/"[^"]*;[^"]*"/!s/;/; ?!AMP?!/
 	}
 }
 # line ends with pipe "...|" -- valid; not missing "&&"
 /|[ 	]*$/bcont
 # missing end-of-line "&&" -- mark suspect
-/&&[ 	]*$/!s/^/?!AMP?!/
+/&&[ 	]*$/!s/$/ ?!AMP?!/
 :cont
 # retrieve and print previous line
 x
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 n
 bslurp
 
@@ -280,8 +283,7 @@ bfolded
 # found here-doc -- swallow it to avoid false hits within its body (but keep
 # the command to which it was attached)
 :heredoc
-s/^\(.*\)<<[ 	]*[-\\'"]*\([A-Za-z0-9_][A-Za-z0-9_]*\)['"]*/<\2>\1<</
-s/[ 	]*<<//
+s/^\(.*\)<<\(-*[ 	]*\)[\\'"]*\([A-Za-z0-9_][A-Za-z0-9_]*\)['"]*/<\3>\1?!HERE?!\2\3/
 :hdocsub
 N
 /^<\([^>]*\)>.*\n[ 	]*\1[ 	]*$/!{
@@ -295,23 +297,31 @@ bfolded
 # found "case ... in" -- pass through untouched
 :case
 x
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 n
+:cascom
+/^[ 	]*#/{
+	N
+	s/.*\n//
+	bcascom
+}
 /^[ 	]*esac/bslurp
 bcase
 
 # found "else" or "elif" -- drop "suspect" from final line before "else" since
 # that line legitimately lacks "&&"
 :else
 x
-s/?!AMP?!//
+s/\( ?!AMP?!\)* ?!AMP?!$//
 x
 bcont
 
 # found "done" closing for-loop or while-loop, or "fi" closing if-then -- drop
 # "suspect" from final contained line since that line legitimately lacks "&&"
 :done
 x
-s/?!AMP?!//
+s/\( ?!AMP?!\)* ?!AMP?!$//
 x
 # is 'done' or 'fi' cuddled with ")" to close subshell?
 /done.*)/bclose
@@ -322,11 +332,18 @@ bchkchn
 :nest
 x
 :nstslrp
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 n
+:nstcom
+# comment -- not closing ")" if in comment
+/^[ 	]*#/{
+	N
+	s/.*\n//
+	bnstcom
+}
 # closing ")" on own line -- stop nested slurp
 /^[ 	]*)/bnstcl
-# comment -- not closing ")" if in comment
-/^[ 	]*#/bnstcnt
 # "$((...))" -- arithmetic expansion; not closing ")"
 /\$(([^)][^)]*))[^)]*$/bnstcnt
 # "$(...)" -- command substitution; not closing ")"
@@ -337,15 +354,22 @@ n
 x
 bnstslrp
 :nstcl
-s/^/>>/
 # is it "))" which closes nested and parent subshells?
 /)[ 	]*)/bslurp
 bchkchn
 
 # found multi-line "{...\n...}" block -- pass through untouched
 :block
 x
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 n
+:blkcom
+/^[ 	]*#/{
+	N
+	s/.*\n//
+	bblkcom
+}
 # closing "}" -- stop block slurp
 /}/bchkchn
 bblock
@@ -354,16 +378,22 @@ bblock
 # since that line legitimately lacks "&&" and exit subshell loop
 :clssolo
 x
-s/?!AMP?!//
+s/\( ?!AMP?!\)* ?!AMP?!$//
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 p
 x
-s/^/>/
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 b
 
 # found closing "...)" -- exit subshell loop
 :close
 x
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 p
 x
-s/^/>/
+s/^\([ 	]*\)^/\1(/
+s/?!HERE?!/<</g
 b

t/chainlint/arithmetic-expansion.expect

@@ -2,8 +2,8 @@
 	foo &&
 	bar=$((42 + 1)) &&
 	baz
->) &&
+) &&
 (
-?!AMP?!	bar=$((42 + 1))
+	bar=$((42 + 1)) ?!AMP?!
 	baz
->)
+)

t/chainlint/bash-array.expect

@@ -2,9 +2,9 @@
 	foo &&
 	bar=(gumbo stumbo wumbo) &&
 	baz
->) &&
+) &&
 (
 	foo &&
 	bar=${#bar[@]} &&
 	baz
->)
+)

t/chainlint/blank-line.expect

@@ -1,4 +1,4 @@
 (
 	nothing &&
 	something
->)
+)

t/chainlint/blank-line.test

@@ -3,7 +3,7 @@
 	nothing &&
 
 	something
-# LINT: swallow blank lines since final _statement_ before subshell end is
+# LINT: ignore blank lines since final _statement_ before subshell end is
 # LINT: significant to "&&"-check, not final _line_ (which might be blank)
 
 

t/chainlint/block-comment.expect

@@ -0,0 +1,6 @@
+(
+	{
+		echo a &&
+		echo b
+	}
+)

t/chainlint/block-comment.test

@@ -0,0 +1,8 @@
+(
+	{
+		# show a
+		echo a &&
+		# show b
+		echo b
+	}
+)

t/chainlint/block.expect

@@ -7,6 +7,6 @@
 	bar &&
 	{
 		echo c
-?!AMP?!	}
+	} ?!AMP?!
 	baz
->)
+)

t/chainlint/block.test

@@ -1,6 +1,5 @@
 (
-# LINT: missing "&&" in block not currently detected (for consistency with
-# LINT: --chain-lint at top level and to provide escape hatch if needed)
+# LINT: missing "&&" after first "echo"
 	foo &&
 	{
 		echo a