Add support for unicode whitespace characters

Author: ntpeters

README.md

@@ -1,6 +1,6 @@
 #Vim Better Whitespace Plugin
 
-This plugin causes all trailing whitespace characters (spaces and tabs) to be
+This plugin causes all trailing whitespace characters (see [Supported Whitespace Characters](#supported-whitespace-characters) below) to be
 highlighted. Whitespace for the current line will not be highlighted
 while in insert mode. It is possible to disable current line highlighting while in other
 modes as well (see options below). A helper function `:StripWhitespace` is also provided
@@ -100,7 +100,7 @@ Whitespace highlighting is enabled by default, with a highlight color of red.
     This replaces the filetypes from the default list of blacklisted filetypes. The
     default types that are blacklisted are:
     ```
-    ['diff', 'gitcommit', 'unite', 'qf', 'help']
+    ['diff', 'gitcommit', 'unite', 'qf', 'help', 'markdown']
     ```
     If you do not want any of these filetypes unignored, simply include them in the
     blacklist:
@@ -114,6 +114,43 @@ Whitespace highlighting is enabled by default, with a highlight color of red.
     let g:better_whitespace_verbosity=1
     ```
 
+##Supported Whitespace Characters
+Due to the fact that the built-in whitespace character class for patterns (`\s`)
+only matches against tabs and spaces, this plugin defines its own list of
+horizontal whitepsace characters to match for both highlighting and stripping.
+
+This is list should match against all ASCII and Unicode horizontal whitespace
+characters:
+```
+    U+0009   TAB
+    U+0020   SPACE
+    U+00A0   NO-BREAK SPACE
+    U+1680   OGHAM SPACE MARK
+    U+180E   MONGOLIAN VOWEL SEPARATOR
+    U+2000   EN QUAD
+    U+2001   EM QUAD
+    U+2002   EN SPACE
+    U+2003   EM SPACE
+    U+2004   THREE-PER-EM SPACE
+    U+2005   FOUR-PER-EM SPACE
+    U+2006   SIX-PER-EM SPACE
+    U+2007   FIGURE SPACE
+    U+2008   PUNCTUATION SPACE
+    U+2009   THIN SPACE
+    U+200A   HAIR SPACE
+    U+200B   ZERO WIDTH SPACE
+    U+202F   NARROW NO-BREAK SPACE
+    U+205F   MEDIUM MATHEMATICAL SPACE
+    U+3000   IDEOGRAPHIC SPACE
+    U+FEFF   ZERO WIDTH NO-BREAK SPACE
+```
+
+A file is provided with samples of each of these characters to check the plugin
+working with them: whitespace_examples.txt
+
+If you encounter any additional whitespace characters I have missed here,
+please submit a pull request.
+
 ##Screenshots
 Here are a couple more screenshots of the plugin at work.
 

plugin/better-whitespace.vim

@@ -39,6 +39,11 @@ call s:InitVariable('g:better_whitespace_filetypes_blacklist', default_blacklist
 " Disable verbosity by default
 call s:InitVariable('g:better_whitespace_verbosity', 0)
 
+" Define custom whitespace character group to include all horizontal unicode
+" whitespace characters. Vim's '\s' class only includes ASCII spaces and tabs.
+let s:whitespace_group='[\u0009\u0020\u00a0\u1680\u180e\u2000-\u200b\u202f\u205f\u3000\ufeff]'
+let s:eol_whitespace_pattern = s:whitespace_group . '\+$'
+
 " Only init once
 let s:better_whitespace_initialized = 0
 
@@ -117,7 +122,7 @@ function! s:CurrentLineWhitespaceOff( level )
         if a:level == 'hard'
             let g:current_line_whitespace_disabled_hard = 1
             let g:current_line_whitespace_disabled_soft = 0
-            call s:InAllWindows('syn clear ExtraWhitespace | match ExtraWhitespace /\s\+$/')
+            call s:InAllWindows('syn clear ExtraWhitespace | match ExtraWhitespace "' . s:eol_whitespace_pattern . '"')
             call <SID>Echo("Current Line Hightlight Off (hard)")
         elseif a:level == 'soft'
             let g:current_line_whitespace_disabled_soft = 1
@@ -136,7 +141,7 @@ function! s:CurrentLineWhitespaceOn()
         let g:current_line_whitespace_disabled_hard = 0
         let g:current_line_whitespace_disabled_soft = 0
         call <SID>SetupAutoCommands()
-        call s:InAllWindows('syn clear ExtraWhitespace | match ExtraWhitespace /\s\+$/')
+        call s:InAllWindows('syn clear ExtraWhitespace | match ExtraWhitespace "' . s:eol_whitespace_pattern . '"')
         call <SID>Echo("Current Line Hightlight On")
     endif
 endfunction
@@ -149,7 +154,7 @@ function! s:StripWhitespace( line1, line2 )
     let c = col(".")
 
     " Strip the whitespace
-    silent! execute ':' . a:line1 . ',' . a:line2 . 's/\s\+$//e'
+    silent! execute ':' . a:line1 . ',' . a:line2 . 's/' . s:eol_whitespace_pattern . '//e'
 
     " Restore the saved search and cursor position
     let @/=_s
@@ -212,7 +217,7 @@ autocmd ColorScheme * call <SID>WhitespaceInit()
 function! s:PerformMatchHighlight(pattern)
     call s:InitVariable('b:better_whitespace_enabled', !<SID>ShouldSkipHighlight())
     if b:better_whitespace_enabled == 1
-        exe 'match ExtraWhitespace ' . a:pattern
+        exe 'match ExtraWhitespace "' . a:pattern . '"'
     else
         match ExtraWhitespace ''
     endif
@@ -222,7 +227,24 @@ function! s:PerformSyntaxHighlight(pattern)
     syn clear ExtraWhitespace
     call s:InitVariable('b:better_whitespace_enabled', !<SID>ShouldSkipHighlight())
     if b:better_whitespace_enabled == 1
-        exe 'syn match ExtraWhitespace excludenl ' . a:pattern
+        exe 'syn match ExtraWhitespace excludenl "' . a:pattern . '"'
+    endif
+endfunction
+
+function! s:HighlightEOLWhitespace(type)
+    if (a:type == 'match')
+        call s:PerformMatchHighlight(s:eol_whitespace_pattern)
+    elseif (a:type == 'syntax')
+        call s:PerformSyntaxHighlight(s:eol_whitespace_pattern)
+    endif
+endfunction
+
+function! s:HighlightEOLWhitespaceExceptCurrentLine(type)
+    let a:exclude_current_line_eol_whitespace_pattern = '\%<' . line(".") .  'l' . s:eol_whitespace_pattern . '\|\%>' . line(".") .  'l' . s:eol_whitespace_pattern
+    if (a:type == 'match')
+        call s:PerformMatchHighlight(a:exclude_current_line_eol_whitespace_pattern)
+    elseif (a:type == 'syntax')
+        call s:PerformSyntaxHighlight(a:exclude_current_line_eol_whitespace_pattern)
     endif
 endfunction
 
@@ -237,28 +259,29 @@ function! <SID>SetupAutoCommands()
                 call <SID>WhitespaceInit()
             endif
 
+
             " Check if current line is disabled softly
             if g:current_line_whitespace_disabled_soft == 0
                 " Highlight all whitespace upon entering buffer
-                call <SID>PerformMatchHighlight('/\s\+$/')
+                call <SID>PerformMatchHighlight(s:eol_whitespace_pattern)
                 " Check if current line highglighting is disabled
                 if g:current_line_whitespace_disabled_hard == 1
                     " Never highlight whitespace on current line
-                    autocmd InsertEnter,CursorMoved,CursorMovedI * call <SID>PerformMatchHighlight('/\%<' . line(".") .  'l\s\+$\|\%>' . line(".") .  'l\s\+$/')
+                    autocmd InsertEnter,CursorMoved,CursorMovedI * call <SID>HighlightEOLWhitespaceExceptCurrentLine('match')
                 else
                     " When in insert mode, do not highlight whitespace on the current line
-                    autocmd InsertEnter,CursorMovedI * call <SID>PerformMatchHighlight('/\%<' . line(".") .  'l\s\+$\|\%>' . line(".") .  'l\s\+$/')
+                    autocmd InsertEnter,CursorMovedI * call <SID>HighlightEOLWhitespaceExceptCurrentLine('match')
                 endif
                 " Highlight all whitespace when exiting insert mode
-                autocmd InsertLeave,BufReadPost * call <SID>PerformMatchHighlight('/\s\+$/')
+                autocmd InsertLeave,BufReadPost * call <SID>HighlightEOLWhitespace('match')
                 " Clear whitespace highlighting when leaving buffer
                 autocmd BufWinLeave * match ExtraWhitespace ''
             else
                 " Highlight extraneous whitespace at the end of lines, but not the
                 " current line.
-                call <SID>PerformSyntaxHighlight('/\s\+$/')
-                autocmd InsertEnter * call <SID>PerformSyntaxHighlight('/\s\+\%#\@!$/')
-                autocmd InsertLeave,BufReadPost * call <SID>PerformSyntaxHighlight('/\s\+$/')
+                call <SID>HighlightEOLWhitespace('syntax')
+                autocmd InsertEnter * call <SID>HighlightEOLWhitespaceExceptCurrentLine('syntax')
+                autocmd InsertLeave,BufReadPost * call <SID>HighlightEOLWhitespace('syntax')
             endif
         endif
 

whitespace_examples.txt

@@ -0,0 +1,21 @@
+U+0009	TAB:	
+U+0020	SPACE: 
+U+00A0	NO-BREAK SPACE: 
+U+1680	OGHAM SPACE MARK: 
+U+180E	MONGOLIAN VOWEL SEPARATOR:᠎
+U+2000	EN QUAD: 
+U+2001	EM QUAD: 
+U+2002	EN SPACE: 
+U+2003	EM SPACE: 
+U+2004	THREE-PER-EM SPACE: 
+U+2005	FOUR-PER-EM SPACE: 
+U+2006	SIX-PER-EM SPACE: 
+U+2007	FIGURE SPACE: 
+U+2008	PUNCTUATION SPACE: 
+U+2009	THIN SPACE: 
+U+200A	HAIR SPACE: 
+U+200B	ZERO WIDTH SPACE:​
+U+202F	NARROW NO-BREAK SPACE: 
+U+205F	MEDIUM MATHEMATICAL SPACE: 
+U+3000	IDEOGRAPHIC SPACE:　
+U+FEFF	ZERO WIDTH NO-BREAK SPACE: