Support for keyword based syntax highlighting (much faster)

See the following issues on GitHub:
 - https://github.com/xolox/vim-easytags/issues/68
 - https://github.com/xolox/vim-easytags/pull/80

Please note that right now this 'feature' is not integrated with
the "accelerated Python syntax highlighting" feature, because I'm
considering ripping that out and replacing it with a *fast* Vim
script implementation (if I can build one :-).

3 files changed, 104 insertions, 31 deletions

diff --git a/README.md b/README.md
index ae6aedf..5b4640a 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,18 @@ By setting this option to true (1) you enable asynchronous tags file updates. Go
 
 Note that asynchronous updates on Windows currently require the installation of my [vim-shell] [shell] plug-in (for obscure technical reasons that I want to fix but don't know how yet).
 
+### The `g:easytags_syntax_keyword` option
+
+When you look into how the dynamic syntax highlighting in the vim-easytags plug-in works you'll realize that vim-easytags is really abusing Vim's syntax highlighting engine. This can cause Vim to slow down to a crawl, depending on how big your tags files are. To make things worse in Vim 7.4 a new regex engine was introduced which exacerbates the problem (the patterns generated by vim-easytags bring out the worst of the new regex engine).
+
+Since version 3.6 the vim-easytags plug-in tries to squeeze as much performance as possible out of Vim by using keyword highlighting where this is possible without sacrificing accuracy. If your Vim's syntax highlighting is still too slow you can add the following to your [vimrc script] [vimrc]:
+
+    let g:easytags_syntax_keyword = 'always'
+
+The default value of this option is 'auto' which means to use keyword highlighting where this is possible without sacrificing accuracy. By changing it to 'always' you're telling vim-easytags to sacrifice accuracy in order to gain performance. Try it out and see what works best for you.
+
+Please note that right now this 'feature' is not integrated with the "accelerated Python syntax highlighting" feature, because I'm considering ripping that out and replacing it with a *fast* Vim script implementation.
+
 ### The `g:easytags_languages` option
 
 Exuberant Ctags supports many languages and can be extended via regular expression patterns, but for some languages separate tools with ctags-compatible output exist (e.g. [jsctags] [jsctags] for Javascript). To use these, the executable and its arguments must be configured:
diff --git a/autoload/xolox/easytags.vim b/autoload/xolox/easytags.vim
index def6dbf..80abc49 100644
--- a/autoload/xolox/easytags.vim
+++ b/autoload/xolox/easytags.vim
@@ -1,9 +1,9 @@
 " Vim script
 " Author: Peter Odding <peter@peterodding.com>
-" Last Change: July 8, 2014
+" Last Change: July 9, 2014
 " URL: http://peterodding.com/code/vim/easytags/
 
-let g:xolox#easytags#version = '3.5'
+let g:xolox#easytags#version = '3.6'
 
 " Plug-in initialization. {{{1
 
@@ -312,18 +312,45 @@ function! xolox#easytags#highlight() " {{{2
           endif
           let matches = filter(copy(taglist), filter)
           if matches != []
-            " Convert matched tags to :syntax command and execute it.
-            let matches = xolox#misc#list#unique(map(matches, 'xolox#misc#escape#pattern(get(v:val, "name"))'))
-            let pattern = tagkind.pattern_prefix . '\%(' . join(matches, '\|') . '\)' . tagkind.pattern_suffix
-            let template = 'syntax match %s /%s/ containedin=ALLBUT,%s'
-            let command = printf(template, hlgroup_tagged, escape(pattern, '/'), xolox#easytags#syntax_groups_to_ignore())
-            call xolox#misc#msg#debug("easytags.vim %s: Executing command '%s'.", g:xolox#easytags#version, command)
-            try
-              execute command
-            catch /^Vim\%((\a\+)\)\=:E339/
-              let msg = "easytags.vim %s: Failed to highlight %i %s tags because pattern is too big! (%i KB)"
-              call xolox#misc#msg#warn(msg, g:xolox#easytags#version, len(matches), tagkind.hlgroup, len(pattern) / 1024)
-            endtry
+            " Convert matched tags to :syntax commands and execute them.
+            let use_keywords_when = xolox#misc#option#get('easytags_syntax_keyword', 'auto')
+            let tagkind_has_patterns = !(empty(tagkind.pattern_prefix) && empty(tagkind.pattern_suffix))
+            if use_keywords_when == 'always' || (use_keywords_when == 'auto' && !tagkind_has_patterns)
+              " Vim's ":syntax keyword" command doesn't use the regular
+              " expression engine and the resulting syntax highlighting is
+              " therefor much faster. Because of this we use the syntax
+              " keyword command when 1) we can do so without sacrificing
+              " accuracy or 2) the user explicitly chose to sacrifice
+              " accuracy in order to make the highlighting faster.
+              let keywords = []
+              for tag in matches
+                if s:is_keyword_compatible(tag)
+                  call add(keywords, tag.name)
+                endif
+              endfor
+              if !empty(keywords)
+                let template = 'syntax keyword %s %s containedin=ALLBUT,%s'
+                let command = printf(template, hlgroup_tagged, join(keywords), xolox#easytags#syntax_groups_to_ignore())
+                call xolox#misc#msg#debug("easytags.vim %s: Executing command '%s'.", g:xolox#easytags#version, command)
+                execute command
+                " Remove the tags that we just highlighted from the list of
+                " tags that still need to be highlighted.
+                call filter(matches, "!s:is_keyword_compatible(v:val)")
+              endif
+            endif
+            if !empty(matches)
+              let matches = xolox#misc#list#unique(map(matches, 'xolox#misc#escape#pattern(get(v:val, "name"))'))
+              let pattern = tagkind.pattern_prefix . '\%(' . join(matches, '\|') . '\)' . tagkind.pattern_suffix
+              let template = 'syntax match %s /%s/ containedin=ALLBUT,%s'
+              let command = printf(template, hlgroup_tagged, escape(pattern, '/'), xolox#easytags#syntax_groups_to_ignore())
+              call xolox#misc#msg#debug("easytags.vim %s: Executing command '%s'.", g:xolox#easytags#version, command)
+              try
+                execute command
+              catch /^Vim\%((\a\+)\)\=:E339/
+                let msg = "easytags.vim %s: Failed to highlight %i %s tags because pattern is too big! (%i KB)"
+                call xolox#misc#msg#warn(msg, g:xolox#easytags#version, len(matches), tagkind.hlgroup, len(pattern) / 1024)
+              endtry
+            endif
           endif
         endif
       endfor
@@ -345,6 +372,13 @@ function! xolox#easytags#highlight() " {{{2
   endtry
 endfunction
 
+function! s:is_keyword_compatible(tag)
+  let name = get(a:tag, 'name', '')
+  if !empty(name)
+    return name =~ '^\k\+$' && len(name) <= 80
+  endif
+endfunction
+
 " Public supporting functions (might be useful to others). {{{1
 
 function! xolox#easytags#get_tagsfile() " {{{2
@@ -391,7 +425,7 @@ function! xolox#easytags#syntax_groups_to_ignore() " {{{2
   " This happens when a group wildcard doesn't match *anything*. Why does Vim
   " always have to make everything so complicated? :-(
   let groups = ['.*String.*', '.*Comment.*']
-  for group_name in ['cIncluded', 'cCppOut2', 'cCppInElse2', 'cCppOutIf2']
+  for group_name in ['cIncluded', 'cCppOut2', 'cCppInElse2', 'cCppOutIf2', 'pythonDocTest', 'pythonDocTest2']
     if hlexists(group_name)
       call add(groups, group_name)
     endif
@@ -433,10 +467,10 @@ endfunction
 
 function! xolox#easytags#define_tagkind(object) " {{{2
   if !has_key(a:object, 'pattern_prefix')
-    let a:object.pattern_prefix = '\C\<'
+    let a:object.pattern_prefix = ''
   endif
   if !has_key(a:object, 'pattern_suffix')
-    let a:object.pattern_suffix = '\>'
+    let a:object.pattern_suffix = ''
   endif
   if !has_key(s:tagkinds, a:object.filetype)
     let s:tagkinds[a:object.filetype] = []
diff --git a/doc/easytags.txt b/doc/easytags.txt
index 24c3fe4..f1f9820 100644
--- a/doc/easytags.txt
+++ b/doc/easytags.txt
@@ -12,20 +12,21 @@ Contents ~
  4. Options                                                  |easytags-options|
   1. The |g:easytags_cmd| option
   2. The |g:easytags_async| option
-  3. The |g:easytags_languages| option
-  4. The |g:easytags_file| option
-  5. The |g:easytags_dynamic_files| option
-  6. The |g:easytags_by_filetype| option
-  7. The |g:easytags_events| option
-  8. The |g:easytags_always_enabled| option
-  9. The |g:easytags_on_cursorhold| option
-  10. The |g:easytags_updatetime_min| option
-  11. The |g:easytags_auto_update| option
-  12. The |g:easytags_auto_highlight| option
-  13. The |g:easytags_autorecurse| option
-  14. The |g:easytags_include_members| option
-  15. The |g:easytags_resolve_links| option
-  16. The |g:easytags_suppress_ctags_warning| option
+  3. The |g:easytags_syntax_keyword| option
+  4. The |g:easytags_languages| option
+  5. The |g:easytags_file| option
+  6. The |g:easytags_dynamic_files| option
+  7. The |g:easytags_by_filetype| option
+  8. The |g:easytags_events| option
+  9. The |g:easytags_always_enabled| option
+  10. The |g:easytags_on_cursorhold| option
+  11. The |g:easytags_updatetime_min| option
+  12. The |g:easytags_auto_update| option
+  13. The |g:easytags_auto_highlight| option
+  14. The |g:easytags_autorecurse| option
+  15. The |g:easytags_include_members| option
+  16. The |g:easytags_resolve_links| option
+  17. The |g:easytags_suppress_ctags_warning| option
  5. Customizing the easytags plug-in             |customizing-easytags-plug-in|
   1. Passing custom command line arguments to Exuberant Ctags |easytags-passing-custom-command-line-arguments-to-exuberant-ctags|
   2. Update & highlight tags immediately after save |easytags-update-highlight-tags-immediately-after-save|
@@ -204,6 +205,32 @@ my vim-shell [12] plug-in (for obscure technical reasons that I want to fix but
 don't know how yet).
 
 -------------------------------------------------------------------------------
+The *g:easytags_syntax_keyword* option
+
+When you look into how the dynamic syntax highlighting in the vim-easytags
+plug-in works you'll realize that vim-easytags is really abusing Vim's syntax
+highlighting engine. This can cause Vim to slow down to a crawl, depending on
+how big your tags files are. To make things worse in Vim 7.4 a new regex engine
+was introduced which exacerbates the problem (the patterns generated by vim-
+easytags bring out the worst of the new regex engine).
+
+Since version 3.6 the vim-easytags plug-in tries to squeeze as much performance
+as possible out of Vim by using keyword highlighting where this is possible
+without sacrificing accuracy. If your Vim's syntax highlighting is still too
+slow you can add the following to your |vimrc| script:
+>
+  let g:easytags_syntax_keyword = 'always'
+<
+The default value of this option is 'auto' which means to use keyword
+highlighting where this is possible without sacrificing accuracy. By changing
+it to 'always' you're telling vim-easytags to sacrifice accuracy in order to
+gain performance. Try it out and see what works best for you.
+
+Please note that right now this 'feature' is not integrated with the
+"accelerated Python syntax highlighting" feature, because I'm considering
+ripping that out and replacing it with a _fast_ Vim script implementation.
+
+-------------------------------------------------------------------------------
 The *g:easytags_languages* option
 
 Exuberant Ctags supports many languages and can be extended via regular