2x faster syntax highlighting using Python Interface to Vim :-)

3 files changed, 156 insertions, 0 deletions

diff --git a/misc/easytags/highlight.py b/misc/easytags/highlight.py
new file mode 100644
index 0000000..b7a9cf3
--- /dev/null
+++ b/misc/easytags/highlight.py
@@ -0,0 +1,55 @@
+'''
+This Python script is part of the easytags plug-in for the Vim text editor. The
+Python Interface to Vim is used to load this script which accelerates dynamic
+syntax highlighting by reimplementing tag file reading and :syntax command
+generation in Python with a focus on doing the least amount of work.
+
+Author: Peter Odding <peter@peterodding.com>
+Last Change: June 14, 2011
+URL: http://peterodding.com/code/vim/easytags
+'''
+
+# TODO Cache the contents of tags files to further improve performance?
+
+import re
+import vim
+import sys
+
+def easytags_ping():
+    print 'it works!'
+
+def easytags_gensyncmd(tagsfiles, filetype, tagkinds, syntaxgroup, prefix, suffix, filters):
+    # Get arguments from Vim.
+    if filters:
+        tagkinds = filters['kind']
+    # Shallow parse tags files for matching identifiers.
+    pattern = '^([^\t]+)\t[^\t]+\t[^\t]+\t' + tagkinds + '\tlanguage:' + filetype
+    compiled_pattern = re.compile(pattern, re.IGNORECASE)
+    matches = {}
+    for fname in tagsfiles:
+        handle = open(fname)
+        for line in handle:
+            m = compiled_pattern.match(line)
+            if m and ('match' not in filters or re.search(filters['match'], line)) \
+                    and ('nomatch' not in filters or not re.search(filters['nomatch'], line)):
+                matches[m.group(1)] = True
+        handle.close()
+    # Generate Vim :syntax command to highlight identifiers.
+    patterns, commands = [], []
+    counter, limit = 0, 1024 * 20
+    to_escape = re.compile(r'[.*^$/\\~\[\]]')
+    for ident in matches.keys():
+        escaped = to_escape.sub(r'\\\0', ident)
+        patterns.append(escaped)
+        counter += len(escaped)
+        if counter > limit:
+            commands.append(_easytags_makecmd(syntaxgroup, prefix, suffix, patterns))
+            patterns = []
+            counter = 0
+    if patterns:
+        commands.append(_easytags_makecmd(syntaxgroup, prefix, suffix, patterns))
+    return ' | '.join(commands)
+
+def _easytags_makecmd(syntaxgroup, prefix, suffix, patterns):
+    template = r'syntax match %s /%s\%%(%s\)%s/ containedin=ALLBUT,.*String.*,.*Comment.*,cIncluded'
+    return template % (syntaxgroup, prefix, r'\|'.join(patterns), suffix)
diff --git a/misc/easytags/normalize-tags.py b/misc/easytags/normalize-tags.py
new file mode 100755
index 0000000..6701c53
--- /dev/null
+++ b/misc/easytags/normalize-tags.py
@@ -0,0 +1,62 @@
+#!/usr/bin/python
+
+'''
+Resolve symbolic links in tags files and remove duplicate entries from the
+resulting list of tags. If your tags files contain symbolic links as well as
+canonical filenames this can significantly reduce the size of your tags file.
+This script makes a backup of the tags file in case something goes wrong.
+
+Author: Peter Odding <peter@peterodding.com>
+Last Change: May 11, 2011
+URL: https://github.com/xolox/vim-easytags/blob/master/normalize-tags.py
+'''
+
+import os, sys, time
+
+tagsfile = os.path.expanduser(len(sys.argv) > 1 and sys.argv[1] or '~/.vimtags')
+tempname = '%s-new-%d' % (tagsfile, time.time())
+results, cache = {}, {}
+infile = open(tagsfile)
+outfile = open(tempname, 'w')
+nprocessed = 0
+fold_case = False
+
+for line in infile:
+  nprocessed += 1
+  line = line.rstrip()
+  fields = line.split('\t')
+  if line.startswith('!_TAG_'):
+    results[line] = True
+    if line.startswith('!_TAG_FILE_SORTED\t2'):
+      fold_case = True
+  else:
+    pathname = fields[1]
+    if pathname not in cache:
+      if not os.path.exists(pathname): continue
+      cache[pathname] = os.path.realpath(pathname)
+    fields[1] = cache[pathname]
+    results['\t'.join(fields)] = True
+
+infile.close()
+
+lines = results.keys()
+if fold_case:
+  lines.sort(key=str.lower)
+else:
+  lines.sort()
+
+outfile.write('\n'.join(lines))
+outfile.write('\n')
+outfile.close()
+
+backup = '%s-backup-%d' % (tagsfile, time.time())
+print "Making a backup of %s as %s" % (tagsfile, backup)
+os.rename(tagsfile, backup)
+
+print "Replacing old", tagsfile, "with new one"
+os.rename(tempname, tagsfile)
+
+nfiltered = nprocessed - len(lines)
+print "Filtered %d out of %d entries" % (nfiltered, nprocessed)
+
+# vim: ts=2 sw=2 et
diff --git a/misc/easytags/why-so-slow.py b/misc/easytags/why-so-slow.py
new file mode 100755
index 0000000..ead62f2
--- /dev/null
+++ b/misc/easytags/why-so-slow.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python
+
+'''
+Determine which files are contributing the most to the size of a tags file. You
+can specify the location of the tags file as a command line argument. If you
+pass a numeric argument, no more than that many files will be reported.
+
+Author: Peter Odding <peter@peterodding.com>
+Last Change: May 11, 2011
+URL: https://github.com/xolox/vim-easytags/blob/master/why-so-slow.py
+'''
+
+import os, sys
+
+tagsfile = '~/.vimtags'
+topfiles = 10
+
+for arg in sys.argv[1:]:
+  if os.path.isfile(arg):
+    tagsfile = arg
+  else:
+    topfiles = int(arg)
+
+infile = open(os.path.expanduser(tagsfile))
+counters = {}
+
+for line in infile:
+  fields = line.split('\t')
+  filename = fields[1]
+  counters[filename] = counters.get(filename, 0) + len(line)
+infile.close()
+
+sortedfiles = sorted([(s, n) for (n, s) in counters.iteritems()], reverse=True)
+for filesize, filename in sortedfiles[:topfiles]:
+  if filename.startswith(os.environ['HOME']):
+    filename = filename.replace(os.environ['HOME'], '~')
+  print '%i KB - %s' % (filesize / 1024, filename)
+
+# vim: ts=2 sw=2 et