VimColor Module

The VimColor module enables you to use vim to markup text from within python. As a bonus it comes with a full-featured moin Parser class for direct use as a plugin.

The parser for moin 1.3 which uses your local vim installation to colorize sources. This is great, because vim knows about 500 different syntaxdefs!

Usage

either as #FORMAT PI or with a code display hash-bang {{{#!VimColor. /!\ Note that the use as formatter requires a lowercase plugin name. So just name it vimcolor.py and refer to it as vimcolor.

Arguments

type to specify the syntax to use. use this if vim failes to recognize the syntax on its own.

Examples

Sources

Version

-- OliverGraf 2004-10-25 06:08:21

   1 #!/usr/bin/python
   2 # -*- coding: iso-8859-1 -*-
   3 """
   4 
   5     Syntax Highlighter using Vim
   6 
   7     Converted from Text::VimColor perl module by
   8     Copyright 2002-2004, Geoff Richards.
   9 
  10     Copyright (c) 2004 by Oliver Graf <ograf@bitart.de>
  11 
  12     Version History:
  13       1.0 - first working version
  14       1.1 - ANSI output, test code
  15       1.2 - subclass css
  16 
  17 """
  18 
  19 ### Imports ###################################################################
  20 
  21 import re
  22 
  23 ### Config ####################################################################
  24 
  25 VIM_COMMAND = '/usr/bin/vim';
  26 VIM_OPTIONS = '-RXZ -i NONE -u NONE -N'
  27 
  28 SYNTAX = {
  29 	'': 'Text',
  30 	'Normal': 'Text',
  31 	'Comment':    'Comment',
  32 	'Constant':   'Constant',
  33 	'Identifier': 'Identifier',
  34 	'Statement':  'Statement',
  35 	'PreProc':    'PreProc',
  36 	'Type':       'Type',
  37 	'Special':    'Special',
  38 	'Underlined': 'Underlined',
  39 	'Error':      'Error',
  40 	'Todo':       'Todo',
  41 	}
  42 
  43 MOIN_SYNTAX = {
  44 	'':           'Char',
  45 	'Normal':     'Char',
  46 	'Comment':    'Comment',
  47 	'Constant':   'ConsWord',
  48 	'Identifier': 'ID',
  49 	'Statement':  'ResWord',
  50 	'PreProc':    'Preprc',
  51 	'Type':       'ResWord2',
  52 	'Special':    'Special',
  53 	'Underlined': 'SPChar',
  54 	'Error':      'Error',
  55 	'Todo':       'Special',
  56 	}
  57 
  58 ### Constants #################################################################
  59 
  60 MODE_RAW=0
  61 MODE_HTML=1
  62 MODE_ANSI=2
  63 #MODE_XML=3
  64 
  65 # just an example
  66 ANSI_SEQ = {'':           ('', ''),
  67 			'Normal':     ('', ''),
  68 			'Text':       ('', ''),
  69 			'Comment':    ('\x1b[34m', '\x1b[0m'),
  70 			'Constant':   ('\x1b[31m', '\x1b[0m'),
  71 			'Identifier': ('\x1b[36m', '\x1b[0m'),
  72 			'Statement':  ('\x1b[33m', '\x1b[0m'),
  73 			'PreProc':    ('\x1b[35m', '\x1b[0m'),
  74 			'Type':       ('', ''),
  75 			'Special':    ('', ''),
  76 			'Underlined': ('', ''),
  77 			'Error':      ('', ''),
  78 			'Todo':       ('', ''),
  79 			}
  80 
  81 ANSI_RESET = '\x1b[0m'
  82 
  83 VIM_MARK_RC='''" mark.vim - turn Vim syntax highlighting into an ad-hoc markup language that
  84 " can be parsed by the Text::VimColor Perl module.
  85 "
  86 " Maintainer: Geoff Richards <qef@laxan.com>
  87 " Based loosely on 2html.vim, by Bram Moolenaar <Bram@vim.org>,
  88 "   modified by David Ne\\v{c}as (Yeti) <yeti@physics.muni.cz>.
  89 
  90 set report=1000000
  91 
  92 " For some reason (I\'m sure it used to work) we now need to get Vim
  93 " to make another attempt to detect the filetype if it wasn\'t set
  94 " explicitly.
  95 if !strlen(&filetype)
  96    filetype detect
  97 endif
  98 syn on
  99 
 100 " Set up the output buffer.
 101 new
 102 set modifiable
 103 set paste
 104 
 105 " Expand tabs.: Without this they come out as \'^I\'.
 106 set isprint+=9
 107 
 108 wincmd p
 109 
 110 " Loop over all lines in the original text
 111 let s:end = line("$")
 112 let s:lnum = 1
 113 while s:lnum <= s:end
 114 
 115   " Get the current line
 116   let s:line = getline(s:lnum)
 117   let s:len = strlen(s:line)
 118   let s:new = ""
 119 
 120   " Loop over each character in the line
 121   let s:col = 1
 122   while s:col <= s:len
 123     let s:startcol = s:col " The start column for processing text
 124     let s:id = synID(s:lnum, s:col, 1)
 125     let s:col = s:col + 1
 126     " Speed loop (it\'s small - that\'s the trick)
 127     " Go along till we find a change in synID
 128     while s:col <= s:len && s:id == synID(s:lnum, s:col, 1) | let s:col = s:col + 1 | endwhile
 129 
 130     " Output the text with the same synID, with class set to c{s:id}
 131     let s:name = synIDattr(s:id, \'name\')
 132     let s:id = synIDtrans(s:id)
 133     let s:class = synIDattr(s:id, \'name\')
 134     let s:new = s:new . \'>\' . s:class . \'>\' . s:name . \'>\' . substitute(substitute(substitute(strpart(s:line, s:startcol - 1, s:col - s:startcol), \'&\', \'\\&a\', \'g\'), \'<\', \'\\&l\', \'g\'), \'>\', \'\\&g\', \'g\') . \'<\' . s:class . \'<\' . s:name . \'<\'
 135 
 136     if s:col > s:len
 137       break
 138     endif
 139   endwhile
 140 
 141   exe "normal \\<C-W>pa" . strtrans(s:new) . "\\n\\e\\<C-W>p"
 142   let s:lnum = s:lnum + 1
 143   +
 144 endwhile
 145 
 146 " Strip whitespace from the ends of lines
 147 %s:\\s\\+$::e
 148 
 149 wincmd p
 150 normal dd
 151 '''
 152 
 153 ### Globals ###################################################################
 154 
 155 SYNre=re.compile(r'>(?P<class>.*?)>(?P<name>.*?)>(.*?)<(?P=class)<(?P=name)<',re.S)
 156 
 157 ### Helper Functions ##########################################################
 158 
 159 def xmlquote(s):
 160 	return s.replace('&','&amp;').replace('>','&gt;').replace('<','&lt')
 161 
 162 ### Interface #################################################################
 163 
 164 class VimColor:
 165 	"""
 166     Vim Colorizer Class
 167     """
 168 
 169 	def __init__(self, mode=MODE_HTML, syntaxmap=None):
 170 		self.mode = mode
 171 		self.syntaxmap = {}
 172 		if syntaxmap is None:
 173 			self.syntaxmap.update(SYNTAX)
 174 		else:
 175 			self.syntaxmap.update(syntaxmap)
 176 
 177 	def _build_syntax(self, filepath, filetype=None):
 178 		import os, tempfile
 179 		self.syntax=[]
 180 		vstemp=tempfile.mktemp('.vim','mark')
 181 		open(vstemp,'w').write(VIM_MARK_RC)
 182 		if not os.path.exists(filepath):
 183 			raise IOError('File "%s" not found'%(filepath))
 184 		# tempfile to save
 185 		outtemp=tempfile.mktemp('.xml')
 186 		# make vim script
 187 		shtemp=tempfile.mktemp('.vim')
 188 		sh=open(shtemp,'w')
 189 		if filetype:
 190 			filetype=':set filetype=%s\n'%(filetype,)
 191 		else:
 192 			filetype=''
 193 		sh.write(''':filetype on
 194 %(filetype)s:source %(vstemp)s
 195 :write! %(outtemp)s
 196 :qall!
 197 '''%{'filetype': filetype,
 198 	 'vstemp': vstemp,
 199 	 'outtemp': outtemp})
 200 		sh.close()
 201 		# run colorize
 202 		os.system("%s %s %s -s %s >/dev/null 2>&1"%(VIM_COMMAND, VIM_OPTIONS, filepath, shtemp))
 203 		# load stuff and kill files
 204 		lines=open(outtemp,'r').readlines()
 205 		try:
 206 			os.unlink(shtemp)
 207 		except:
 208 			pass
 209 		try:
 210 			os.unlink(outtemp)
 211 		except:
 212 			pass
 213 		try:
 214 			os.unlink(vstemp)
 215 		except:
 216 			pass
 217 		# make syntax list
 218 		for line in lines:
 219 			self.syntax.append([('__NL__', '__NL__'), '\n'])
 220 			for cls, syn, text in SYNre.findall(line.rstrip()):
 221 				cls=self.syntaxmap.get(cls,'Unknown_'+cls)
 222 				text=text.replace('&l','<')
 223 				text=text.replace('&g','>')
 224 				text=text.replace('&a','&')
 225 				if len(self.syntax) and self.syntax[-1][0]==(cls, syn):
 226 					self.syntax[-1][1]+=text
 227 				else:
 228 					self.syntax.append([(cls, syn),text])
 229 
 230 	def _mark(self, filepath, filetype=None):
 231 		self._build_syntax(filepath,filetype)
 232 		if self.mode==MODE_RAW:
 233 			return self.syntax
 234 		elif self.mode==MODE_HTML:
 235 			res='<pre>'
 236 			for (cls, syn), text in self.syntax:
 237 				if cls=='__NL__':
 238 					res+='\n<span class="LineNumber"></span>'
 239 				else:
 240 					if syn!='':
 241 						syn=' '+syn
 242 					res+='<span class="%s%s">%s</span>'%(cls,syn,xmlquote(text))
 243 			res+='\n</pre>'
 244 			return res
 245 		elif self.mode==MODE_ANSI:
 246 			res=''
 247 			for (cls, syn), text in self.syntax:
 248 				if syn=='__NL__':
 249 					res+='\n'
 250 				else:
 251 					res+='%s%s%s'%(ANSI_SEQ.get(cls,('',''))[0],
 252 								   text,
 253 								   ANSI_SEQ.get(cls,('',''))[1])
 254 			res+='%s\n'%(ANSI_RESET,)
 255 			return res[1:-1] # strip first and last \n
 256 		raise ValueError("unknown mode %d"%(self.mode))
 257 		
 258 	def markFile(self, filepath, filetype=None):
 259 		return self._mark(filepath, filetype)
 260 
 261 	def markString(self, string, extension='.dat', filetype=None):
 262 		import tempfile
 263 		fn=tempfile.mktemp(extension)
 264 		f=open(fn,'w')
 265 		f.write(string)
 266 		f.close()
 267 		res=self._mark(fn, filetype)
 268 		try:
 269 			os.unlink(fn)
 270 		except:
 271 			pass
 272 		return res
 273 
 274 ### MoinMoin parser ###########################################################
 275 
 276 Dependencies = []
 277 
 278 class Parser:
 279 	"""
 280     MoinMoin Sytax Parser Class using VimColor
 281     """
 282 
 283 	parsername = "VimColor"
 284 	extensions = '*'
 285 
 286 	def __init__(self, raw, request, **kw):
 287 		""" Store the source text.
 288         """
 289 		from MoinMoin.util.ParserBase import parse_start_step
 290 
 291 		self.raw = raw.expandtabs().rstrip()
 292 		self.request = request
 293 		self.form = request.form
 294 		self._ = request.getText
 295 		self.show_num, self.num_start, self.num_step, args = parse_start_step(request, kw.get('format_args',''))
 296 		self.filetype = None
 297 		if args.has_key('type'):
 298 			# get rid of those quotes no quotes and only the first valid part
 299 			self.filetype = re.split(r'[^a-z0-9]', args['type'][1:-1])[0]
 300 
 301 	def format(self, formatter):
 302 		""" Parse and send the colored source.
 303 		"""
 304 		from MoinMoin import config
 305 		import sha
 306 		
 307 		self._code_id = sha.new(self.raw.encode(config.charset)).hexdigest()
 308 
 309 		self.request.write(formatter.code_area(1, self._code_id, 'VimColorizer', self.show_num, self.num_start, self.num_step))
 310 
 311 		vim=VimColor(MODE_RAW, MOIN_SYNTAX)
 312 		for (cls, syn), tok in vim.markString(self.raw,filetype=self.filetype):
 313 			if cls=='__NL__':
 314 				self.request.write(formatter.code_line(1))
 315 			else:
 316 				if syn!='':
 317 					cls=cls+' '+syn
 318 				self.request.write(formatter.code_token(1, cls) +
 319 								   formatter.text(tok)+
 320 								   formatter.code_token(0, cls))
 321 
 322 		self.request.write(formatter.code_area(0, self._code_id))
 323 
 324 ### MAIN Test #################################################################
 325 
 326 TEST="""#!/usr/bin/python
 327 
 328 # just a test
 329 import sys
 330 
 331 def main(world='World'):
 332     sys.stdout.write('Hello %s!' % (world,))
 333 
 334 if __name__=='__main__':
 335     main(sys.argv[1])
 336     sys.exit(0)
 337 
 338 """
 339 
 340 if __name__=='__main__':
 341 	import sys
 342 	if len(sys.argv)>1:
 343 		try:
 344 			data=open(sys.argv[1],'r').read()
 345 		except:
 346 			sys.stderr.write("can't read %s\n"%(sys.argv[1]))
 347 			sys.exit(1)
 348 	else:
 349 		data=TEST
 350 	vc=VimColor(MODE_ANSI)
 351 	print vc.markString(data)
 352 
 353 ###############################################################################
VimColor.py

Comments

MoinMoin: OliverGraf/VimColor (last edited 2007-10-29 19:06:35 by localhost)