299 lines
8.0 KiB
Ruby
299 lines
8.0 KiB
Ruby
module CodeRay
|
|
|
|
require 'coderay/helpers/plugin'
|
|
|
|
# = Scanners
|
|
#
|
|
# This module holds the Scanner class and its subclasses.
|
|
# For example, the Ruby scanner is named CodeRay::Scanners::Ruby
|
|
# can be found in coderay/scanners/ruby.
|
|
#
|
|
# Scanner also provides methods and constants for the register
|
|
# mechanism and the [] method that returns the Scanner class
|
|
# belonging to the given lang.
|
|
#
|
|
# See PluginHost.
|
|
module Scanners
|
|
extend PluginHost
|
|
plugin_path File.dirname(__FILE__), 'scanners'
|
|
|
|
require 'strscan'
|
|
|
|
# = Scanner
|
|
#
|
|
# The base class for all Scanners.
|
|
#
|
|
# It is a subclass of Ruby's great +StringScanner+, which
|
|
# makes it easy to access the scanning methods inside.
|
|
#
|
|
# It is also +Enumerable+, so you can use it like an Array of
|
|
# Tokens:
|
|
#
|
|
# require 'coderay'
|
|
#
|
|
# c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
|
|
#
|
|
# for text, kind in c_scanner
|
|
# puts text if kind == :operator
|
|
# end
|
|
#
|
|
# # prints: (*==)++;
|
|
#
|
|
# OK, this is a very simple example :)
|
|
# You can also use +map+, +any?+, +find+ and even +sort_by+,
|
|
# if you want.
|
|
class Scanner < StringScanner
|
|
|
|
extend Plugin
|
|
plugin_host Scanners
|
|
|
|
# Raised if a Scanner fails while scanning
|
|
ScanError = Class.new(Exception)
|
|
|
|
require 'coderay/helpers/word_list'
|
|
|
|
# The default options for all scanner classes.
|
|
#
|
|
# Define @default_options for subclasses.
|
|
DEFAULT_OPTIONS = { :stream => false }
|
|
|
|
KINDS_NOT_LOC = [:comment, :doctype]
|
|
|
|
class << self
|
|
|
|
# Returns if the Scanner can be used in streaming mode.
|
|
def streamable?
|
|
is_a? Streamable
|
|
end
|
|
|
|
def normify code
|
|
code = code.to_s
|
|
if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
|
|
code = code.dup
|
|
original_encoding = code.encoding
|
|
code.force_encoding 'Windows-1252'
|
|
unless code.valid_encoding?
|
|
code.force_encoding original_encoding
|
|
if code.encoding.name == 'UTF-8'
|
|
code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?'
|
|
end
|
|
code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?'
|
|
end
|
|
end
|
|
code.to_unix
|
|
end
|
|
|
|
def file_extension extension = nil
|
|
if extension
|
|
@file_extension = extension.to_s
|
|
else
|
|
@file_extension ||= plugin_id.to_s
|
|
end
|
|
end
|
|
|
|
end
|
|
|
|
=begin
|
|
## Excluded for speed reasons; protected seems to make methods slow.
|
|
|
|
# Save the StringScanner methods from being called.
|
|
# This would not be useful for highlighting.
|
|
strscan_public_methods =
|
|
StringScanner.instance_methods -
|
|
StringScanner.ancestors[1].instance_methods
|
|
protected(*strscan_public_methods)
|
|
=end
|
|
|
|
# Create a new Scanner.
|
|
#
|
|
# * +code+ is the input String and is handled by the superclass
|
|
# StringScanner.
|
|
# * +options+ is a Hash with Symbols as keys.
|
|
# It is merged with the default options of the class (you can
|
|
# overwrite default options here.)
|
|
# * +block+ is the callback for streamed highlighting.
|
|
#
|
|
# If you set :stream to +true+ in the options, the Scanner uses a
|
|
# TokenStream with the +block+ as callback to handle the tokens.
|
|
#
|
|
# Else, a Tokens object is used.
|
|
def initialize code='', options = {}, &block
|
|
raise "I am only the basic Scanner class. I can't scan "\
|
|
"anything. :( Use my subclasses." if self.class == Scanner
|
|
|
|
@options = self.class::DEFAULT_OPTIONS.merge options
|
|
|
|
super Scanner.normify(code)
|
|
|
|
@tokens = options[:tokens]
|
|
if @options[:stream]
|
|
warn "warning in CodeRay::Scanner.new: :stream is set, "\
|
|
"but no block was given" unless block_given?
|
|
raise NotStreamableError, self unless kind_of? Streamable
|
|
@tokens ||= TokenStream.new(&block)
|
|
else
|
|
warn "warning in CodeRay::Scanner.new: Block given, "\
|
|
"but :stream is #{@options[:stream]}" if block_given?
|
|
@tokens ||= Tokens.new
|
|
end
|
|
@tokens.scanner = self
|
|
|
|
setup
|
|
end
|
|
|
|
def reset
|
|
super
|
|
reset_instance
|
|
end
|
|
|
|
def string= code
|
|
code = Scanner.normify(code)
|
|
if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1']
|
|
reset_state
|
|
@string = code
|
|
else
|
|
super code
|
|
end
|
|
reset_instance
|
|
end
|
|
|
|
# More mnemonic accessor name for the input string.
|
|
alias code string
|
|
alias code= string=
|
|
|
|
# Returns the Plugin ID for this scanner.
|
|
def lang
|
|
self.class.plugin_id
|
|
end
|
|
|
|
# Scans the code and returns all tokens in a Tokens object.
|
|
def tokenize new_string=nil, options = {}
|
|
options = @options.merge(options)
|
|
self.string = new_string if new_string
|
|
@cached_tokens =
|
|
if @options[:stream] # :stream must have been set already
|
|
reset unless new_string
|
|
scan_tokens @tokens, options
|
|
@tokens
|
|
else
|
|
scan_tokens @tokens, options
|
|
end
|
|
end
|
|
|
|
def tokens
|
|
@cached_tokens ||= tokenize
|
|
end
|
|
|
|
# Whether the scanner is in streaming mode.
|
|
def streaming?
|
|
!!@options[:stream]
|
|
end
|
|
|
|
# Traverses the tokens.
|
|
def each &block
|
|
raise ArgumentError,
|
|
'Cannot traverse TokenStream.' if @options[:stream]
|
|
tokens.each(&block)
|
|
end
|
|
include Enumerable
|
|
|
|
# The current line position of the scanner.
|
|
#
|
|
# Beware, this is implemented inefficiently. It should be used
|
|
# for debugging only.
|
|
def line
|
|
string[0..pos].count("\n") + 1
|
|
end
|
|
|
|
def column pos = self.pos
|
|
return 0 if pos <= 0
|
|
string = string()
|
|
if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
|
|
@bin_string ||= string.dup.force_encoding('binary')
|
|
string = @bin_string
|
|
end
|
|
pos - (string.rindex(?\n, pos) || 0)
|
|
end
|
|
|
|
def marshal_dump
|
|
@options
|
|
end
|
|
|
|
def marshal_load options
|
|
@options = options
|
|
end
|
|
|
|
protected
|
|
|
|
# Can be implemented by subclasses to do some initialization
|
|
# that has to be done once per instance.
|
|
#
|
|
# Use reset for initialization that has to be done once per
|
|
# scan.
|
|
def setup
|
|
end
|
|
|
|
# This is the central method, and commonly the only one a
|
|
# subclass implements.
|
|
#
|
|
# Subclasses must implement this method; it must return +tokens+
|
|
# and must only use Tokens#<< for storing scanned tokens!
|
|
def scan_tokens tokens, options
|
|
raise NotImplementedError,
|
|
"#{self.class}#scan_tokens not implemented."
|
|
end
|
|
|
|
def reset_instance
|
|
@tokens.clear unless @options[:keep_tokens]
|
|
@cached_tokens = nil
|
|
@bin_string = nil if defined? @bin_string
|
|
end
|
|
|
|
# Scanner error with additional status information
|
|
def raise_inspect msg, tokens, state = 'No state given!', ambit = 30
|
|
raise ScanError, <<-EOE % [
|
|
|
|
|
|
***ERROR in %s: %s (after %d tokens)
|
|
|
|
tokens:
|
|
%s
|
|
|
|
current line: %d column: %d pos: %d
|
|
matched: %p state: %p
|
|
bol? = %p, eos? = %p
|
|
|
|
surrounding code:
|
|
%p ~~ %p
|
|
|
|
|
|
***ERROR***
|
|
|
|
EOE
|
|
File.basename(caller[0]),
|
|
msg,
|
|
tokens.size,
|
|
tokens.last(10).map { |t| t.inspect }.join("\n"),
|
|
line, column, pos,
|
|
matched, state, bol?, eos?,
|
|
string[pos - ambit, ambit],
|
|
string[pos, ambit],
|
|
]
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
end
|
|
|
|
class String
|
|
# I love this hack. It seems to silence all dos/unix/mac newline problems.
|
|
def to_unix
|
|
if index ?\r
|
|
gsub(/\r\n?/, "\n")
|
|
else
|
|
self
|
|
end
|
|
end
|
|
end
|