286 lines
9.0 KiB
Ruby
286 lines
9.0 KiB
Ruby
|
module CodeRay
|
||
|
|
module Scanners
|
||
|
|
|
||
|
|
# Bases on pygments' PythonLexer, see
|
||
|
|
# http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
|
||
|
|
class Python < Scanner
|
||
|
|
|
||
|
|
include Streamable
|
||
|
|
|
||
|
|
register_for :python
|
||
|
|
file_extension 'py'
|
||
|
|
|
||
|
|
KEYWORDS = [
|
||
|
|
'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
|
||
|
|
'del', 'elif', 'else', 'except', 'finally', 'for',
|
||
|
|
'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
|
||
|
|
'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
|
||
|
|
'nonlocal', # new in Python 3
|
||
|
|
]
|
||
|
|
|
||
|
|
OLD_KEYWORDS = [
|
||
|
|
'exec', 'print', # gone in Python 3
|
||
|
|
]
|
||
|
|
|
||
|
|
PREDEFINED_METHODS_AND_TYPES = %w[
|
||
|
|
__import__ abs all any apply basestring bin bool buffer
|
||
|
|
bytearray bytes callable chr classmethod cmp coerce compile
|
||
|
|
complex delattr dict dir divmod enumerate eval execfile exit
|
||
|
|
file filter float frozenset getattr globals hasattr hash hex id
|
||
|
|
input int intern isinstance issubclass iter len list locals
|
||
|
|
long map max min next object oct open ord pow property range
|
||
|
|
raw_input reduce reload repr reversed round set setattr slice
|
||
|
|
sorted staticmethod str sum super tuple type unichr unicode
|
||
|
|
vars xrange zip
|
||
|
|
]
|
||
|
|
|
||
|
|
PREDEFINED_EXCEPTIONS = %w[
|
||
|
|
ArithmeticError AssertionError AttributeError
|
||
|
|
BaseException DeprecationWarning EOFError EnvironmentError
|
||
|
|
Exception FloatingPointError FutureWarning GeneratorExit IOError
|
||
|
|
ImportError ImportWarning IndentationError IndexError KeyError
|
||
|
|
KeyboardInterrupt LookupError MemoryError NameError
|
||
|
|
NotImplemented NotImplementedError OSError OverflowError
|
||
|
|
OverflowWarning PendingDeprecationWarning ReferenceError
|
||
|
|
RuntimeError RuntimeWarning StandardError StopIteration
|
||
|
|
SyntaxError SyntaxWarning SystemError SystemExit TabError
|
||
|
|
TypeError UnboundLocalError UnicodeDecodeError
|
||
|
|
UnicodeEncodeError UnicodeError UnicodeTranslateError
|
||
|
|
UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
|
||
|
|
]
|
||
|
|
|
||
|
|
PREDEFINED_VARIABLES_AND_CONSTANTS = [
|
||
|
|
'False', 'True', 'None', # "keywords" since Python 3
|
||
|
|
'self', 'Ellipsis', 'NotImplemented',
|
||
|
|
]
|
||
|
|
|
||
|
|
IDENT_KIND = WordList.new(:ident).
|
||
|
|
add(KEYWORDS, :keyword).
|
||
|
|
add(OLD_KEYWORDS, :old_keyword).
|
||
|
|
add(PREDEFINED_METHODS_AND_TYPES, :predefined).
|
||
|
|
add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
|
||
|
|
add(PREDEFINED_EXCEPTIONS, :exception)
|
||
|
|
|
||
|
|
NAME = / [^\W\d] \w* /x
|
||
|
|
ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||
|
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
|
||
|
|
|
||
|
|
OPERATOR = /
|
||
|
|
\.\.\. | # ellipsis
|
||
|
|
\.(?!\d) | # dot but not decimal point
|
||
|
|
[,;:()\[\]{}] | # simple delimiters
|
||
|
|
\/\/=? | \*\*=? | # special math
|
||
|
|
[-+*\/%&|^]=? | # ordinary math and binary logic
|
||
|
|
[~`] | # binary complement and inspection
|
||
|
|
<<=? | >>=? | [<>=]=? | != # comparison and assignment
|
||
|
|
/x
|
||
|
|
|
||
|
|
STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
|
||
|
|
h[delimiter] = Regexp.union delimiter
|
||
|
|
end
|
||
|
|
|
||
|
|
STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
|
||
|
|
h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
|
||
|
|
end
|
||
|
|
|
||
|
|
DEF_NEW_STATE = WordList.new(:initial).
|
||
|
|
add(%w(def), :def_expected).
|
||
|
|
add(%w(import from), :include_expected).
|
||
|
|
add(%w(class), :class_expected)
|
||
|
|
|
||
|
|
DESCRIPTOR = /
|
||
|
|
#{NAME}
|
||
|
|
(?: \. #{NAME} )*
|
||
|
|
| \*
|
||
|
|
/x
|
||
|
|
|
||
|
|
def scan_tokens tokens, options
|
||
|
|
|
||
|
|
state = :initial
|
||
|
|
string_delimiter = nil
|
||
|
|
string_raw = false
|
||
|
|
import_clause = class_name_follows = last_token_dot = false
|
||
|
|
unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
|
||
|
|
from_import_state = []
|
||
|
|
|
||
|
|
until eos?
|
||
|
|
|
||
|
|
kind = nil
|
||
|
|
match = nil
|
||
|
|
|
||
|
|
if state == :string
|
||
|
|
if scan(STRING_DELIMITER_REGEXP[string_delimiter])
|
||
|
|
tokens << [matched, :delimiter]
|
||
|
|
tokens << [:close, :string]
|
||
|
|
state = :initial
|
||
|
|
next
|
||
|
|
elsif string_delimiter.size == 3 && scan(/\n/)
|
||
|
|
kind = :content
|
||
|
|
elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
|
||
|
|
kind = :content
|
||
|
|
elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
|
||
|
|
kind = :char
|
||
|
|
elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
|
||
|
|
kind = :char
|
||
|
|
elsif scan(/ \\ . /x)
|
||
|
|
kind = :content
|
||
|
|
elsif scan(/ \\ | $ /x)
|
||
|
|
tokens << [:close, :string]
|
||
|
|
kind = :error
|
||
|
|
state = :initial
|
||
|
|
else
|
||
|
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
|
||
|
|
end
|
||
|
|
|
||
|
|
elsif match = scan(/ [ \t]+ | \\\n /x)
|
||
|
|
tokens << [match, :space]
|
||
|
|
next
|
||
|
|
|
||
|
|
elsif match = scan(/\n/)
|
||
|
|
tokens << [match, :space]
|
||
|
|
state = :initial if state == :include_expected
|
||
|
|
next
|
||
|
|
|
||
|
|
elsif match = scan(/ \# [^\n]* /mx)
|
||
|
|
tokens << [match, :comment]
|
||
|
|
next
|
||
|
|
|
||
|
|
elsif state == :initial
|
||
|
|
|
||
|
|
if scan(/#{OPERATOR}/o)
|
||
|
|
kind = :operator
|
||
|
|
|
||
|
|
elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
|
||
|
|
tokens << [:open, :string]
|
||
|
|
string_delimiter = self[2]
|
||
|
|
string_raw = false
|
||
|
|
modifiers = self[1]
|
||
|
|
unless modifiers.empty?
|
||
|
|
string_raw = !!modifiers.index(?r)
|
||
|
|
tokens << [modifiers, :modifier]
|
||
|
|
match = string_delimiter
|
||
|
|
end
|
||
|
|
state = :string
|
||
|
|
kind = :delimiter
|
||
|
|
|
||
|
|
# TODO: backticks
|
||
|
|
|
||
|
|
elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
|
||
|
|
kind = IDENT_KIND[match]
|
||
|
|
# TODO: keyword arguments
|
||
|
|
kind = :ident if last_token_dot
|
||
|
|
if kind == :old_keyword
|
||
|
|
kind = check(/\(/) ? :ident : :keyword
|
||
|
|
elsif kind == :predefined && check(/ *=/)
|
||
|
|
kind = :ident
|
||
|
|
elsif kind == :keyword
|
||
|
|
state = DEF_NEW_STATE[match]
|
||
|
|
from_import_state << match.to_sym if state == :include_expected
|
||
|
|
end
|
||
|
|
|
||
|
|
elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
|
||
|
|
kind = :decorator
|
||
|
|
|
||
|
|
elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
|
||
|
|
kind = :hex
|
||
|
|
|
||
|
|
elsif scan(/0[bB][01]+[lL]?/)
|
||
|
|
kind = :bin
|
||
|
|
|
||
|
|
elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
|
||
|
|
kind = :float
|
||
|
|
if scan(/[jJ]/)
|
||
|
|
match << matched
|
||
|
|
kind = :imaginary
|
||
|
|
end
|
||
|
|
|
||
|
|
elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
|
||
|
|
kind = :oct
|
||
|
|
|
||
|
|
elsif match = scan(/\d+([lL])?/)
|
||
|
|
kind = :integer
|
||
|
|
if self[1] == nil && scan(/[jJ]/)
|
||
|
|
match << matched
|
||
|
|
kind = :imaginary
|
||
|
|
end
|
||
|
|
|
||
|
|
else
|
||
|
|
getch
|
||
|
|
kind = :error
|
||
|
|
|
||
|
|
end
|
||
|
|
|
||
|
|
elsif state == :def_expected
|
||
|
|
state = :initial
|
||
|
|
if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
|
||
|
|
kind = :method
|
||
|
|
else
|
||
|
|
next
|
||
|
|
end
|
||
|
|
|
||
|
|
elsif state == :class_expected
|
||
|
|
state = :initial
|
||
|
|
if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
|
||
|
|
kind = :class
|
||
|
|
else
|
||
|
|
next
|
||
|
|
end
|
||
|
|
|
||
|
|
elsif state == :include_expected
|
||
|
|
if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
|
||
|
|
kind = :include
|
||
|
|
if match == 'as'
|
||
|
|
kind = :keyword
|
||
|
|
from_import_state << :as
|
||
|
|
elsif from_import_state.first == :from && match == 'import'
|
||
|
|
kind = :keyword
|
||
|
|
from_import_state << :import
|
||
|
|
elsif from_import_state.last == :as
|
||
|
|
# kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
|
||
|
|
kind = :ident
|
||
|
|
from_import_state.pop
|
||
|
|
elsif IDENT_KIND[match] == :keyword
|
||
|
|
unscan
|
||
|
|
match = nil
|
||
|
|
state = :initial
|
||
|
|
next
|
||
|
|
end
|
||
|
|
elsif match = scan(/,/)
|
||
|
|
from_import_state.pop if from_import_state.last == :as
|
||
|
|
kind = :operator
|
||
|
|
else
|
||
|
|
from_import_state = []
|
||
|
|
state = :initial
|
||
|
|
next
|
||
|
|
end
|
||
|
|
|
||
|
|
else
|
||
|
|
raise_inspect 'Unknown state', tokens, state
|
||
|
|
|
||
|
|
end
|
||
|
|
|
||
|
|
match ||= matched
|
||
|
|
if $CODERAY_DEBUG and not kind
|
||
|
|
raise_inspect 'Error token %p in line %d' %
|
||
|
|
[[match, kind], line], tokens, state
|
||
|
|
end
|
||
|
|
raise_inspect 'Empty token', tokens, state unless match
|
||
|
|
|
||
|
|
last_token_dot = match == '.'
|
||
|
|
|
||
|
|
tokens << [match, kind]
|
||
|
|
|
||
|
|
end
|
||
|
|
|
||
|
|
if state == :string
|
||
|
|
tokens << [:close, :string]
|
||
|
|
end
|
||
|
|
|
||
|
|
tokens
|
||
|
|
end
|
||
|
|
|
||
|
|
end
|
||
|
|
|
||
|
|
end
|
||
|
|
end
|