Upgrade CodeRay to 0.9.2 (#3359).
git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@3592 e93f8b46-1217-0410-a6f0-8f06a7374b81
This commit is contained in:
parent
e6c8760ad7
commit
0097770626
|
@ -1,53 +0,0 @@
|
||||||
= CodeRay - Trunk folder structure
|
|
||||||
|
|
||||||
== bench - Benchmarking system
|
|
||||||
|
|
||||||
All benchmarking stuff goes here.
|
|
||||||
|
|
||||||
Test inputs are stored in files named <code>example.<lang></code>.
|
|
||||||
Test outputs go to <code>bench/test.<encoder-default-file-extension></code>.
|
|
||||||
|
|
||||||
Run <code>bench/bench.rb</code> to get a usage description.
|
|
||||||
|
|
||||||
Run <code>rake bench</code> to perform an example benchmark.
|
|
||||||
|
|
||||||
|
|
||||||
== bin - Scripts
|
|
||||||
|
|
||||||
Executional files for CodeRay.
|
|
||||||
|
|
||||||
|
|
||||||
== demo - Demos and functional tests
|
|
||||||
|
|
||||||
Demonstrational scripts to show of CodeRay's features.
|
|
||||||
|
|
||||||
Run them as functional tests with <code>rake test:demos</code>.
|
|
||||||
|
|
||||||
|
|
||||||
== etc - Lots of stuff
|
|
||||||
|
|
||||||
Some addidtional files for CodeRay, mainly graphics and Vim scripts.
|
|
||||||
|
|
||||||
|
|
||||||
== gem_server - Gem output folder
|
|
||||||
|
|
||||||
For <code>rake gem</code>.
|
|
||||||
|
|
||||||
|
|
||||||
== lib - CodeRay library code
|
|
||||||
|
|
||||||
This is the base directory for the CodeRay library.
|
|
||||||
|
|
||||||
|
|
||||||
== rake_helpers - Rake helper libraries
|
|
||||||
|
|
||||||
Some files to enhance Rake, including the Autumnal Rdoc template and some scripts.
|
|
||||||
|
|
||||||
|
|
||||||
== test - Tests
|
|
||||||
|
|
||||||
Tests for the scanners.
|
|
||||||
|
|
||||||
Each language has its own subfolder and sub-suite.
|
|
||||||
|
|
||||||
Run with <code>rake test</code>.
|
|
|
@ -1,82 +0,0 @@
|
||||||
#!/usr/bin/env ruby
|
|
||||||
# CodeRay Executable
|
|
||||||
#
|
|
||||||
# Version: 0.1
|
|
||||||
# Author: murphy
|
|
||||||
|
|
||||||
def err msg
|
|
||||||
$stderr.puts msg
|
|
||||||
end
|
|
||||||
|
|
||||||
begin
|
|
||||||
require 'coderay'
|
|
||||||
|
|
||||||
if ARGV.empty?
|
|
||||||
puts <<-USAGE
|
|
||||||
CodeRay #{CodeRay::VERSION} (http://rd.cYcnus.de/coderay)
|
|
||||||
Usage:
|
|
||||||
coderay -<lang> [-<format>] < file > output
|
|
||||||
coderay file [-<format>]
|
|
||||||
Example:
|
|
||||||
coderay -ruby -statistic < foo.rb
|
|
||||||
coderay codegen.c # generates codegen.c.html
|
|
||||||
USAGE
|
|
||||||
end
|
|
||||||
|
|
||||||
first, second = ARGV
|
|
||||||
|
|
||||||
if first
|
|
||||||
if first[/-(\w+)/] == first
|
|
||||||
lang = $1.to_sym
|
|
||||||
input = $stdin.read
|
|
||||||
tokens = :scan
|
|
||||||
elsif first == '-'
|
|
||||||
lang = $1.to_sym
|
|
||||||
input = $stdin.read
|
|
||||||
tokens = :scan
|
|
||||||
else
|
|
||||||
file = first
|
|
||||||
tokens = CodeRay.scan_file file
|
|
||||||
output_filename, output_ext = file, /#{Regexp.escape(File.extname(file))}$/
|
|
||||||
end
|
|
||||||
else
|
|
||||||
puts 'No lang/file given.'
|
|
||||||
exit 1
|
|
||||||
end
|
|
||||||
|
|
||||||
if second
|
|
||||||
if second[/-(\w+)/] == second
|
|
||||||
format = $1.to_sym
|
|
||||||
else
|
|
||||||
raise 'Invalid format (must be -xxx).'
|
|
||||||
end
|
|
||||||
else
|
|
||||||
$stderr.puts 'No format given; setting to default (HTML Page)'
|
|
||||||
format = :page
|
|
||||||
end
|
|
||||||
|
|
||||||
# TODO: allow streaming
|
|
||||||
if tokens == :scan
|
|
||||||
output = CodeRay::Duo[lang => format].highlight input #, :stream => true
|
|
||||||
else
|
|
||||||
output = tokens.encode format
|
|
||||||
end
|
|
||||||
out = $stdout
|
|
||||||
if output_filename
|
|
||||||
output_filename += '.' + CodeRay::Encoders[format]::FILE_EXTENSION
|
|
||||||
if File.exist? output_filename
|
|
||||||
err 'File %s already exists.' % output_filename
|
|
||||||
exit
|
|
||||||
else
|
|
||||||
out = File.open output_filename, 'w'
|
|
||||||
end
|
|
||||||
end
|
|
||||||
out.print output
|
|
||||||
|
|
||||||
rescue => boom
|
|
||||||
err "Error: #{boom.message}\n"
|
|
||||||
err boom.backtrace
|
|
||||||
err '-' * 50
|
|
||||||
err ARGV
|
|
||||||
exit 1
|
|
||||||
end
|
|
|
@ -1,4 +0,0 @@
|
||||||
#!/usr/bin/env ruby
|
|
||||||
require 'coderay'
|
|
||||||
|
|
||||||
puts CodeRay::Encoders[:html]::CSS.new.stylesheet
|
|
|
@ -1,9 +0,0 @@
|
||||||
module CodeRay
|
|
||||||
module Encoders
|
|
||||||
|
|
||||||
map :stats => :statistic,
|
|
||||||
:plain => :text,
|
|
||||||
:tex => :latex
|
|
||||||
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -1,44 +0,0 @@
|
||||||
module CodeRay
|
|
||||||
module Encoders
|
|
||||||
|
|
||||||
# The Tokens encoder converts the tokens to a simple
|
|
||||||
# readable format. It doesn't use colors and is mainly
|
|
||||||
# intended for console output.
|
|
||||||
#
|
|
||||||
# The tokens are converted with Tokens.write_token.
|
|
||||||
#
|
|
||||||
# The format is:
|
|
||||||
#
|
|
||||||
# <token-kind> \t <escaped token-text> \n
|
|
||||||
#
|
|
||||||
# Example:
|
|
||||||
#
|
|
||||||
# require 'coderay'
|
|
||||||
# puts CodeRay.scan("puts 3 + 4", :ruby).tokens
|
|
||||||
#
|
|
||||||
# prints:
|
|
||||||
#
|
|
||||||
# ident puts
|
|
||||||
# space
|
|
||||||
# integer 3
|
|
||||||
# space
|
|
||||||
# operator +
|
|
||||||
# space
|
|
||||||
# integer 4
|
|
||||||
#
|
|
||||||
class Tokens < Encoder
|
|
||||||
|
|
||||||
include Streamable
|
|
||||||
register_for :tokens
|
|
||||||
|
|
||||||
FILE_EXTENSION = 'tok'
|
|
||||||
|
|
||||||
protected
|
|
||||||
def token text, kind
|
|
||||||
@out << CodeRay::Tokens.write_token(text, kind)
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -1,15 +0,0 @@
|
||||||
module CodeRay
|
|
||||||
module Scanners
|
|
||||||
|
|
||||||
map :cpp => :c,
|
|
||||||
:plain => :plaintext,
|
|
||||||
:pascal => :delphi,
|
|
||||||
:irb => :ruby,
|
|
||||||
:xml => :html,
|
|
||||||
:xhtml => :nitro_xhtml,
|
|
||||||
:nitro => :nitro_xhtml
|
|
||||||
|
|
||||||
default :plain
|
|
||||||
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -1,130 +0,0 @@
|
||||||
module CodeRay
|
|
||||||
module Scanners
|
|
||||||
class Java < Scanner
|
|
||||||
|
|
||||||
register_for :java
|
|
||||||
|
|
||||||
RESERVED_WORDS = %w(abstract assert break case catch class
|
|
||||||
const continue default do else enum extends final finally for
|
|
||||||
goto if implements import instanceof interface native new
|
|
||||||
package private protected public return static strictfp super switch
|
|
||||||
synchronized this throw throws transient try void volatile while)
|
|
||||||
|
|
||||||
PREDEFINED_TYPES = %w(boolean byte char double float int long short)
|
|
||||||
|
|
||||||
PREDEFINED_CONSTANTS = %w(true false null)
|
|
||||||
|
|
||||||
IDENT_KIND = WordList.new(:ident).
|
|
||||||
add(RESERVED_WORDS, :reserved).
|
|
||||||
add(PREDEFINED_TYPES, :pre_type).
|
|
||||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
|
||||||
|
|
||||||
ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
|
||||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
|
||||||
|
|
||||||
def scan_tokens tokens, options
|
|
||||||
state = :initial
|
|
||||||
|
|
||||||
until eos?
|
|
||||||
kind = nil
|
|
||||||
match = nil
|
|
||||||
|
|
||||||
case state
|
|
||||||
when :initial
|
|
||||||
|
|
||||||
if scan(/ \s+ | \\\n /x)
|
|
||||||
kind = :space
|
|
||||||
|
|
||||||
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
|
||||||
kind = :comment
|
|
||||||
|
|
||||||
elsif match = scan(/ \# \s* if \s* 0 /x)
|
|
||||||
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
|
|
||||||
kind = :comment
|
|
||||||
|
|
||||||
elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
|
|
||||||
kind = :operator
|
|
||||||
|
|
||||||
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
|
|
||||||
kind = IDENT_KIND[match]
|
|
||||||
if kind == :ident and check(/:(?!:)/)
|
|
||||||
match << scan(/:/)
|
|
||||||
kind = :label
|
|
||||||
end
|
|
||||||
|
|
||||||
elsif match = scan(/L?"/)
|
|
||||||
tokens << [:open, :string]
|
|
||||||
if match[0] == ?L
|
|
||||||
tokens << ['L', :modifier]
|
|
||||||
match = '"'
|
|
||||||
end
|
|
||||||
state = :string
|
|
||||||
kind = :delimiter
|
|
||||||
|
|
||||||
elsif scan(%r! \@ .* !x)
|
|
||||||
kind = :preprocessor
|
|
||||||
|
|
||||||
elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
|
|
||||||
kind = :char
|
|
||||||
|
|
||||||
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
|
||||||
kind = :hex
|
|
||||||
|
|
||||||
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
|
|
||||||
kind = :oct
|
|
||||||
|
|
||||||
elsif scan(/(?:\d+)(?![.eEfF])/)
|
|
||||||
kind = :integer
|
|
||||||
|
|
||||||
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
|
||||||
kind = :float
|
|
||||||
|
|
||||||
else
|
|
||||||
getch
|
|
||||||
kind = :error
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
when :string
|
|
||||||
if scan(/[^\\\n"]+/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/"/)
|
|
||||||
tokens << ['"', :delimiter]
|
|
||||||
tokens << [:close, :string]
|
|
||||||
state = :initial
|
|
||||||
next
|
|
||||||
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
|
||||||
kind = :char
|
|
||||||
elsif scan(/ \\ | $ /x)
|
|
||||||
tokens << [:close, :string]
|
|
||||||
kind = :error
|
|
||||||
state = :initial
|
|
||||||
else
|
|
||||||
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
|
||||||
end
|
|
||||||
|
|
||||||
else
|
|
||||||
raise_inspect 'Unknown state', tokens
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
match ||= matched
|
|
||||||
if $DEBUG and not kind
|
|
||||||
raise_inspect 'Error token %p in line %d' %
|
|
||||||
[[match, kind], line], tokens
|
|
||||||
end
|
|
||||||
raise_inspect 'Empty token', tokens unless match
|
|
||||||
|
|
||||||
tokens << [match, kind]
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
if state == :string
|
|
||||||
tokens << [:close, :string]
|
|
||||||
end
|
|
||||||
|
|
||||||
tokens
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
||||||
end
|
|
|
@ -1,176 +0,0 @@
|
||||||
# http://pastie.textmate.org/50774/
|
|
||||||
module CodeRay module Scanners
|
|
||||||
|
|
||||||
class JavaScript < Scanner
|
|
||||||
|
|
||||||
register_for :javascript
|
|
||||||
|
|
||||||
RESERVED_WORDS = [
|
|
||||||
'asm', 'break', 'case', 'continue', 'default', 'do', 'else',
|
|
||||||
'for', 'goto', 'if', 'return', 'switch', 'while',
|
|
||||||
# 'struct', 'union', 'enum', 'typedef',
|
|
||||||
# 'static', 'register', 'auto', 'extern',
|
|
||||||
# 'sizeof',
|
|
||||||
'typeof',
|
|
||||||
# 'volatile', 'const', # C89
|
|
||||||
# 'inline', 'restrict', # C99
|
|
||||||
'var', 'function','try','new','in',
|
|
||||||
'instanceof','throw','catch'
|
|
||||||
]
|
|
||||||
|
|
||||||
PREDEFINED_CONSTANTS = [
|
|
||||||
'void', 'null', 'this',
|
|
||||||
'true', 'false','undefined',
|
|
||||||
]
|
|
||||||
|
|
||||||
IDENT_KIND = WordList.new(:ident).
|
|
||||||
add(RESERVED_WORDS, :reserved).
|
|
||||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
|
||||||
|
|
||||||
ESCAPE = / [rbfnrtv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
|
||||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
|
||||||
|
|
||||||
def scan_tokens tokens, options
|
|
||||||
|
|
||||||
state = :initial
|
|
||||||
string_type = nil
|
|
||||||
regexp_allowed = true
|
|
||||||
|
|
||||||
until eos?
|
|
||||||
|
|
||||||
kind = :error
|
|
||||||
match = nil
|
|
||||||
|
|
||||||
if state == :initial
|
|
||||||
|
|
||||||
if scan(/ \s+ | \\\n /x)
|
|
||||||
kind = :space
|
|
||||||
|
|
||||||
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
|
||||||
kind = :comment
|
|
||||||
regexp_allowed = false
|
|
||||||
|
|
||||||
elsif match = scan(/ \# \s* if \s* 0 /x)
|
|
||||||
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
|
|
||||||
kind = :comment
|
|
||||||
regexp_allowed = false
|
|
||||||
|
|
||||||
elsif regexp_allowed and scan(/\//)
|
|
||||||
tokens << [:open, :regexp]
|
|
||||||
state = :regex
|
|
||||||
kind = :delimiter
|
|
||||||
|
|
||||||
elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%] | \.(?!\d) /x)
|
|
||||||
kind = :operator
|
|
||||||
regexp_allowed=true
|
|
||||||
|
|
||||||
elsif match = scan(/ [$A-Za-z_][A-Za-z_0-9]* /x)
|
|
||||||
kind = IDENT_KIND[match]
|
|
||||||
# if kind == :ident and check(/:(?!:)/)
|
|
||||||
# match << scan(/:/)
|
|
||||||
# kind = :label
|
|
||||||
# end
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif match = scan(/["']/)
|
|
||||||
tokens << [:open, :string]
|
|
||||||
string_type = matched
|
|
||||||
state = :string
|
|
||||||
kind = :delimiter
|
|
||||||
|
|
||||||
# elsif scan(/#\s*(\w*)/)
|
|
||||||
# kind = :preprocessor # FIXME multiline preprocs
|
|
||||||
# state = :include_expected if self[1] == 'include'
|
|
||||||
#
|
|
||||||
# elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
|
|
||||||
# kind = :char
|
|
||||||
|
|
||||||
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
|
||||||
kind = :hex
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
|
|
||||||
kind = :oct
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif scan(/(?:\d+)(?![.eEfF])/)
|
|
||||||
kind = :integer
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
|
||||||
kind = :float
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
else
|
|
||||||
getch
|
|
||||||
end
|
|
||||||
|
|
||||||
elsif state == :regex
|
|
||||||
if scan(/[^\\\/]+/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/\\\/|\\\\/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/\//)
|
|
||||||
tokens << [matched, :delimiter]
|
|
||||||
tokens << [:close, :regexp]
|
|
||||||
state = :initial
|
|
||||||
next
|
|
||||||
else
|
|
||||||
getch
|
|
||||||
kind = :content
|
|
||||||
end
|
|
||||||
|
|
||||||
elsif state == :string
|
|
||||||
if scan(/[^\\"']+/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/["']/)
|
|
||||||
if string_type==matched
|
|
||||||
tokens << [matched, :delimiter]
|
|
||||||
tokens << [:close, :string]
|
|
||||||
state = :initial
|
|
||||||
string_type=nil
|
|
||||||
next
|
|
||||||
else
|
|
||||||
kind = :content
|
|
||||||
end
|
|
||||||
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
|
||||||
kind = :char
|
|
||||||
elsif scan(/ \\ | $ /x)
|
|
||||||
kind = :error
|
|
||||||
state = :initial
|
|
||||||
else
|
|
||||||
raise "else case \" reached; %p not handled." % peek(1), tokens
|
|
||||||
end
|
|
||||||
|
|
||||||
# elsif state == :include_expected
|
|
||||||
# if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
|
|
||||||
# kind = :include
|
|
||||||
# state = :initial
|
|
||||||
#
|
|
||||||
# elsif match = scan(/\s+/)
|
|
||||||
# kind = :space
|
|
||||||
# state = :initial if match.index ?\n
|
|
||||||
#
|
|
||||||
# else
|
|
||||||
# getch
|
|
||||||
#
|
|
||||||
# end
|
|
||||||
#
|
|
||||||
else
|
|
||||||
raise 'else-case reached', tokens
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
match ||= matched
|
|
||||||
# raise [match, kind], tokens if kind == :error
|
|
||||||
|
|
||||||
tokens << [match, kind]
|
|
||||||
|
|
||||||
end
|
|
||||||
tokens
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end end
|
|
|
@ -1,165 +0,0 @@
|
||||||
module CodeRay module Scanners
|
|
||||||
|
|
||||||
class PHP < Scanner
|
|
||||||
|
|
||||||
register_for :php
|
|
||||||
|
|
||||||
RESERVED_WORDS = [
|
|
||||||
'and', 'or', 'xor', '__FILE__', 'exception', '__LINE__', 'array', 'as', 'break', 'case',
|
|
||||||
'class', 'const', 'continue', 'declare', 'default',
|
|
||||||
'die', 'do', 'echo', 'else', 'elseif',
|
|
||||||
'empty', 'enddeclare', 'endfor', 'endforeach', 'endif',
|
|
||||||
'endswitch', 'endwhile', 'eval', 'exit', 'extends',
|
|
||||||
'for', 'foreach', 'function', 'global', 'if',
|
|
||||||
'include', 'include_once', 'isset', 'list', 'new',
|
|
||||||
'print', 'require', 'require_once', 'return', 'static',
|
|
||||||
'switch', 'unset', 'use', 'var', 'while',
|
|
||||||
'__FUNCTION__', '__CLASS__', '__METHOD__', 'final', 'php_user_filter',
|
|
||||||
'interface', 'implements', 'extends', 'public', 'private',
|
|
||||||
'protected', 'abstract', 'clone', 'try', 'catch',
|
|
||||||
'throw', 'cfunction', 'old_function'
|
|
||||||
]
|
|
||||||
|
|
||||||
PREDEFINED_CONSTANTS = [
|
|
||||||
'null', '$this', 'true', 'false'
|
|
||||||
]
|
|
||||||
|
|
||||||
IDENT_KIND = WordList.new(:ident).
|
|
||||||
add(RESERVED_WORDS, :reserved).
|
|
||||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
|
||||||
|
|
||||||
ESCAPE = / [\$\wrbfnrtv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
|
||||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
|
||||||
|
|
||||||
def scan_tokens tokens, options
|
|
||||||
|
|
||||||
state = :waiting_php
|
|
||||||
string_type = nil
|
|
||||||
regexp_allowed = true
|
|
||||||
|
|
||||||
until eos?
|
|
||||||
|
|
||||||
kind = :error
|
|
||||||
match = nil
|
|
||||||
|
|
||||||
if state == :initial
|
|
||||||
|
|
||||||
if scan(/ \s+ | \\\n /x)
|
|
||||||
kind = :space
|
|
||||||
|
|
||||||
elsif scan(/\?>/)
|
|
||||||
kind = :char
|
|
||||||
state = :waiting_php
|
|
||||||
|
|
||||||
elsif scan(%r{ (//|\#) [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) }mx)
|
|
||||||
kind = :comment
|
|
||||||
regexp_allowed = false
|
|
||||||
|
|
||||||
elsif match = scan(/ \# \s* if \s* 0 /x)
|
|
||||||
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
|
|
||||||
kind = :comment
|
|
||||||
regexp_allowed = false
|
|
||||||
|
|
||||||
elsif regexp_allowed and scan(/\//)
|
|
||||||
tokens << [:open, :regexp]
|
|
||||||
state = :regex
|
|
||||||
kind = :delimiter
|
|
||||||
|
|
||||||
elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%] | \.(?!\d) /x)
|
|
||||||
kind = :operator
|
|
||||||
regexp_allowed=true
|
|
||||||
|
|
||||||
elsif match = scan(/ [$@A-Za-z_][A-Za-z_0-9]* /x)
|
|
||||||
kind = IDENT_KIND[match]
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif match = scan(/["']/)
|
|
||||||
tokens << [:open, :string]
|
|
||||||
string_type = matched
|
|
||||||
state = :string
|
|
||||||
kind = :delimiter
|
|
||||||
|
|
||||||
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
|
||||||
kind = :hex
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
|
|
||||||
kind = :oct
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif scan(/(?:\d+)(?![.eEfF])/)
|
|
||||||
kind = :integer
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
|
||||||
kind = :float
|
|
||||||
regexp_allowed=false
|
|
||||||
|
|
||||||
else
|
|
||||||
getch
|
|
||||||
end
|
|
||||||
|
|
||||||
elsif state == :regex
|
|
||||||
if scan(/[^\\\/]+/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/\\\/|\\/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/\//)
|
|
||||||
tokens << [matched, :delimiter]
|
|
||||||
tokens << [:close, :regexp]
|
|
||||||
state = :initial
|
|
||||||
next
|
|
||||||
else
|
|
||||||
getch
|
|
||||||
kind = :content
|
|
||||||
end
|
|
||||||
|
|
||||||
elsif state == :string
|
|
||||||
if scan(/[^\\"']+/)
|
|
||||||
kind = :content
|
|
||||||
elsif scan(/["']/)
|
|
||||||
if string_type==matched
|
|
||||||
tokens << [matched, :delimiter]
|
|
||||||
tokens << [:close, :string]
|
|
||||||
state = :initial
|
|
||||||
string_type=nil
|
|
||||||
next
|
|
||||||
else
|
|
||||||
kind = :content
|
|
||||||
end
|
|
||||||
elsif scan(/ \\ (?: \S ) /mox)
|
|
||||||
kind = :char
|
|
||||||
elsif scan(/ \\ | $ /x)
|
|
||||||
kind = :error
|
|
||||||
state = :initial
|
|
||||||
else
|
|
||||||
raise "else case \" reached; %p not handled." % peek(1), tokens
|
|
||||||
end
|
|
||||||
|
|
||||||
elsif state == :waiting_php
|
|
||||||
if scan(/<\?php/m)
|
|
||||||
kind = :char
|
|
||||||
state = :initial
|
|
||||||
elsif scan(/[^<]+/)
|
|
||||||
kind = :comment
|
|
||||||
else
|
|
||||||
kind = :comment
|
|
||||||
getch
|
|
||||||
end
|
|
||||||
else
|
|
||||||
raise 'else-case reached', tokens
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
match ||= matched
|
|
||||||
|
|
||||||
tokens << [match, kind]
|
|
||||||
|
|
||||||
end
|
|
||||||
tokens
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
end end
|
|
|
@ -1,7 +1,7 @@
|
||||||
= CodeRay
|
= CodeRay
|
||||||
|
|
||||||
[- Tired of blue'n'gray? Try the original version of this documentation on
|
[- Tired of blue'n'gray? Try the original version of this documentation on
|
||||||
http://rd.cYcnus.de/coderay/doc (use Ctrl+Click to open it in its own frame.) -]
|
coderay.rubychan.de[http://coderay.rubychan.de/doc/] (use Ctrl+Click to open it in its own frame.) -]
|
||||||
|
|
||||||
== About
|
== About
|
||||||
CodeRay is a Ruby library for syntax highlighting.
|
CodeRay is a Ruby library for syntax highlighting.
|
||||||
|
@ -18,14 +18,11 @@ And with line numbers.
|
||||||
* is what everybody should have on their website
|
* is what everybody should have on their website
|
||||||
* solves all your problems and makes the girls run after you
|
* solves all your problems and makes the girls run after you
|
||||||
|
|
||||||
Version: 0.7.4 (2006.october.20)
|
Version: 0.9.2
|
||||||
Author:: murphy (Kornelius Kalnbach)
|
Author:: murphy (Kornelius Kalnbach)
|
||||||
Contact:: murphy rubychan de
|
Contact:: murphy rubychan de
|
||||||
Website:: coderay.rubychan.de[http://coderay.rubychan.de]
|
Website:: coderay.rubychan.de[http://coderay.rubychan.de]
|
||||||
License:: GNU LGPL; see LICENSE file in the main directory.
|
License:: GNU LGPL; see LICENSE file in the main directory.
|
||||||
Subversion:: $Id: README 219 2006-10-20 15:52:25Z murphy $
|
|
||||||
|
|
||||||
-----
|
|
||||||
|
|
||||||
== Installation
|
== Installation
|
||||||
|
|
||||||
|
@ -33,17 +30,10 @@ You need RubyGems[http://rubyforge.org/frs/?group_id=126].
|
||||||
|
|
||||||
% gem install coderay
|
% gem install coderay
|
||||||
|
|
||||||
Since CodeRay is still in beta stage, nightly buildy may be useful:
|
|
||||||
|
|
||||||
% gem install coderay -rs rd.cYcnus.de/coderay
|
|
||||||
|
|
||||||
|
|
||||||
=== Dependencies
|
=== Dependencies
|
||||||
|
|
||||||
CodeRay needs Ruby 1.8 and the
|
CodeRay needs Ruby 1.8.6 or later. It also runs with Ruby 1.9.1+ and JRuby 1.1+.
|
||||||
strscan[http://www.ruby-doc.org/stdlib/libdoc/strscan/rdoc/index.htm]
|
|
||||||
library (part of the standard library.) It should also run with Ruby 1.9 and
|
|
||||||
yarv.
|
|
||||||
|
|
||||||
|
|
||||||
== Example Usage
|
== Example Usage
|
||||||
|
@ -60,11 +50,9 @@ yarv.
|
||||||
|
|
||||||
See CodeRay.
|
See CodeRay.
|
||||||
|
|
||||||
Please report errors in this documentation to <coderay cycnus de>.
|
Please report errors in this documentation to <murphy rubychan de>.
|
||||||
|
|
||||||
|
|
||||||
-----
|
|
||||||
|
|
||||||
== Credits
|
== Credits
|
||||||
|
|
||||||
=== Special Thanks to
|
=== Special Thanks to
|
||||||
|
@ -72,30 +60,43 @@ Please report errors in this documentation to <coderay cycnus de>.
|
||||||
* licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder
|
* licenser (Heinz N. Gies) for ending my QBasic career, inventing the Coder
|
||||||
project and the input/output plugin system.
|
project and the input/output plugin system.
|
||||||
CodeRay would not exist without him.
|
CodeRay would not exist without him.
|
||||||
|
* bovi (Daniel Bovensiepen) for helping me out on various occasions.
|
||||||
|
|
||||||
=== Thanks to
|
=== Thanks to
|
||||||
|
|
||||||
* Caleb Clausen for writing RubyLexer (see
|
* Caleb Clausen for writing RubyLexer (see
|
||||||
http://rubyforge.org/projects/rubylexer) and lots of very interesting mail
|
http://rubyforge.org/projects/rubylexer) and lots of very interesting mail
|
||||||
traffic
|
traffic
|
||||||
* birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur. You
|
* birkenfeld (Georg Brandl) and mitsuhiku (Arnim Ronacher) for PyKleur, now pygments.
|
||||||
guys rock!
|
You guys rock!
|
||||||
* Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax)
|
* Jamis Buck for writing Syntax (see http://rubyforge.org/projects/syntax)
|
||||||
I got some useful ideas from it.
|
I got some useful ideas from it.
|
||||||
* Doug Kearns and everyone else who worked on ruby.vim - it not only helped me
|
* Doug Kearns and everyone else who worked on ruby.vim - it not only helped me
|
||||||
coding CodeRay, but also gave me a wonderful target to reach for the Ruby
|
coding CodeRay, but also gave me a wonderful target to reach for the Ruby
|
||||||
scanner.
|
scanner.
|
||||||
* everyone who used CodeBB on http://www.rubyforen.de and
|
* everyone who uses CodeBB on http://www.rubyforen.de and http://www.python-forum.de
|
||||||
http://www.infhu.de/mx
|
* iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de
|
||||||
* iGEL, magichisoka, manveru, WoNáDo and everyone I forgot from rubyforen.de
|
* Dethix from ruby-mine.de
|
||||||
* Daniel and Dethix from ruby-mine.de
|
* zickzackw
|
||||||
* Dookie (who is no longer with us...) and Leonidas from
|
* Dookie (who is no longer with us...) and Leonidas from http://www.python-forum.de
|
||||||
http://www.python-forum.de
|
|
||||||
* Andreas Schwarz for finding out that CaseIgnoringWordList was not case
|
* Andreas Schwarz for finding out that CaseIgnoringWordList was not case
|
||||||
ignoring! Such things really make you write tests.
|
ignoring! Such things really make you write tests.
|
||||||
|
* closure for the first version of the Scheme scanner.
|
||||||
|
* Stefan Walk for the first version of the JavaScript and PHP scanners.
|
||||||
|
* Josh Goebel for another version of the JavaScript scanner, a SQL and a Diff scanner.
|
||||||
|
* Jonathan Younger for pointing out the licence confusion caused by wrong LICENSE file.
|
||||||
|
* Jeremy Hinegardner for finding the shebang-on-empty-file bug in FileType.
|
||||||
|
* Charles Oliver Nutter and Yehuda Katz for helping me benchmark CodeRay on JRuby.
|
||||||
|
* Andreas Neuhaus for pointing out a markup bug in coderay/for_redcloth.
|
||||||
|
* 0xf30fc7 for the FileType patch concerning Delphi file extensions.
|
||||||
|
* The folks at redmine.org - thank you for using and fixing CodeRay!
|
||||||
|
* Keith Pitt for his SQL scanners
|
||||||
|
* Rob Aldred for the terminal encoder
|
||||||
|
* Trans for pointing out $DEBUG dependencies
|
||||||
|
* Flameeyes for finding that Term::ANSIColor was obsolete
|
||||||
* matz and all Ruby gods and gurus
|
* matz and all Ruby gods and gurus
|
||||||
* The inventors of: the computer, the internet, the true color display, HTML &
|
* The inventors of: the computer, the internet, the true color display, HTML &
|
||||||
CSS, VIM, RUBY, pizza, microwaves, guitars, scouting, programming, anime,
|
CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime,
|
||||||
manga, coke and green ice tea.
|
manga, coke and green ice tea.
|
||||||
|
|
||||||
Where would we be without all those people?
|
Where would we be without all those people?
|
||||||
|
@ -103,23 +104,27 @@ Where would we be without all those people?
|
||||||
=== Created using
|
=== Created using
|
||||||
|
|
||||||
* Ruby[http://ruby-lang.org/]
|
* Ruby[http://ruby-lang.org/]
|
||||||
* Chihiro (my Sony VAIO laptop), Henrietta (my new MacBook) and
|
* Chihiro (my Sony VAIO laptop); Henrietta (my old MacBook);
|
||||||
Seras (my Athlon 2200+ tower)
|
Triella, born Rico (my new MacBook); as well as
|
||||||
* VIM[http://vim.org] and TextMate[http://macromates.com]
|
Seras and Hikari (my PCs)
|
||||||
* RDE[http://homepage2.nifty.com/sakazuki/rde_e.html]
|
* RDE[http://homepage2.nifty.com/sakazuki/rde_e.html],
|
||||||
* Microsoft Windows (yes, I confess!) and MacOS X
|
VIM[http://vim.org] and TextMate[http://macromates.com]
|
||||||
* Firefox[http://www.mozilla.org/products/firefox/] and
|
* Subversion[http://subversion.tigris.org/]
|
||||||
|
* Redmine[http://redmine.org/]
|
||||||
|
* Firefox[http://www.mozilla.org/products/firefox/],
|
||||||
|
Firebug[http://getfirebug.com/], Safari[http://www.apple.com/safari/], and
|
||||||
Thunderbird[http://www.mozilla.org/products/thunderbird/]
|
Thunderbird[http://www.mozilla.org/products/thunderbird/]
|
||||||
* Rake[http://rake.rubyforge.org/]
|
* RubyGems[http://docs.rubygems.org/] and Rake[http://rake.rubyforge.org/]
|
||||||
* RubyGems[http://docs.rubygems.org/]
|
* TortoiseSVN[http://tortoisesvn.tigris.org/] using Apache via
|
||||||
* {Subversion/TortoiseSVN}[http://tortoisesvn.tigris.org/] using Apache via
|
|
||||||
XAMPP[http://www.apachefriends.org/en/xampp.html]
|
XAMPP[http://www.apachefriends.org/en/xampp.html]
|
||||||
* RDoc (though I'm quite unsatisfied with it)
|
* RDoc (though I'm quite unsatisfied with it)
|
||||||
|
* Microsoft Windows (yes, I confess!) and MacOS X
|
||||||
* GNUWin32, MinGW and some other tools to make the shell under windows a bit
|
* GNUWin32, MinGW and some other tools to make the shell under windows a bit
|
||||||
more useful
|
less useless
|
||||||
* Term::ANSIColor[http://term-ansicolor.rubyforge.org/]
|
* Term::ANSIColor[http://term-ansicolor.rubyforge.org/]
|
||||||
|
* PLEAC[http://pleac.sourceforge.net/] code examples
|
||||||
|
|
||||||
---
|
=== Free
|
||||||
|
|
||||||
* As you can see, CodeRay was created under heavy use of *free* software.
|
* As you can see, CodeRay was created under heavy use of *free* software.
|
||||||
* So CodeRay is also *free*.
|
* So CodeRay is also *free*.
|
|
@ -1,7 +1,5 @@
|
||||||
# = CodeRay Library
|
# = CodeRay Library
|
||||||
#
|
#
|
||||||
# $Id: coderay.rb 227 2007-04-24 12:26:18Z murphy $
|
|
||||||
#
|
|
||||||
# CodeRay is a Ruby library for syntax highlighting.
|
# CodeRay is a Ruby library for syntax highlighting.
|
||||||
#
|
#
|
||||||
# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
|
# I try to make CodeRay easy to use and intuitive, but at the same time fully featured, complete,
|
||||||
|
@ -16,7 +14,7 @@
|
||||||
#
|
#
|
||||||
# Here's a fancy graphic to light up this gray docu:
|
# Here's a fancy graphic to light up this gray docu:
|
||||||
#
|
#
|
||||||
# http://rd.cYcnus.de/coderay/scheme.png
|
# http://cycnus.de/raindark/coderay/scheme.png
|
||||||
#
|
#
|
||||||
# == Documentation
|
# == Documentation
|
||||||
#
|
#
|
||||||
|
@ -107,7 +105,7 @@
|
||||||
#
|
#
|
||||||
# CodeRay.scan_stream:: Scan in stream mode.
|
# CodeRay.scan_stream:: Scan in stream mode.
|
||||||
#
|
#
|
||||||
# == All-in-One Encoding
|
# == All-in-One Encoding
|
||||||
#
|
#
|
||||||
# CodeRay.encode:: Highlight a string with a given input and output format.
|
# CodeRay.encode:: Highlight a string with a given input and output format.
|
||||||
#
|
#
|
||||||
|
@ -128,15 +126,18 @@
|
||||||
# If you want to re-use scanners and encoders (because that is faster), see
|
# If you want to re-use scanners and encoders (because that is faster), see
|
||||||
# CodeRay::Duo for the most convenient (and recommended) interface.
|
# CodeRay::Duo for the most convenient (and recommended) interface.
|
||||||
module CodeRay
|
module CodeRay
|
||||||
|
|
||||||
|
$CODERAY_DEBUG ||= false
|
||||||
|
|
||||||
# Version: Major.Minor.Teeny[.Revision]
|
# Version: Major.Minor.Teeny[.Revision]
|
||||||
# Major: 0 for pre-release
|
# Major: 0 for pre-stable, 1 for stable
|
||||||
# Minor: odd for beta, even for stable
|
# Minor: feature milestone
|
||||||
# Teeny: development state
|
# Teeny: development state, 0 for pre-release
|
||||||
# Revision: Subversion Revision number (generated on rake)
|
# Revision: Subversion Revision number (generated on rake gem:make)
|
||||||
VERSION = '0.7.6'
|
VERSION = '0.9.2'
|
||||||
|
|
||||||
require 'coderay/tokens'
|
require 'coderay/tokens'
|
||||||
|
require 'coderay/token_classes'
|
||||||
require 'coderay/scanner'
|
require 'coderay/scanner'
|
||||||
require 'coderay/encoder'
|
require 'coderay/encoder'
|
||||||
require 'coderay/duo'
|
require 'coderay/duo'
|
||||||
|
@ -315,6 +316,7 @@ end
|
||||||
# Run a test script.
|
# Run a test script.
|
||||||
if $0 == __FILE__
|
if $0 == __FILE__
|
||||||
$stderr.print 'Press key to print demo.'; gets
|
$stderr.print 'Press key to print demo.'; gets
|
||||||
|
# Just use this file as an example of Ruby code.
|
||||||
code = File.read(__FILE__)[/module CodeRay.*/m]
|
code = File.read(__FILE__)[/module CodeRay.*/m]
|
||||||
print CodeRay.scan(code, :ruby).html
|
print CodeRay.scan(code, :ruby).html
|
||||||
end
|
end
|
|
@ -2,8 +2,6 @@ module CodeRay
|
||||||
|
|
||||||
# = Duo
|
# = Duo
|
||||||
#
|
#
|
||||||
# $Id: scanner.rb 123 2006-03-21 14:46:34Z murphy $
|
|
||||||
#
|
|
||||||
# A Duo is a convenient way to use CodeRay. You just create a Duo,
|
# A Duo is a convenient way to use CodeRay. You just create a Duo,
|
||||||
# giving it a lang (language of the input code) and a format (desired
|
# giving it a lang (language of the input code) and a format (desired
|
||||||
# output format), and call Duo#highlight with the code.
|
# output format), and call Duo#highlight with the code.
|
|
@ -1,5 +1,3 @@
|
||||||
require "stringio"
|
|
||||||
|
|
||||||
module CodeRay
|
module CodeRay
|
||||||
|
|
||||||
# This module holds the Encoder class and its subclasses.
|
# This module holds the Encoder class and its subclasses.
|
||||||
|
@ -126,36 +124,66 @@ module CodeRay
|
||||||
@out = ''
|
@out = ''
|
||||||
end
|
end
|
||||||
|
|
||||||
# Called with +text+ and +kind+ of the currently scanned token.
|
# Called with +content+ and +kind+ of the currently scanned token.
|
||||||
# For simple scanners, it's enougth to implement this method.
|
# For simple scanners, it's enougth to implement this method.
|
||||||
#
|
#
|
||||||
# By default, it calls text_token or block_token, depending on
|
# By default, it calls text_token or block_token, depending on
|
||||||
# whether +text+ is a String.
|
# whether +content+ is a String.
|
||||||
def token text, kind
|
def token content, kind
|
||||||
out =
|
encoded_token =
|
||||||
if text.is_a? ::String # Ruby 1.9: :open.is_a? String
|
if content.is_a? ::String
|
||||||
text_token text, kind
|
text_token content, kind
|
||||||
elsif text.is_a? ::Symbol
|
elsif content.is_a? ::Symbol
|
||||||
block_token text, kind
|
block_token content, kind
|
||||||
else
|
else
|
||||||
raise 'Unknown token text type: %p' % text
|
raise 'Unknown token content type: %p' % [content]
|
||||||
end
|
end
|
||||||
@out << out if @out
|
append_encoded_token_to_output encoded_token
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def append_encoded_token_to_output encoded_token
|
||||||
|
@out << encoded_token if encoded_token && defined?(@out) && @out
|
||||||
|
end
|
||||||
|
|
||||||
|
# Called for each text token ([text, kind]), where text is a String.
|
||||||
def text_token text, kind
|
def text_token text, kind
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Called for each block (non-text) token ([action, kind]),
|
||||||
|
# where +action+ is a Symbol.
|
||||||
|
#
|
||||||
|
# Calls open_token, close_token, begin_line, and end_line according to
|
||||||
|
# the value of +action+.
|
||||||
def block_token action, kind
|
def block_token action, kind
|
||||||
case action
|
case action
|
||||||
when :open
|
when :open
|
||||||
open_token kind
|
open_token kind
|
||||||
when :close
|
when :close
|
||||||
close_token kind
|
close_token kind
|
||||||
|
when :begin_line
|
||||||
|
begin_line kind
|
||||||
|
when :end_line
|
||||||
|
end_line kind
|
||||||
else
|
else
|
||||||
raise 'unknown block action: %p' % action
|
raise 'unknown block action: %p' % action
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Called for each block token at the start of the block ([:open, kind]).
|
||||||
|
def open_token kind
|
||||||
|
end
|
||||||
|
|
||||||
|
# Called for each block token end of the block ([:close, kind]).
|
||||||
|
def close_token kind
|
||||||
|
end
|
||||||
|
|
||||||
|
# Called for each line token block at the start of the line ([:begin_line, kind]).
|
||||||
|
def begin_line kind
|
||||||
|
end
|
||||||
|
|
||||||
|
# Called for each line token block at the end of the line ([:end_line, kind]).
|
||||||
|
def end_line kind
|
||||||
|
end
|
||||||
|
|
||||||
# Called with merged options after encoding starts.
|
# Called with merged options after encoding starts.
|
||||||
# The return value is the result of encoding, typically @out.
|
# The return value is the result of encoding, typically @out.
|
||||||
|
@ -167,8 +195,16 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# The already created +tokens+ object must be used; it can be a
|
# The already created +tokens+ object must be used; it can be a
|
||||||
# TokenStream or a Tokens object.
|
# TokenStream or a Tokens object.
|
||||||
def compile tokens, options
|
if RUBY_VERSION >= '1.9'
|
||||||
tokens.each(&self)
|
def compile tokens, options
|
||||||
|
for text, kind in tokens
|
||||||
|
token text, kind
|
||||||
|
end
|
||||||
|
end
|
||||||
|
else
|
||||||
|
def compile tokens, options
|
||||||
|
tokens.each(&self)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
|
@ -0,0 +1,12 @@
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
|
||||||
|
map \
|
||||||
|
:loc => :lines_of_code,
|
||||||
|
:plain => :text,
|
||||||
|
:stats => :statistic,
|
||||||
|
:terminal => :term,
|
||||||
|
:tex => :latex
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,43 @@
|
||||||
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
|
||||||
|
load :token_class_filter
|
||||||
|
|
||||||
|
class CommentFilter < TokenClassFilter
|
||||||
|
|
||||||
|
register_for :comment_filter
|
||||||
|
|
||||||
|
DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
|
||||||
|
:exclude => [:comment]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if $0 == __FILE__
|
||||||
|
$VERBOSE = true
|
||||||
|
$: << File.join(File.dirname(__FILE__), '..')
|
||||||
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
require 'test/unit'
|
||||||
|
|
||||||
|
class CommentFilterTest < Test::Unit::TestCase
|
||||||
|
|
||||||
|
def test_filtering_comments
|
||||||
|
tokens = CodeRay.scan <<-RUBY, :ruby
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
# a minimal Ruby program
|
||||||
|
puts "Hello world!"
|
||||||
|
RUBY
|
||||||
|
assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
puts "Hello world!"
|
||||||
|
RUBY_FILTERED
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
|
@ -35,6 +35,14 @@ module Encoders
|
||||||
">"
|
">"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def begin_line kind
|
||||||
|
"#{kind}["
|
||||||
|
end
|
||||||
|
|
||||||
|
def end_line kind
|
||||||
|
"]"
|
||||||
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
|
@ -9,10 +9,9 @@ module Encoders
|
||||||
|
|
||||||
register_for :div
|
register_for :div
|
||||||
|
|
||||||
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
|
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
|
||||||
:css => :style,
|
:css => :style,
|
||||||
:wrap => :div,
|
:wrap => :div
|
||||||
})
|
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,75 @@
|
||||||
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
|
||||||
|
class Filter < Encoder
|
||||||
|
|
||||||
|
register_for :filter
|
||||||
|
|
||||||
|
protected
|
||||||
|
def setup options
|
||||||
|
@out = Tokens.new
|
||||||
|
end
|
||||||
|
|
||||||
|
def text_token text, kind
|
||||||
|
[text, kind] if include_text_token? text, kind
|
||||||
|
end
|
||||||
|
|
||||||
|
def include_text_token? text, kind
|
||||||
|
true
|
||||||
|
end
|
||||||
|
|
||||||
|
def block_token action, kind
|
||||||
|
[action, kind] if include_block_token? action, kind
|
||||||
|
end
|
||||||
|
|
||||||
|
def include_block_token? action, kind
|
||||||
|
true
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if $0 == __FILE__
|
||||||
|
$VERBOSE = true
|
||||||
|
$: << File.join(File.dirname(__FILE__), '..')
|
||||||
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
require 'test/unit'
|
||||||
|
|
||||||
|
class FilterTest < Test::Unit::TestCase
|
||||||
|
|
||||||
|
def test_creation
|
||||||
|
assert CodeRay::Encoders::Filter < CodeRay::Encoders::Encoder
|
||||||
|
filter = nil
|
||||||
|
assert_nothing_raised do
|
||||||
|
filter = CodeRay.encoder :filter
|
||||||
|
end
|
||||||
|
assert_kind_of CodeRay::Encoders::Encoder, filter
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_filtering_text_tokens
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
10.times do |i|
|
||||||
|
tokens << [i.to_s, :index]
|
||||||
|
end
|
||||||
|
assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
|
||||||
|
assert_equal tokens, tokens.filter
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_filtering_block_tokens
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
10.times do |i|
|
||||||
|
tokens << [:open, :index]
|
||||||
|
tokens << [i.to_s, :content]
|
||||||
|
tokens << [:close, :index]
|
||||||
|
end
|
||||||
|
assert_equal tokens, CodeRay::Encoders::Filter.new.encode_tokens(tokens)
|
||||||
|
assert_equal tokens, tokens.filter
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
|
@ -1,4 +1,4 @@
|
||||||
require "set"
|
require 'set'
|
||||||
|
|
||||||
module CodeRay
|
module CodeRay
|
||||||
module Encoders
|
module Encoders
|
||||||
|
@ -25,10 +25,6 @@ module Encoders
|
||||||
#
|
#
|
||||||
# == Options
|
# == Options
|
||||||
#
|
#
|
||||||
# === :escape
|
|
||||||
# Escape html entities
|
|
||||||
# Default: true
|
|
||||||
#
|
|
||||||
# === :tab_width
|
# === :tab_width
|
||||||
# Convert \t characters to +n+ spaces (a number.)
|
# Convert \t characters to +n+ spaces (a number.)
|
||||||
# Default: 8
|
# Default: 8
|
||||||
|
@ -45,6 +41,12 @@ module Encoders
|
||||||
#
|
#
|
||||||
# Default: nil
|
# Default: nil
|
||||||
#
|
#
|
||||||
|
# === :title
|
||||||
|
#
|
||||||
|
# The title of the HTML page (works only when :wrap is set to :page.)
|
||||||
|
#
|
||||||
|
# Default: 'CodeRay output'
|
||||||
|
#
|
||||||
# === :line_numbers
|
# === :line_numbers
|
||||||
# Include line numbers in :table, :inline, :list or nil (no line numbers)
|
# Include line numbers in :table, :inline, :list or nil (no line numbers)
|
||||||
#
|
#
|
||||||
|
@ -60,6 +62,16 @@ module Encoders
|
||||||
#
|
#
|
||||||
# Default: 10
|
# Default: 10
|
||||||
#
|
#
|
||||||
|
# === :highlight_lines
|
||||||
|
#
|
||||||
|
# Highlights certain line numbers.
|
||||||
|
# Can be any Enumerable, typically just an Array or Range, of numbers.
|
||||||
|
#
|
||||||
|
# Bolding is deactivated when :highlight_lines is set. It only makes sense
|
||||||
|
# in combination with :line_numbers.
|
||||||
|
#
|
||||||
|
# Default: nil
|
||||||
|
#
|
||||||
# === :hint
|
# === :hint
|
||||||
# Include some information into the output using the title attribute.
|
# Include some information into the output using the title attribute.
|
||||||
# Can be :info (show token type on mouse-over), :info_long (with full path)
|
# Can be :info (show token type on mouse-over), :info_long (with full path)
|
||||||
|
@ -74,19 +86,18 @@ module Encoders
|
||||||
FILE_EXTENSION = 'html'
|
FILE_EXTENSION = 'html'
|
||||||
|
|
||||||
DEFAULT_OPTIONS = {
|
DEFAULT_OPTIONS = {
|
||||||
:escape => true,
|
|
||||||
:tab_width => 8,
|
:tab_width => 8,
|
||||||
|
|
||||||
:level => :xhtml,
|
|
||||||
:css => :class,
|
:css => :class,
|
||||||
|
|
||||||
:style => :cycnus,
|
:style => :cycnus,
|
||||||
|
|
||||||
:wrap => nil,
|
:wrap => nil,
|
||||||
|
:title => 'CodeRay output',
|
||||||
|
|
||||||
:line_numbers => nil,
|
:line_numbers => nil,
|
||||||
:line_number_start => 1,
|
:line_number_start => 1,
|
||||||
:bold_every => 10,
|
:bold_every => 10,
|
||||||
|
:highlight_lines => nil,
|
||||||
|
|
||||||
:hint => false,
|
:hint => false,
|
||||||
}
|
}
|
||||||
|
@ -141,7 +152,7 @@ module Encoders
|
||||||
when :debug
|
when :debug
|
||||||
classes.inspect
|
classes.inspect
|
||||||
end
|
end
|
||||||
" title=\"#{title}\""
|
title ? " title=\"#{title}\"" : ''
|
||||||
end
|
end
|
||||||
|
|
||||||
def setup options
|
def setup options
|
||||||
|
@ -150,7 +161,6 @@ module Encoders
|
||||||
@HTML_ESCAPE = HTML_ESCAPE.dup
|
@HTML_ESCAPE = HTML_ESCAPE.dup
|
||||||
@HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
|
@HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
|
||||||
|
|
||||||
@escape = options[:escape]
|
|
||||||
@opened = [nil]
|
@opened = [nil]
|
||||||
@css = CSS.new options[:style]
|
@css = CSS.new options[:style]
|
||||||
|
|
||||||
|
@ -164,7 +174,7 @@ module Encoders
|
||||||
|
|
||||||
when :class
|
when :class
|
||||||
@css_style = Hash.new do |h, k|
|
@css_style = Hash.new do |h, k|
|
||||||
c = Tokens::ClassOfKind[k.first]
|
c = CodeRay::Tokens::ClassOfKind[k.first]
|
||||||
if c == :NO_HIGHLIGHT and not hint
|
if c == :NO_HIGHLIGHT and not hint
|
||||||
h[k.dup] = false
|
h[k.dup] = false
|
||||||
else
|
else
|
||||||
|
@ -222,43 +232,70 @@ module Encoders
|
||||||
@out.css = @css
|
@out.css = @css
|
||||||
@out.numerize! options[:line_numbers], options
|
@out.numerize! options[:line_numbers], options
|
||||||
@out.wrap! options[:wrap]
|
@out.wrap! options[:wrap]
|
||||||
|
@out.apply_title! options[:title]
|
||||||
|
|
||||||
super
|
super
|
||||||
end
|
end
|
||||||
|
|
||||||
def token text, type
|
def token text, type = :plain
|
||||||
if text.is_a? ::String
|
case text
|
||||||
if @escape && (text =~ /#{HTML_ESCAPE_PATTERN}/o)
|
|
||||||
|
when nil
|
||||||
|
# raise 'Token with nil as text was given: %p' % [[text, type]]
|
||||||
|
|
||||||
|
when String
|
||||||
|
if text =~ /#{HTML_ESCAPE_PATTERN}/o
|
||||||
text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
|
text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
|
||||||
end
|
end
|
||||||
@opened[0] = type
|
@opened[0] = type
|
||||||
if style = @css_style[@opened]
|
if text != "\n" && style = @css_style[@opened]
|
||||||
@out << style << text << '</span>'
|
@out << style << text << '</span>'
|
||||||
else
|
else
|
||||||
@out << text
|
@out << text
|
||||||
end
|
end
|
||||||
else
|
|
||||||
case text
|
|
||||||
when :open
|
# token groups, eg. strings
|
||||||
@opened[0] = type
|
when :open
|
||||||
@out << (@css_style[@opened] || '<span>')
|
@opened[0] = type
|
||||||
@opened << type
|
@out << (@css_style[@opened] || '<span>')
|
||||||
when :close
|
@opened << type
|
||||||
if @opened.empty?
|
when :close
|
||||||
# nothing to close
|
if @opened.empty?
|
||||||
else
|
# nothing to close
|
||||||
if $DEBUG and (@opened.size == 1 or @opened.last != type)
|
|
||||||
raise 'Malformed token stream: Trying to close a token (%p) \
|
|
||||||
that is not open. Open are: %p.' % [type, @opened[1..-1]]
|
|
||||||
end
|
|
||||||
@out << '</span>'
|
|
||||||
@opened.pop
|
|
||||||
end
|
|
||||||
when nil
|
|
||||||
raise 'Token with nil as text was given: %p' % [[text, type]]
|
|
||||||
else
|
else
|
||||||
raise 'unknown token kind: %p' % text
|
if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
|
||||||
|
raise 'Malformed token stream: Trying to close a token (%p) \
|
||||||
|
that is not open. Open are: %p.' % [type, @opened[1..-1]]
|
||||||
|
end
|
||||||
|
@out << '</span>'
|
||||||
|
@opened.pop
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# whole lines to be highlighted, eg. a deleted line in a diff
|
||||||
|
when :begin_line
|
||||||
|
@opened[0] = type
|
||||||
|
if style = @css_style[@opened]
|
||||||
|
@out << style.sub('<span', '<div')
|
||||||
|
else
|
||||||
|
@out << '<div>'
|
||||||
|
end
|
||||||
|
@opened << type
|
||||||
|
when :end_line
|
||||||
|
if @opened.empty?
|
||||||
|
# nothing to close
|
||||||
|
else
|
||||||
|
if $CODERAY_DEBUG and (@opened.size == 1 or @opened.last != type)
|
||||||
|
raise 'Malformed token stream: Trying to close a line (%p) \
|
||||||
|
that is not open. Open are: %p.' % [type, @opened[1..-1]]
|
||||||
|
end
|
||||||
|
@out << '</div>'
|
||||||
|
@opened.pop
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise 'unknown token kind: %p' % [text]
|
||||||
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -27,16 +27,19 @@ module Encoders
|
||||||
1.upto(styles.size) do |offset|
|
1.upto(styles.size) do |offset|
|
||||||
break if style = cl[styles[offset .. -1]]
|
break if style = cl[styles[offset .. -1]]
|
||||||
end
|
end
|
||||||
raise 'Style not found: %p' % [styles] if $DEBUG and style.empty?
|
# warn 'Style not found: %p' % [styles] if style.empty?
|
||||||
return style
|
return style
|
||||||
end
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
|
|
||||||
CSS_CLASS_PATTERN = /
|
CSS_CLASS_PATTERN = /
|
||||||
( (?: # $1 = classes
|
( # $1 = selectors
|
||||||
\s* \. [-\w]+
|
(?:
|
||||||
)+ )
|
(?: \s* \. [-\w]+ )+
|
||||||
|
\s* ,?
|
||||||
|
)+
|
||||||
|
)
|
||||||
\s* \{ \s*
|
\s* \{ \s*
|
||||||
( [^\}]+ )? # $2 = style
|
( [^\}]+ )? # $2 = style
|
||||||
\s* \} \s*
|
\s* \} \s*
|
||||||
|
@ -44,12 +47,14 @@ module Encoders
|
||||||
( . ) # $3 = error
|
( . ) # $3 = error
|
||||||
/mx
|
/mx
|
||||||
def parse stylesheet
|
def parse stylesheet
|
||||||
stylesheet.scan CSS_CLASS_PATTERN do |classes, style, error|
|
stylesheet.scan CSS_CLASS_PATTERN do |selectors, style, error|
|
||||||
raise "CSS parse error: '#{error.inspect}' not recognized" if error
|
raise "CSS parse error: '#{error.inspect}' not recognized" if error
|
||||||
styles = classes.scan(/[-\w]+/)
|
for selector in selectors.split(',')
|
||||||
cl = styles.pop
|
classes = selector.scan(/[-\w]+/)
|
||||||
@classes[cl] ||= Hash.new
|
cl = classes.pop
|
||||||
@classes[cl][styles] = style.to_s.strip
|
@classes[cl] ||= Hash.new
|
||||||
|
@classes[cl][classes] = style.to_s.strip.delete(' ').chomp(';')
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -32,9 +32,19 @@ module Encoders
|
||||||
#end
|
#end
|
||||||
|
|
||||||
bold_every = options[:bold_every]
|
bold_every = options[:bold_every]
|
||||||
|
highlight_lines = options[:highlight_lines]
|
||||||
bolding =
|
bolding =
|
||||||
if bold_every == false
|
if bold_every == false && highlight_lines == nil
|
||||||
proc { |line| line.to_s }
|
proc { |line| line.to_s }
|
||||||
|
elsif highlight_lines.is_a? Enumerable
|
||||||
|
highlight_lines = highlight_lines.to_set
|
||||||
|
proc do |line|
|
||||||
|
if highlight_lines.include? line
|
||||||
|
"<strong class=\"highlighted\">#{line}</strong>" # highlighted line numbers in bold
|
||||||
|
else
|
||||||
|
line.to_s
|
||||||
|
end
|
||||||
|
end
|
||||||
elsif bold_every.is_a? Integer
|
elsif bold_every.is_a? Integer
|
||||||
raise ArgumentError, ":bolding can't be 0." if bold_every == 0
|
raise ArgumentError, ":bolding can't be 0." if bold_every == 0
|
||||||
proc do |line|
|
proc do |line|
|
||||||
|
@ -51,12 +61,12 @@ module Encoders
|
||||||
case mode
|
case mode
|
||||||
when :inline
|
when :inline
|
||||||
max_width = (start + line_count).to_s.size
|
max_width = (start + line_count).to_s.size
|
||||||
line = start
|
line_number = start
|
||||||
gsub!(/^/) do
|
gsub!(/^/) do
|
||||||
line_number = bolding.call line
|
line_number_text = bolding.call line_number
|
||||||
indent = ' ' * (max_width - line.to_s.size)
|
indent = ' ' * (max_width - line_number.to_s.size) # TODO: Optimize (10^x)
|
||||||
res = "<span class=\"no\">#{indent}#{line_number}</span> "
|
res = "<span class=\"no\">#{indent}#{line_number_text}</span> "
|
||||||
line += 1
|
line_number += 1
|
||||||
res
|
res
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -65,12 +75,12 @@ module Encoders
|
||||||
# Because even monospace fonts seem to have different heights when bold,
|
# Because even monospace fonts seem to have different heights when bold,
|
||||||
# I make the newline bold, both in the code and the line numbers.
|
# I make the newline bold, both in the code and the line numbers.
|
||||||
# FIXME Still not working perfect for Mr. Internet Exploder
|
# FIXME Still not working perfect for Mr. Internet Exploder
|
||||||
# FIXME Firefox struggles with very long codes (> 200 lines)
|
|
||||||
line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")
|
line_numbers = (start ... start + line_count).to_a.map(&bolding).join("\n")
|
||||||
line_numbers << "\n" # also for Mr. MS Internet Exploder :-/
|
line_numbers << "\n" # also for Mr. MS Internet Exploder :-/
|
||||||
line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
|
line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
|
||||||
|
|
||||||
line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
|
line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
|
||||||
|
gsub!(/<\/div>\n/) { '</div>' }
|
||||||
gsub!(/\n/) { "<tt>\n</tt>" }
|
gsub!(/\n/) { "<tt>\n</tt>" }
|
||||||
wrap_in! line_numbers_table_tpl
|
wrap_in! line_numbers_table_tpl
|
||||||
@wrapped_in = :div
|
@wrapped_in = :div
|
||||||
|
@ -90,8 +100,9 @@ module Encoders
|
||||||
end
|
end
|
||||||
close = '</span>' * opened_tags.size
|
close = '</span>' * opened_tags.size
|
||||||
|
|
||||||
"<li>#{open}#{line}#{close}</li>"
|
"<li>#{open}#{line}#{close}</li>\n"
|
||||||
end
|
end
|
||||||
|
chomp!("\n")
|
||||||
wrap_in! LIST
|
wrap_in! LIST
|
||||||
@wrapped_in = :div
|
@wrapped_in = :div
|
||||||
|
|
|
@ -86,6 +86,11 @@ module Encoders
|
||||||
Template.wrap! self, template, 'CONTENT'
|
Template.wrap! self, template, 'CONTENT'
|
||||||
self
|
self
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def apply_title! title
|
||||||
|
self.sub!(/(<title>)(<\/title>)/) { $1 + title + $2 }
|
||||||
|
self
|
||||||
|
end
|
||||||
|
|
||||||
def wrap! element, *args
|
def wrap! element, *args
|
||||||
return self if not element or element == wrapped_in
|
return self if not element or element == wrapped_in
|
||||||
|
@ -100,6 +105,10 @@ module Encoders
|
||||||
wrap! :div if wrapped_in? nil
|
wrap! :div if wrapped_in? nil
|
||||||
raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
|
raise "Can't wrap %p in %p" % [wrapped_in, element] unless wrapped_in? :div
|
||||||
wrap_in! Output.page_template_for_css(@css)
|
wrap_in! Output.page_template_for_css(@css)
|
||||||
|
if args.first.is_a?(Hash) && title = args.first[:title]
|
||||||
|
apply_title! title
|
||||||
|
end
|
||||||
|
self
|
||||||
when nil
|
when nil
|
||||||
return self
|
return self
|
||||||
else
|
else
|
||||||
|
@ -166,7 +175,9 @@ module Encoders
|
||||||
# title="double click to expand"
|
# title="double click to expand"
|
||||||
|
|
||||||
LIST = <<-`LIST`
|
LIST = <<-`LIST`
|
||||||
<ol class="CodeRay"><%CONTENT%></ol>
|
<ol class="CodeRay">
|
||||||
|
<%CONTENT%>
|
||||||
|
</ol>
|
||||||
LIST
|
LIST
|
||||||
|
|
||||||
PAGE = <<-`PAGE`
|
PAGE = <<-`PAGE`
|
||||||
|
@ -175,7 +186,7 @@ module Encoders
|
||||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
|
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="de">
|
||||||
<head>
|
<head>
|
||||||
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
|
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
|
||||||
<title>CodeRay HTML Encoder Example</title>
|
<title></title>
|
||||||
<style type="text/css">
|
<style type="text/css">
|
||||||
<%CSS%>
|
<%CSS%>
|
||||||
</style>
|
</style>
|
|
@ -0,0 +1,69 @@
|
||||||
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
|
||||||
|
# = JSON Encoder
|
||||||
|
class JSON < Encoder
|
||||||
|
|
||||||
|
register_for :json
|
||||||
|
FILE_EXTENSION = 'json'
|
||||||
|
|
||||||
|
protected
|
||||||
|
def setup options
|
||||||
|
begin
|
||||||
|
require 'json'
|
||||||
|
rescue LoadError
|
||||||
|
require 'rubygems'
|
||||||
|
require 'json'
|
||||||
|
end
|
||||||
|
@out = []
|
||||||
|
end
|
||||||
|
|
||||||
|
def text_token text, kind
|
||||||
|
{ :type => 'text', :text => text, :kind => kind }
|
||||||
|
end
|
||||||
|
|
||||||
|
def block_token action, kind
|
||||||
|
{ :type => 'block', :action => action, :kind => kind }
|
||||||
|
end
|
||||||
|
|
||||||
|
def finish options
|
||||||
|
@out.to_json
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if $0 == __FILE__
|
||||||
|
$VERBOSE = true
|
||||||
|
$: << File.join(File.dirname(__FILE__), '..')
|
||||||
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
require 'test/unit'
|
||||||
|
$:.delete '.'
|
||||||
|
require 'rubygems' if RUBY_VERSION < '1.9'
|
||||||
|
|
||||||
|
class JSONEncoderTest < Test::Unit::TestCase
|
||||||
|
|
||||||
|
def test_json_output
|
||||||
|
tokens = CodeRay.scan <<-RUBY, :ruby
|
||||||
|
puts "Hello world!"
|
||||||
|
RUBY
|
||||||
|
require 'json'
|
||||||
|
assert_equal [
|
||||||
|
{"type"=>"text", "text"=>"puts", "kind"=>"ident"},
|
||||||
|
{"type"=>"text", "text"=>" ", "kind"=>"space"},
|
||||||
|
{"type"=>"block", "action"=>"open", "kind"=>"string"},
|
||||||
|
{"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
|
||||||
|
{"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
|
||||||
|
{"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
|
||||||
|
{"type"=>"block", "action"=>"close", "kind"=>"string"},
|
||||||
|
{"type"=>"text", "text"=>"\n", "kind"=>"space"}
|
||||||
|
], JSON.load(tokens.json)
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
|
@ -0,0 +1,90 @@
|
||||||
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
|
||||||
|
# Counts the LoC (Lines of Code). Returns an Integer >= 0.
|
||||||
|
#
|
||||||
|
# Alias: :loc
|
||||||
|
#
|
||||||
|
# Everything that is not comment, markup, doctype/shebang, or an empty line,
|
||||||
|
# is considered to be code.
|
||||||
|
#
|
||||||
|
# For example,
|
||||||
|
# * HTML files not containing JavaScript have 0 LoC
|
||||||
|
# * in a Java class without comments, LoC is the number of non-empty lines
|
||||||
|
#
|
||||||
|
# A Scanner class should define the token kinds that are not code in the
|
||||||
|
# KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
|
||||||
|
class LinesOfCode < Encoder
|
||||||
|
|
||||||
|
register_for :lines_of_code
|
||||||
|
|
||||||
|
NON_EMPTY_LINE = /^\s*\S.*$/
|
||||||
|
|
||||||
|
def compile tokens, options
|
||||||
|
if scanner = tokens.scanner
|
||||||
|
kinds_not_loc = scanner.class::KINDS_NOT_LOC
|
||||||
|
else
|
||||||
|
warn ArgumentError, 'Tokens have no scanner.' if $VERBOSE
|
||||||
|
kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
|
||||||
|
end
|
||||||
|
code = tokens.token_class_filter :exclude => kinds_not_loc
|
||||||
|
@loc = code.text.scan(NON_EMPTY_LINE).size
|
||||||
|
end
|
||||||
|
|
||||||
|
def finish options
|
||||||
|
@loc
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if $0 == __FILE__
|
||||||
|
$VERBOSE = true
|
||||||
|
$: << File.join(File.dirname(__FILE__), '..')
|
||||||
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
require 'test/unit'
|
||||||
|
|
||||||
|
class LinesOfCodeTest < Test::Unit::TestCase
|
||||||
|
|
||||||
|
def test_creation
|
||||||
|
assert CodeRay::Encoders::LinesOfCode < CodeRay::Encoders::Encoder
|
||||||
|
filter = nil
|
||||||
|
assert_nothing_raised do
|
||||||
|
filter = CodeRay.encoder :loc
|
||||||
|
end
|
||||||
|
assert_kind_of CodeRay::Encoders::LinesOfCode, filter
|
||||||
|
assert_nothing_raised do
|
||||||
|
filter = CodeRay.encoder :lines_of_code
|
||||||
|
end
|
||||||
|
assert_kind_of CodeRay::Encoders::LinesOfCode, filter
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_lines_of_code
|
||||||
|
tokens = CodeRay.scan <<-RUBY, :ruby
|
||||||
|
#!/usr/bin/env ruby
|
||||||
|
|
||||||
|
# a minimal Ruby program
|
||||||
|
puts "Hello world!"
|
||||||
|
RUBY
|
||||||
|
assert_equal 1, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
|
||||||
|
assert_equal 1, tokens.lines_of_code
|
||||||
|
assert_equal 1, tokens.loc
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_filtering_block_tokens
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
tokens << ["Hello\n", :world]
|
||||||
|
tokens << ["Hello\n", :space]
|
||||||
|
tokens << ["Hello\n", :comment]
|
||||||
|
assert_equal 2, CodeRay::Encoders::LinesOfCode.new.encode_tokens(tokens)
|
||||||
|
assert_equal 2, tokens.lines_of_code
|
||||||
|
assert_equal 2, tokens.loc
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
|
@ -9,11 +9,10 @@ module Encoders
|
||||||
|
|
||||||
register_for :page
|
register_for :page
|
||||||
|
|
||||||
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
|
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
|
||||||
:css => :class,
|
:css => :class,
|
||||||
:wrap => :page,
|
:wrap => :page,
|
||||||
:line_numbers => :table
|
:line_numbers => :table
|
||||||
})
|
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -9,10 +9,9 @@ module Encoders
|
||||||
|
|
||||||
register_for :span
|
register_for :span
|
||||||
|
|
||||||
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
|
DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
|
||||||
:css => :style,
|
:css => :style,
|
||||||
:wrap => :span,
|
:wrap => :span
|
||||||
})
|
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,158 @@
|
||||||
|
# encoders/term.rb
|
||||||
|
# By Rob Aldred (http://robaldred.co.uk)
|
||||||
|
# Based on idea by Nathan Weizenbaum (http://nex-3.com)
|
||||||
|
# MIT License (http://www.opensource.org/licenses/mit-license.php)
|
||||||
|
#
|
||||||
|
# A CodeRay encoder that outputs code highlighted for a color terminal.
|
||||||
|
# Check out http://robaldred.co.uk
|
||||||
|
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
class Term < Encoder
|
||||||
|
register_for :term
|
||||||
|
|
||||||
|
TOKEN_COLORS = {
|
||||||
|
:annotation => '35',
|
||||||
|
:attribute_name => '33',
|
||||||
|
:attribute_name_fat => '33',
|
||||||
|
:attribute_value => '31',
|
||||||
|
:attribute_value_fat => '31',
|
||||||
|
:bin => '1;35',
|
||||||
|
:char => {:self => '36', :delimiter => '34'},
|
||||||
|
:class => '1;35',
|
||||||
|
:class_variable => '36',
|
||||||
|
:color => '32',
|
||||||
|
:comment => '37',
|
||||||
|
:complex => '34',
|
||||||
|
:constant => ['34', '4'],
|
||||||
|
:decoration => '35',
|
||||||
|
:definition => '1;32',
|
||||||
|
:directive => ['32', '4'],
|
||||||
|
:doc => '46',
|
||||||
|
:doctype => '1;30',
|
||||||
|
:doc_string => ['31', '4'],
|
||||||
|
:entity => '33',
|
||||||
|
:error => ['1;33', '41'],
|
||||||
|
:exception => '1;31',
|
||||||
|
:float => '1;35',
|
||||||
|
:function => '1;34',
|
||||||
|
:global_variable => '42',
|
||||||
|
:hex => '1;36',
|
||||||
|
:important => '1;31',
|
||||||
|
:include => '33',
|
||||||
|
:integer => '1;34',
|
||||||
|
:interpreted => '1;35',
|
||||||
|
:key => '35',
|
||||||
|
:label => '1;4',
|
||||||
|
:local_variable => '33',
|
||||||
|
:oct => '1;35',
|
||||||
|
:operator_name => '1;29',
|
||||||
|
:pre_constant => '1;36',
|
||||||
|
:pre_type => '1;30',
|
||||||
|
:predefined => ['4', '1;34'],
|
||||||
|
:preprocessor => '36',
|
||||||
|
:pseudo_class => '34',
|
||||||
|
:regexp => {
|
||||||
|
:content => '31',
|
||||||
|
:delimiter => '1;29',
|
||||||
|
:modifier => '35',
|
||||||
|
:function => '1;29'
|
||||||
|
},
|
||||||
|
:reserved => '1;31',
|
||||||
|
:shell => {
|
||||||
|
:self => '42',
|
||||||
|
:content => '1;29',
|
||||||
|
:delimiter => '37',
|
||||||
|
},
|
||||||
|
:string => {
|
||||||
|
:self => '32',
|
||||||
|
:modifier => '1;32',
|
||||||
|
:escape => '1;36',
|
||||||
|
:delimiter => '1;32',
|
||||||
|
},
|
||||||
|
:symbol => '1;32',
|
||||||
|
:tag => '34',
|
||||||
|
:tag_fat => '1;34',
|
||||||
|
:tag_special => ['34', '4'],
|
||||||
|
:type => '1;34',
|
||||||
|
:value => '36',
|
||||||
|
:variable => '34',
|
||||||
|
:insert => '42',
|
||||||
|
:delete => '41',
|
||||||
|
:change => '44',
|
||||||
|
:head => '45',
|
||||||
|
}
|
||||||
|
TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
|
||||||
|
TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
|
||||||
|
TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
|
||||||
|
TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
|
||||||
|
|
||||||
|
protected
|
||||||
|
|
||||||
|
def setup(options)
|
||||||
|
@out = ''
|
||||||
|
@opened = [nil]
|
||||||
|
@subcolors = nil
|
||||||
|
end
|
||||||
|
|
||||||
|
def finish(options)
|
||||||
|
super
|
||||||
|
end
|
||||||
|
|
||||||
|
def token text, type = :plain
|
||||||
|
case text
|
||||||
|
|
||||||
|
when nil
|
||||||
|
# raise 'Token with nil as text was given: %p' % [[text, type]]
|
||||||
|
|
||||||
|
when String
|
||||||
|
|
||||||
|
if color = (@subcolors || TOKEN_COLORS)[type]
|
||||||
|
color = color[:self] || return if Hash === color
|
||||||
|
|
||||||
|
@out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
|
||||||
|
@out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
|
||||||
|
else
|
||||||
|
@out << text
|
||||||
|
end
|
||||||
|
|
||||||
|
# token groups, eg. strings
|
||||||
|
when :open
|
||||||
|
@opened[0] = type
|
||||||
|
if color = TOKEN_COLORS[type]
|
||||||
|
if Hash === color
|
||||||
|
@subcolors = color
|
||||||
|
@out << col(color[:self]) if color[:self]
|
||||||
|
else
|
||||||
|
@subcolors = {}
|
||||||
|
@out << col(color)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
@opened << type
|
||||||
|
when :close
|
||||||
|
if @opened.empty?
|
||||||
|
# nothing to close
|
||||||
|
else
|
||||||
|
@out << col(0) if (@subcolors || {})[:self]
|
||||||
|
@subcolors = nil
|
||||||
|
@opened.pop
|
||||||
|
end
|
||||||
|
|
||||||
|
# whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
|
||||||
|
when :begin_line
|
||||||
|
|
||||||
|
when :end_line
|
||||||
|
|
||||||
|
else
|
||||||
|
raise 'unknown token kind: %p' % [text]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
private
|
||||||
|
|
||||||
|
def col(color)
|
||||||
|
Array(color).map { |c| "\e[#{c}m" }.join
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -14,16 +14,16 @@ module Encoders
|
||||||
|
|
||||||
protected
|
protected
|
||||||
def setup options
|
def setup options
|
||||||
@out = ''
|
super
|
||||||
@sep = options[:separator]
|
@sep = options[:separator]
|
||||||
end
|
end
|
||||||
|
|
||||||
def token text, kind
|
def text_token text, kind
|
||||||
@out << text + @sep if text.is_a? ::String
|
text + @sep
|
||||||
end
|
end
|
||||||
|
|
||||||
def finish options
|
def finish options
|
||||||
@out.chomp @sep
|
super.chomp @sep
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
|
@ -0,0 +1,84 @@
|
||||||
|
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
|
||||||
|
module CodeRay
|
||||||
|
module Encoders
|
||||||
|
|
||||||
|
load :filter
|
||||||
|
|
||||||
|
class TokenClassFilter < Filter
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
register_for :token_class_filter
|
||||||
|
|
||||||
|
DEFAULT_OPTIONS = {
|
||||||
|
:exclude => [],
|
||||||
|
:include => :all
|
||||||
|
}
|
||||||
|
|
||||||
|
protected
|
||||||
|
def setup options
|
||||||
|
super
|
||||||
|
@exclude = options[:exclude]
|
||||||
|
@exclude = Array(@exclude) unless @exclude == :all
|
||||||
|
@include = options[:include]
|
||||||
|
@include = Array(@include) unless @include == :all
|
||||||
|
end
|
||||||
|
|
||||||
|
def include_text_token? text, kind
|
||||||
|
(@include == :all || @include.include?(kind)) &&
|
||||||
|
!(@exclude == :all || @exclude.include?(kind))
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if $0 == __FILE__
|
||||||
|
$VERBOSE = true
|
||||||
|
$: << File.join(File.dirname(__FILE__), '..')
|
||||||
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
require 'test/unit'
|
||||||
|
|
||||||
|
class TokenClassFilterTest < Test::Unit::TestCase
|
||||||
|
|
||||||
|
def test_creation
|
||||||
|
assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
|
||||||
|
assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
|
||||||
|
filter = nil
|
||||||
|
assert_nothing_raised do
|
||||||
|
filter = CodeRay.encoder :token_class_filter
|
||||||
|
end
|
||||||
|
assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_filtering_text_tokens
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
for i in 1..10
|
||||||
|
tokens << [i.to_s, :index]
|
||||||
|
tokens << [' ', :space] if i < 10
|
||||||
|
end
|
||||||
|
assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
|
||||||
|
assert_equal 10, tokens.token_class_filter(:exclude => :space).size
|
||||||
|
assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
|
||||||
|
assert_equal 9, tokens.token_class_filter(:include => :space).size
|
||||||
|
assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
|
||||||
|
assert_equal 0, tokens.token_class_filter(:exclude => :all).size
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_filtering_block_tokens
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
10.times do |i|
|
||||||
|
tokens << [:open, :index]
|
||||||
|
tokens << [i.to_s, :content]
|
||||||
|
tokens << [:close, :index]
|
||||||
|
end
|
||||||
|
assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
|
||||||
|
assert_equal 20, tokens.token_class_filter(:include => :blubb).size
|
||||||
|
assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
|
||||||
|
assert_equal 30, tokens.token_class_filter(:exclude => :index).size
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
|
@ -29,6 +29,7 @@ module Encoders
|
||||||
end
|
end
|
||||||
|
|
||||||
def finish options
|
def finish options
|
||||||
|
@out = ''
|
||||||
@doc.write @out, options[:pretty], options[:transitive], true
|
@doc.write @out, options[:pretty], options[:transitive], true
|
||||||
@out
|
@out
|
||||||
end
|
end
|
|
@ -0,0 +1,95 @@
|
||||||
|
module CodeRay
|
||||||
|
|
||||||
|
# A little hack to enable CodeRay highlighting in RedCloth.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# require 'coderay'
|
||||||
|
# require 'coderay/for_redcloth'
|
||||||
|
# RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
|
||||||
|
#
|
||||||
|
# Make sure you have RedCloth 4.0.3 activated, for example by calling
|
||||||
|
# require 'rubygems'
|
||||||
|
# before RedCloth is loaded and before calling CodeRay.for_redcloth.
|
||||||
|
module ForRedCloth
|
||||||
|
|
||||||
|
def self.install
|
||||||
|
gem 'RedCloth', '>= 4.0.3' if defined? gem
|
||||||
|
require 'redcloth'
|
||||||
|
unless RedCloth::VERSION.to_s >= '4.0.3'
|
||||||
|
if defined? gem
|
||||||
|
raise 'CodeRay.for_redcloth needs RedCloth version 4.0.3 or later. ' +
|
||||||
|
"You have #{RedCloth::VERSION}. Please gem install RedCloth."
|
||||||
|
else
|
||||||
|
$".delete 'redcloth.rb' # sorry, but it works
|
||||||
|
require 'rubygems'
|
||||||
|
return install # retry
|
||||||
|
end
|
||||||
|
end
|
||||||
|
unless RedCloth::VERSION.to_s >= '4.2.2'
|
||||||
|
warn 'CodeRay.for_redcloth works best with RedCloth version 4.2.2 or later.'
|
||||||
|
end
|
||||||
|
RedCloth::TextileDoc.send :include, ForRedCloth::TextileDoc
|
||||||
|
RedCloth::Formatters::HTML.module_eval do
|
||||||
|
def unescape(html)
|
||||||
|
replacements = {
|
||||||
|
'&' => '&',
|
||||||
|
'"' => '"',
|
||||||
|
'>' => '>',
|
||||||
|
'<' => '<',
|
||||||
|
}
|
||||||
|
html.gsub(/&(?:amp|quot|[gl]t);/) { |entity| replacements[entity] }
|
||||||
|
end
|
||||||
|
undef code, bc_open, bc_close, escape_pre
|
||||||
|
def code(opts) # :nodoc:
|
||||||
|
opts[:block] = true
|
||||||
|
if !opts[:lang] && RedCloth::VERSION.to_s >= '4.2.0'
|
||||||
|
# simulating pre-4.2 behavior
|
||||||
|
if opts[:text].sub!(/\A\[(\w+)\]/, '')
|
||||||
|
if CodeRay::Scanners[$1].plugin_id == 'plaintext'
|
||||||
|
opts[:text] = $& + opts[:text]
|
||||||
|
else
|
||||||
|
opts[:lang] = $1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if opts[:lang] && !filter_coderay
|
||||||
|
require 'coderay'
|
||||||
|
@in_bc ||= nil
|
||||||
|
format = @in_bc ? :div : :span
|
||||||
|
opts[:text] = unescape(opts[:text]) unless @in_bc
|
||||||
|
highlighted_code = CodeRay.encode opts[:text], opts[:lang], format, :stream => true
|
||||||
|
highlighted_code.sub!(/\A<(span|div)/) { |m| m + pba(@in_bc || opts) }
|
||||||
|
highlighted_code
|
||||||
|
else
|
||||||
|
"<code#{pba(opts)}>#{opts[:text]}</code>"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
def bc_open(opts) # :nodoc:
|
||||||
|
opts[:block] = true
|
||||||
|
@in_bc = opts
|
||||||
|
opts[:lang] ? '' : "<pre#{pba(opts)}>"
|
||||||
|
end
|
||||||
|
def bc_close(opts) # :nodoc:
|
||||||
|
opts = @in_bc
|
||||||
|
@in_bc = nil
|
||||||
|
opts[:lang] ? '' : "</pre>\n"
|
||||||
|
end
|
||||||
|
def escape_pre(text)
|
||||||
|
if @in_bc ||= nil
|
||||||
|
text
|
||||||
|
else
|
||||||
|
html_esc(text, :html_escape_preformatted)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
module TextileDoc # :nodoc:
|
||||||
|
attr_accessor :filter_coderay
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
CodeRay::ForRedCloth.install
|
|
@ -1,3 +1,4 @@
|
||||||
|
#!/usr/bin/env ruby
|
||||||
module CodeRay
|
module CodeRay
|
||||||
|
|
||||||
# = FileType
|
# = FileType
|
||||||
|
@ -33,12 +34,14 @@ module FileType
|
||||||
# That means you can get filetypes from files that don't exist.
|
# That means you can get filetypes from files that don't exist.
|
||||||
def [] filename, read_shebang = false
|
def [] filename, read_shebang = false
|
||||||
name = File.basename filename
|
name = File.basename filename
|
||||||
ext = File.extname name
|
ext = File.extname(name).sub(/^\./, '') # from last dot, delete the leading dot
|
||||||
ext.sub!(/^\./, '') # delete the leading dot
|
ext2 = filename.to_s[/\.(.*)/, 1] # from first dot
|
||||||
|
|
||||||
type =
|
type =
|
||||||
TypeFromExt[ext] ||
|
TypeFromExt[ext] ||
|
||||||
TypeFromExt[ext.downcase] ||
|
TypeFromExt[ext.downcase] ||
|
||||||
|
(TypeFromExt[ext2] if ext2) ||
|
||||||
|
(TypeFromExt[ext2.downcase] if ext2) ||
|
||||||
TypeFromName[name] ||
|
TypeFromName[name] ||
|
||||||
TypeFromName[name.downcase]
|
TypeFromName[name.downcase]
|
||||||
type ||= shebang(filename) if read_shebang
|
type ||= shebang(filename) if read_shebang
|
||||||
|
@ -49,8 +52,11 @@ module FileType
|
||||||
def shebang filename
|
def shebang filename
|
||||||
begin
|
begin
|
||||||
File.open filename, 'r' do |f|
|
File.open filename, 'r' do |f|
|
||||||
first_line = f.gets
|
if first_line = f.gets
|
||||||
first_line[TypeFromShebang]
|
if type = first_line[TypeFromShebang]
|
||||||
|
type.to_sym
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
rescue IOError
|
rescue IOError
|
||||||
nil
|
nil
|
||||||
|
@ -77,30 +83,46 @@ module FileType
|
||||||
end
|
end
|
||||||
|
|
||||||
TypeFromExt = {
|
TypeFromExt = {
|
||||||
'rb' => :ruby,
|
|
||||||
'rbw' => :ruby,
|
|
||||||
'rake' => :ruby,
|
|
||||||
'mab' => :ruby,
|
|
||||||
'cpp' => :c,
|
|
||||||
'c' => :c,
|
'c' => :c,
|
||||||
|
'css' => :css,
|
||||||
|
'diff' => :diff,
|
||||||
|
'dpr' => :delphi,
|
||||||
|
'groovy' => :groovy,
|
||||||
|
'gvy' => :groovy,
|
||||||
'h' => :c,
|
'h' => :c,
|
||||||
'java' => :java,
|
|
||||||
'js' => :javascript,
|
|
||||||
'xml' => :xml,
|
|
||||||
'htm' => :html,
|
'htm' => :html,
|
||||||
'html' => :html,
|
'html' => :html,
|
||||||
|
'html.erb' => :rhtml,
|
||||||
|
'java' => :java,
|
||||||
|
'js' => :java_script,
|
||||||
|
'json' => :json,
|
||||||
|
'mab' => :ruby,
|
||||||
|
'pas' => :delphi,
|
||||||
|
'patch' => :diff,
|
||||||
'php' => :php,
|
'php' => :php,
|
||||||
'php3' => :php,
|
'php3' => :php,
|
||||||
'php4' => :php,
|
'php4' => :php,
|
||||||
'php5' => :php,
|
'php5' => :php,
|
||||||
'xhtml' => :xhtml,
|
'py' => :python,
|
||||||
|
'py3' => :python,
|
||||||
|
'pyw' => :python,
|
||||||
|
'rake' => :ruby,
|
||||||
'raydebug' => :debug,
|
'raydebug' => :debug,
|
||||||
|
'rb' => :ruby,
|
||||||
|
'rbw' => :ruby,
|
||||||
'rhtml' => :rhtml,
|
'rhtml' => :rhtml,
|
||||||
'ss' => :scheme,
|
'rxml' => :ruby,
|
||||||
'sch' => :scheme,
|
'sch' => :scheme,
|
||||||
|
'sql' => :sql,
|
||||||
|
'ss' => :scheme,
|
||||||
|
'xhtml' => :xhtml,
|
||||||
|
'xml' => :xml,
|
||||||
'yaml' => :yaml,
|
'yaml' => :yaml,
|
||||||
'yml' => :yaml,
|
'yml' => :yaml,
|
||||||
}
|
}
|
||||||
|
for cpp_alias in %w[cc cpp cp cxx c++ C hh hpp h++ cu]
|
||||||
|
TypeFromExt[cpp_alias] = :cpp
|
||||||
|
end
|
||||||
|
|
||||||
TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
|
TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
|
||||||
|
|
||||||
|
@ -115,15 +137,16 @@ end
|
||||||
|
|
||||||
if $0 == __FILE__
|
if $0 == __FILE__
|
||||||
$VERBOSE = true
|
$VERBOSE = true
|
||||||
eval DATA.read, nil, $0, __LINE__+4
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
end
|
end
|
||||||
|
|
||||||
__END__
|
__END__
|
||||||
|
|
||||||
require 'test/unit'
|
require 'test/unit'
|
||||||
|
|
||||||
class TC_FileType < Test::Unit::TestCase
|
class FileTypeTests < Test::Unit::TestCase
|
||||||
|
|
||||||
|
include CodeRay
|
||||||
|
|
||||||
def test_fetch
|
def test_fetch
|
||||||
assert_raise FileType::UnknownFileType do
|
assert_raise FileType::UnknownFileType do
|
||||||
FileType.fetch ''
|
FileType.fetch ''
|
||||||
|
@ -150,6 +173,8 @@ class TC_FileType < Test::Unit::TestCase
|
||||||
|
|
||||||
def test_ruby
|
def test_ruby
|
||||||
assert_equal :ruby, FileType['test.rb']
|
assert_equal :ruby, FileType['test.rb']
|
||||||
|
assert_equal :ruby, FileType['test.java.rb']
|
||||||
|
assert_equal :java, FileType['test.rb.java']
|
||||||
assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
|
assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
|
||||||
assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
|
assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
|
||||||
assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
|
assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
|
||||||
|
@ -169,11 +194,23 @@ class TC_FileType < Test::Unit::TestCase
|
||||||
assert_not_equal :c, FileType['~/projects/blabla/c']
|
assert_not_equal :c, FileType['~/projects/blabla/c']
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def test_cpp
|
||||||
|
assert_equal :cpp, FileType['test.c++']
|
||||||
|
assert_equal :cpp, FileType['test.cxx']
|
||||||
|
assert_equal :cpp, FileType['test.hh']
|
||||||
|
assert_equal :cpp, FileType['test.hpp']
|
||||||
|
assert_equal :cpp, FileType['test.cu']
|
||||||
|
assert_equal :cpp, FileType['test.C']
|
||||||
|
assert_not_equal :cpp, FileType['test.c']
|
||||||
|
assert_not_equal :cpp, FileType['test.h']
|
||||||
|
end
|
||||||
|
|
||||||
def test_html
|
def test_html
|
||||||
assert_equal :html, FileType['test.htm']
|
assert_equal :html, FileType['test.htm']
|
||||||
assert_equal :xhtml, FileType['test.xhtml']
|
assert_equal :xhtml, FileType['test.xhtml']
|
||||||
assert_equal :xhtml, FileType['test.html.xhtml']
|
assert_equal :xhtml, FileType['test.html.xhtml']
|
||||||
assert_equal :rhtml, FileType['_form.rhtml']
|
assert_equal :rhtml, FileType['_form.rhtml']
|
||||||
|
assert_equal :rhtml, FileType['_form.html.erb']
|
||||||
end
|
end
|
||||||
|
|
||||||
def test_yaml
|
def test_yaml
|
||||||
|
@ -183,7 +220,16 @@ class TC_FileType < Test::Unit::TestCase
|
||||||
assert_not_equal :yaml, FileType['YAML']
|
assert_not_equal :yaml, FileType['YAML']
|
||||||
end
|
end
|
||||||
|
|
||||||
def test_shebang
|
def test_pathname
|
||||||
|
require 'pathname'
|
||||||
|
pn = Pathname.new 'test.rb'
|
||||||
|
assert_equal :ruby, FileType[pn]
|
||||||
|
dir = Pathname.new '/etc/var/blubb'
|
||||||
|
assert_equal :ruby, FileType[dir + pn]
|
||||||
|
assert_equal :cpp, FileType[dir + 'test.cpp']
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_no_shebang
|
||||||
dir = './test'
|
dir = './test'
|
||||||
if File.directory? dir
|
if File.directory? dir
|
||||||
Dir.chdir dir do
|
Dir.chdir dir do
|
||||||
|
@ -191,5 +237,19 @@ class TC_FileType < Test::Unit::TestCase
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def test_shebang_empty_file
|
||||||
|
require 'tmpdir'
|
||||||
|
tmpfile = File.join(Dir.tmpdir, 'bla')
|
||||||
|
File.open(tmpfile, 'w') { } # touch
|
||||||
|
assert_equal nil, FileType[tmpfile]
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_shebang
|
||||||
|
require 'tmpdir'
|
||||||
|
tmpfile = File.join(Dir.tmpdir, 'bla')
|
||||||
|
File.open(tmpfile, 'w') { |f| f.puts '#!/usr/bin/env ruby' }
|
||||||
|
assert_equal :ruby, FileType[tmpfile, true]
|
||||||
|
end
|
||||||
|
|
||||||
end
|
end
|
|
@ -2,7 +2,7 @@
|
||||||
#
|
#
|
||||||
# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
|
# A simplified interface to the gzip library +zlib+ (from the Ruby Standard Library.)
|
||||||
#
|
#
|
||||||
# Author: murphy (mail to murphy cYcnus de)
|
# Author: murphy (mail to murphy rubychan de)
|
||||||
#
|
#
|
||||||
# Version: 0.2 (2005.may.28)
|
# Version: 0.2 (2005.may.28)
|
||||||
#
|
#
|
|
@ -2,8 +2,6 @@ module CodeRay
|
||||||
|
|
||||||
# = PluginHost
|
# = PluginHost
|
||||||
#
|
#
|
||||||
# $Id: plugin.rb 220 2007-01-01 02:58:58Z murphy $
|
|
||||||
#
|
|
||||||
# A simple subclass plugin system.
|
# A simple subclass plugin system.
|
||||||
#
|
#
|
||||||
# Example:
|
# Example:
|
||||||
|
@ -22,7 +20,7 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# Generators[:fancy] #-> FancyGenerator
|
# Generators[:fancy] #-> FancyGenerator
|
||||||
# # or
|
# # or
|
||||||
# require_plugin 'Generators/fancy'
|
# CodeRay.require_plugin 'Generators/fancy'
|
||||||
module PluginHost
|
module PluginHost
|
||||||
|
|
||||||
# Raised if Encoders::[] fails because:
|
# Raised if Encoders::[] fails because:
|
||||||
|
@ -135,9 +133,13 @@ module PluginHost
|
||||||
# map :navy => :dark_blue
|
# map :navy => :dark_blue
|
||||||
# default :gray
|
# default :gray
|
||||||
# end
|
# end
|
||||||
def default id
|
def default id = nil
|
||||||
id = validate_id id
|
if id
|
||||||
plugin_hash[nil] = id
|
id = validate_id id
|
||||||
|
plugin_hash[nil] = id
|
||||||
|
else
|
||||||
|
plugin_hash[nil]
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Every plugin must register itself for one or more
|
# Every plugin must register itself for one or more
|
||||||
|
@ -174,7 +176,7 @@ module PluginHost
|
||||||
def inspect
|
def inspect
|
||||||
map = plugin_hash.dup
|
map = plugin_hash.dup
|
||||||
map.each do |id, plugin|
|
map.each do |id, plugin|
|
||||||
map[id] = plugin.to_s[/(?>[\w_]+)$/]
|
map[id] = plugin.to_s[/(?>\w+)$/]
|
||||||
end
|
end
|
||||||
"#{name}[#{host_id}]#{map.inspect}"
|
"#{name}[#{host_id}]#{map.inspect}"
|
||||||
end
|
end
|
||||||
|
@ -212,7 +214,7 @@ protected
|
||||||
mapfile = path_to '_map'
|
mapfile = path_to '_map'
|
||||||
if File.exist? mapfile
|
if File.exist? mapfile
|
||||||
require mapfile
|
require mapfile
|
||||||
elsif $DEBUG
|
elsif $VERBOSE
|
||||||
warn 'no _map.rb found for %s' % name
|
warn 'no _map.rb found for %s' % name
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -241,7 +243,7 @@ protected
|
||||||
id
|
id
|
||||||
elsif id.is_a? String
|
elsif id.is_a? String
|
||||||
if id[/\w+/] == id
|
if id[/\w+/] == id
|
||||||
id.to_sym
|
id.downcase.to_sym
|
||||||
else
|
else
|
||||||
raise ArgumentError, "Invalid id: '#{id}' given."
|
raise ArgumentError, "Invalid id: '#{id}' given."
|
||||||
end
|
end
|
||||||
|
@ -278,6 +280,16 @@ module Plugin
|
||||||
def register_for *ids
|
def register_for *ids
|
||||||
plugin_host.register self, *ids
|
plugin_host.register self, *ids
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Returns the title of the plugin, or sets it to the
|
||||||
|
# optional argument +title+.
|
||||||
|
def title title = nil
|
||||||
|
if title
|
||||||
|
@title = title.to_s
|
||||||
|
else
|
||||||
|
@title ||= name[/([^:]+)$/, 1]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
# The host for this Plugin class.
|
# The host for this Plugin class.
|
||||||
def plugin_host host = nil
|
def plugin_host host = nil
|
||||||
|
@ -299,15 +311,23 @@ module Plugin
|
||||||
#
|
#
|
||||||
# The above example loads the file myplugin/my_helper.rb relative to the
|
# The above example loads the file myplugin/my_helper.rb relative to the
|
||||||
# file in which MyPlugin was defined.
|
# file in which MyPlugin was defined.
|
||||||
|
#
|
||||||
|
# You can also load a helper from a different plugin:
|
||||||
|
#
|
||||||
|
# helper 'other_plugin/helper_name'
|
||||||
def helper *helpers
|
def helper *helpers
|
||||||
for helper in helpers
|
for helper in helpers
|
||||||
self::PLUGIN_HOST.require_helper plugin_id, helper.to_s
|
if helper.is_a?(String) && helper[/\//]
|
||||||
|
self::PLUGIN_HOST.require_helper $`, $'
|
||||||
|
else
|
||||||
|
self::PLUGIN_HOST.require_helper plugin_id, helper.to_s
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# Returns the pulgin id used by the engine.
|
# Returns the pulgin id used by the engine.
|
||||||
def plugin_id
|
def plugin_id
|
||||||
name[/[\w_]+$/].downcase
|
name[/\w+$/].downcase
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
@ -318,7 +338,7 @@ end
|
||||||
# CodeRay.require_plugin '<Host ID>/<Plugin ID>'
|
# CodeRay.require_plugin '<Host ID>/<Plugin ID>'
|
||||||
#
|
#
|
||||||
# Returns the loaded plugin.
|
# Returns the loaded plugin.
|
||||||
def require_plugin path
|
def self.require_plugin path
|
||||||
host_id, plugin_id = path.split '/', 2
|
host_id, plugin_id = path.split '/', 2
|
||||||
host = PluginHost.host_by_id(host_id)
|
host = PluginHost.host_by_id(host_id)
|
||||||
raise PluginHost::HostNotFound,
|
raise PluginHost::HostNotFound,
|
|
@ -98,15 +98,22 @@ class CaseIgnoringWordList < WordList
|
||||||
# Creates a new case-insensitive WordList with +default+ as default value.
|
# Creates a new case-insensitive WordList with +default+ as default value.
|
||||||
#
|
#
|
||||||
# You can activate caching to store the results for every [] request.
|
# You can activate caching to store the results for every [] request.
|
||||||
|
# This speeds up subsequent lookups for the same word, but also
|
||||||
|
# uses memory.
|
||||||
def initialize default = false, caching = false
|
def initialize default = false, caching = false
|
||||||
if caching
|
if caching
|
||||||
super(default, false) do |h, k|
|
super(default, false) do |h, k|
|
||||||
h[k] = h.fetch k.downcase, default
|
h[k] = h.fetch k.downcase, default
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
def self.[] key # :nodoc:
|
super(default, false)
|
||||||
super(key.downcase)
|
extend Uncached
|
||||||
end
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
module Uncached # :nodoc:
|
||||||
|
def [] key
|
||||||
|
super(key.downcase)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -120,4 +127,12 @@ class CaseIgnoringWordList < WordList
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
# check memory consumption
|
||||||
|
END {
|
||||||
|
ObjectSpace.each_object(CodeRay::CaseIgnoringWordList) do |wl|
|
||||||
|
p wl.inject(0) { |memo, key, value| memo + key.size + 24 }
|
||||||
|
end
|
||||||
|
}
|
|
@ -4,8 +4,6 @@ module CodeRay
|
||||||
|
|
||||||
# = Scanners
|
# = Scanners
|
||||||
#
|
#
|
||||||
# $Id: scanner.rb 222 2007-01-01 16:26:17Z murphy $
|
|
||||||
#
|
|
||||||
# This module holds the Scanner class and its subclasses.
|
# This module holds the Scanner class and its subclasses.
|
||||||
# For example, the Ruby scanner is named CodeRay::Scanners::Ruby
|
# For example, the Ruby scanner is named CodeRay::Scanners::Ruby
|
||||||
# can be found in coderay/scanners/ruby.
|
# can be found in coderay/scanners/ruby.
|
||||||
|
@ -45,6 +43,7 @@ module CodeRay
|
||||||
# You can also use +map+, +any?+, +find+ and even +sort_by+,
|
# You can also use +map+, +any?+, +find+ and even +sort_by+,
|
||||||
# if you want.
|
# if you want.
|
||||||
class Scanner < StringScanner
|
class Scanner < StringScanner
|
||||||
|
|
||||||
extend Plugin
|
extend Plugin
|
||||||
plugin_host Scanners
|
plugin_host Scanners
|
||||||
|
|
||||||
|
@ -57,6 +56,8 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# Define @default_options for subclasses.
|
# Define @default_options for subclasses.
|
||||||
DEFAULT_OPTIONS = { :stream => false }
|
DEFAULT_OPTIONS = { :stream => false }
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = [:comment, :doctype]
|
||||||
|
|
||||||
class << self
|
class << self
|
||||||
|
|
||||||
|
@ -66,7 +67,19 @@ module CodeRay
|
||||||
end
|
end
|
||||||
|
|
||||||
def normify code
|
def normify code
|
||||||
code = code.to_s.to_unix
|
code = code.to_s
|
||||||
|
if code.respond_to? :force_encoding
|
||||||
|
debug, $DEBUG = $DEBUG, false
|
||||||
|
begin
|
||||||
|
code.force_encoding 'utf-8'
|
||||||
|
code[/\z/] # raises an ArgumentError when code contains a non-UTF-8 char
|
||||||
|
rescue ArgumentError
|
||||||
|
code.force_encoding 'binary'
|
||||||
|
ensure
|
||||||
|
$DEBUG = debug
|
||||||
|
end
|
||||||
|
end
|
||||||
|
code.to_unix
|
||||||
end
|
end
|
||||||
|
|
||||||
def file_extension extension = nil
|
def file_extension extension = nil
|
||||||
|
@ -75,7 +88,7 @@ module CodeRay
|
||||||
else
|
else
|
||||||
@file_extension ||= plugin_id.to_s
|
@file_extension ||= plugin_id.to_s
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -104,9 +117,10 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# Else, a Tokens object is used.
|
# Else, a Tokens object is used.
|
||||||
def initialize code='', options = {}, &block
|
def initialize code='', options = {}, &block
|
||||||
@options = self.class::DEFAULT_OPTIONS.merge options
|
|
||||||
raise "I am only the basic Scanner class. I can't scan "\
|
raise "I am only the basic Scanner class. I can't scan "\
|
||||||
"anything. :( Use my subclasses." if self.class == Scanner
|
"anything. :( Use my subclasses." if self.class == Scanner
|
||||||
|
|
||||||
|
@options = self.class::DEFAULT_OPTIONS.merge options
|
||||||
|
|
||||||
super Scanner.normify(code)
|
super Scanner.normify(code)
|
||||||
|
|
||||||
|
@ -121,6 +135,7 @@ module CodeRay
|
||||||
"but :stream is #{@options[:stream]}" if block_given?
|
"but :stream is #{@options[:stream]}" if block_given?
|
||||||
@tokens ||= Tokens.new
|
@tokens ||= Tokens.new
|
||||||
end
|
end
|
||||||
|
@tokens.scanner = self
|
||||||
|
|
||||||
setup
|
setup
|
||||||
end
|
end
|
||||||
|
@ -140,6 +155,11 @@ module CodeRay
|
||||||
alias code string
|
alias code string
|
||||||
alias code= string=
|
alias code= string=
|
||||||
|
|
||||||
|
# Returns the Plugin ID for this scanner.
|
||||||
|
def lang
|
||||||
|
self.class.plugin_id
|
||||||
|
end
|
||||||
|
|
||||||
# Scans the code and returns all tokens in a Tokens object.
|
# Scans the code and returns all tokens in a Tokens object.
|
||||||
def tokenize new_string=nil, options = {}
|
def tokenize new_string=nil, options = {}
|
||||||
options = @options.merge(options)
|
options = @options.merge(options)
|
||||||
|
@ -178,6 +198,24 @@ module CodeRay
|
||||||
def line
|
def line
|
||||||
string[0..pos].count("\n") + 1
|
string[0..pos].count("\n") + 1
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def column pos = self.pos
|
||||||
|
return 0 if pos <= 0
|
||||||
|
string = string()
|
||||||
|
if string.respond_to?(:bytesize) && (defined?(@bin_string) || string.bytesize != string.size)
|
||||||
|
@bin_string ||= string.dup.force_encoding('binary')
|
||||||
|
string = @bin_string
|
||||||
|
end
|
||||||
|
pos - (string.rindex(?\n, pos) || 0)
|
||||||
|
end
|
||||||
|
|
||||||
|
def marshal_dump
|
||||||
|
@options
|
||||||
|
end
|
||||||
|
|
||||||
|
def marshal_load options
|
||||||
|
@options = options
|
||||||
|
end
|
||||||
|
|
||||||
protected
|
protected
|
||||||
|
|
||||||
|
@ -202,6 +240,7 @@ module CodeRay
|
||||||
def reset_instance
|
def reset_instance
|
||||||
@tokens.clear unless @options[:keep_tokens]
|
@tokens.clear unless @options[:keep_tokens]
|
||||||
@cached_tokens = nil
|
@cached_tokens = nil
|
||||||
|
@bin_string = nil if defined? @bin_string
|
||||||
end
|
end
|
||||||
|
|
||||||
# Scanner error with additional status information
|
# Scanner error with additional status information
|
||||||
|
@ -214,7 +253,7 @@ module CodeRay
|
||||||
tokens:
|
tokens:
|
||||||
%s
|
%s
|
||||||
|
|
||||||
current line: %d pos = %d
|
current line: %d column: %d pos: %d
|
||||||
matched: %p state: %p
|
matched: %p state: %p
|
||||||
bol? = %p, eos? = %p
|
bol? = %p, eos? = %p
|
||||||
|
|
||||||
|
@ -229,10 +268,10 @@ surrounding code:
|
||||||
msg,
|
msg,
|
||||||
tokens.size,
|
tokens.size,
|
||||||
tokens.last(10).map { |t| t.inspect }.join("\n"),
|
tokens.last(10).map { |t| t.inspect }.join("\n"),
|
||||||
line, pos,
|
line, column, pos,
|
||||||
matched, state, bol?, eos?,
|
matched, state, bol?, eos?,
|
||||||
string[pos-ambit,ambit],
|
string[pos - ambit, ambit],
|
||||||
string[pos,ambit],
|
string[pos, ambit],
|
||||||
]
|
]
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
map \
|
||||||
|
:h => :c,
|
||||||
|
:cplusplus => :cpp,
|
||||||
|
:'c++' => :cpp,
|
||||||
|
:ecma => :java_script,
|
||||||
|
:ecmascript => :java_script,
|
||||||
|
:ecma_script => :java_script,
|
||||||
|
:irb => :ruby,
|
||||||
|
:javascript => :java_script,
|
||||||
|
:js => :java_script,
|
||||||
|
:nitro => :nitro_xhtml,
|
||||||
|
:pascal => :delphi,
|
||||||
|
:plain => :plaintext,
|
||||||
|
:xhtml => :html,
|
||||||
|
:yml => :yaml
|
||||||
|
|
||||||
|
default :plain
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -3,42 +3,50 @@ module Scanners
|
||||||
|
|
||||||
class C < Scanner
|
class C < Scanner
|
||||||
|
|
||||||
register_for :c
|
|
||||||
|
|
||||||
include Streamable
|
include Streamable
|
||||||
|
|
||||||
|
register_for :c
|
||||||
|
file_extension 'c'
|
||||||
|
|
||||||
RESERVED_WORDS = [
|
RESERVED_WORDS = [
|
||||||
'asm', 'break', 'case', 'continue', 'default', 'do', 'else',
|
'asm', 'break', 'case', 'continue', 'default', 'do',
|
||||||
'for', 'goto', 'if', 'return', 'switch', 'while',
|
'else', 'enum', 'for', 'goto', 'if', 'return',
|
||||||
'struct', 'union', 'enum', 'typedef',
|
'sizeof', 'struct', 'switch', 'typedef', 'union', 'while',
|
||||||
'static', 'register', 'auto', 'extern',
|
'restrict', # added in C99
|
||||||
'sizeof',
|
|
||||||
'volatile', 'const', # C89
|
|
||||||
'inline', 'restrict', # C99
|
|
||||||
]
|
]
|
||||||
|
|
||||||
PREDEFINED_TYPES = [
|
PREDEFINED_TYPES = [
|
||||||
'int', 'long', 'short', 'char', 'void',
|
'int', 'long', 'short', 'char',
|
||||||
'signed', 'unsigned', 'float', 'double',
|
'signed', 'unsigned', 'float', 'double',
|
||||||
'bool', 'complex', # C99
|
'bool', 'complex', # added in C99
|
||||||
]
|
]
|
||||||
|
|
||||||
PREDEFINED_CONSTANTS = [
|
PREDEFINED_CONSTANTS = [
|
||||||
'EOF', 'NULL',
|
'EOF', 'NULL',
|
||||||
'true', 'false', # C99
|
'true', 'false', # added in C99
|
||||||
|
]
|
||||||
|
DIRECTIVES = [
|
||||||
|
'auto', 'extern', 'register', 'static', 'void',
|
||||||
|
'const', 'volatile', # added in C89
|
||||||
|
'inline', # added in C99
|
||||||
]
|
]
|
||||||
|
|
||||||
IDENT_KIND = WordList.new(:ident).
|
IDENT_KIND = WordList.new(:ident).
|
||||||
add(RESERVED_WORDS, :reserved).
|
add(RESERVED_WORDS, :reserved).
|
||||||
add(PREDEFINED_TYPES, :pre_type).
|
add(PREDEFINED_TYPES, :pre_type).
|
||||||
|
add(DIRECTIVES, :directive).
|
||||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
add(PREDEFINED_CONSTANTS, :pre_constant)
|
||||||
|
|
||||||
ESCAPE = / [rbfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||||
|
|
||||||
def scan_tokens tokens, options
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
state = :initial
|
state = :initial
|
||||||
|
label_expected = true
|
||||||
|
case_expected = false
|
||||||
|
label_expected_before_preproc_line = nil
|
||||||
|
in_preproc_line = false
|
||||||
|
|
||||||
until eos?
|
until eos?
|
||||||
|
|
||||||
|
@ -49,8 +57,13 @@ module Scanners
|
||||||
|
|
||||||
when :initial
|
when :initial
|
||||||
|
|
||||||
if scan(/ \s+ | \\\n /x)
|
if match = scan(/ \s+ | \\\n /x)
|
||||||
kind = :space
|
if in_preproc_line && match != "\\\n" && match.index(?\n)
|
||||||
|
in_preproc_line = false
|
||||||
|
label_expected = label_expected_before_preproc_line
|
||||||
|
end
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
|
||||||
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
||||||
kind = :comment
|
kind = :comment
|
||||||
|
@ -59,16 +72,32 @@ module Scanners
|
||||||
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
|
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
|
||||||
kind = :comment
|
kind = :comment
|
||||||
|
|
||||||
elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
|
elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
|
||||||
|
label_expected = match =~ /[;\{\}]/
|
||||||
|
if case_expected
|
||||||
|
label_expected = true if match == ':'
|
||||||
|
case_expected = false
|
||||||
|
end
|
||||||
kind = :operator
|
kind = :operator
|
||||||
|
|
||||||
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
|
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
|
||||||
kind = IDENT_KIND[match]
|
kind = IDENT_KIND[match]
|
||||||
if kind == :ident and check(/:(?!:)/)
|
if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
|
||||||
match << scan(/:/)
|
|
||||||
kind = :label
|
kind = :label
|
||||||
|
match << matched
|
||||||
|
else
|
||||||
|
label_expected = false
|
||||||
|
if kind == :reserved
|
||||||
|
case match
|
||||||
|
when 'case', 'default'
|
||||||
|
case_expected = true
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
elsif scan(/\$/)
|
||||||
|
kind = :ident
|
||||||
|
|
||||||
elsif match = scan(/L?"/)
|
elsif match = scan(/L?"/)
|
||||||
tokens << [:open, :string]
|
tokens << [:open, :string]
|
||||||
if match[0] == ?L
|
if match[0] == ?L
|
||||||
|
@ -78,23 +107,30 @@ module Scanners
|
||||||
state = :string
|
state = :string
|
||||||
kind = :delimiter
|
kind = :delimiter
|
||||||
|
|
||||||
elsif scan(/#\s*(\w*)/)
|
elsif scan(/#[ \t]*(\w*)/)
|
||||||
kind = :preprocessor # FIXME multiline preprocs
|
kind = :preprocessor
|
||||||
|
in_preproc_line = true
|
||||||
|
label_expected_before_preproc_line = label_expected
|
||||||
state = :include_expected if self[1] == 'include'
|
state = :include_expected if self[1] == 'include'
|
||||||
|
|
||||||
elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
|
elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
|
||||||
|
label_expected = false
|
||||||
kind = :char
|
kind = :char
|
||||||
|
|
||||||
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
||||||
|
label_expected = false
|
||||||
kind = :hex
|
kind = :hex
|
||||||
|
|
||||||
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
|
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
|
||||||
|
label_expected = false
|
||||||
kind = :oct
|
kind = :oct
|
||||||
|
|
||||||
elsif scan(/(?:\d+)(?![.eEfF])/)
|
elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
|
||||||
|
label_expected = false
|
||||||
kind = :integer
|
kind = :integer
|
||||||
|
|
||||||
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
||||||
|
label_expected = false
|
||||||
kind = :float
|
kind = :float
|
||||||
|
|
||||||
else
|
else
|
||||||
|
@ -110,6 +146,7 @@ module Scanners
|
||||||
tokens << ['"', :delimiter]
|
tokens << ['"', :delimiter]
|
||||||
tokens << [:close, :string]
|
tokens << [:close, :string]
|
||||||
state = :initial
|
state = :initial
|
||||||
|
label_expected = false
|
||||||
next
|
next
|
||||||
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||||
kind = :char
|
kind = :char
|
||||||
|
@ -117,6 +154,7 @@ module Scanners
|
||||||
tokens << [:close, :string]
|
tokens << [:close, :string]
|
||||||
kind = :error
|
kind = :error
|
||||||
state = :initial
|
state = :initial
|
||||||
|
label_expected = false
|
||||||
else
|
else
|
||||||
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
end
|
end
|
||||||
|
@ -131,8 +169,8 @@ module Scanners
|
||||||
state = :initial if match.index ?\n
|
state = :initial if match.index ?\n
|
||||||
|
|
||||||
else
|
else
|
||||||
getch
|
state = :initial
|
||||||
kind = :error
|
next
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -142,7 +180,7 @@ module Scanners
|
||||||
end
|
end
|
||||||
|
|
||||||
match ||= matched
|
match ||= matched
|
||||||
if $DEBUG and not kind
|
if $CODERAY_DEBUG and not kind
|
||||||
raise_inspect 'Error token %p in line %d' %
|
raise_inspect 'Error token %p in line %d' %
|
||||||
[[match, kind], line], tokens
|
[[match, kind], line], tokens
|
||||||
end
|
end
|
|
@ -0,0 +1,228 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
class CPlusPlus < Scanner
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
|
||||||
|
register_for :cpp
|
||||||
|
file_extension 'cpp'
|
||||||
|
title 'C++'
|
||||||
|
|
||||||
|
# http://www.cppreference.com/wiki/keywords/start
|
||||||
|
RESERVED_WORDS = [
|
||||||
|
'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
|
||||||
|
'case', 'catch', 'class', 'compl', 'const_cast',
|
||||||
|
'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
|
||||||
|
'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
|
||||||
|
'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
|
||||||
|
'sizeof', 'static_cast', 'struct', 'switch', 'template',
|
||||||
|
'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
|
||||||
|
'while', 'xor', 'xor_eq'
|
||||||
|
]
|
||||||
|
|
||||||
|
PREDEFINED_TYPES = [
|
||||||
|
'bool', 'char', 'double', 'float', 'int', 'long',
|
||||||
|
'short', 'signed', 'unsigned', 'wchar_t', 'string'
|
||||||
|
]
|
||||||
|
PREDEFINED_CONSTANTS = [
|
||||||
|
'false', 'true',
|
||||||
|
'EOF', 'NULL',
|
||||||
|
]
|
||||||
|
PREDEFINED_VARIABLES = [
|
||||||
|
'this'
|
||||||
|
]
|
||||||
|
DIRECTIVES = [
|
||||||
|
'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
|
||||||
|
'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
|
||||||
|
'volatile'
|
||||||
|
]
|
||||||
|
|
||||||
|
IDENT_KIND = WordList.new(:ident).
|
||||||
|
add(RESERVED_WORDS, :reserved).
|
||||||
|
add(PREDEFINED_TYPES, :pre_type).
|
||||||
|
add(PREDEFINED_VARIABLES, :local_variable).
|
||||||
|
add(DIRECTIVES, :directive).
|
||||||
|
add(PREDEFINED_CONSTANTS, :pre_constant)
|
||||||
|
|
||||||
|
ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
label_expected = true
|
||||||
|
case_expected = false
|
||||||
|
label_expected_before_preproc_line = nil
|
||||||
|
in_preproc_line = false
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
case state
|
||||||
|
|
||||||
|
when :initial
|
||||||
|
|
||||||
|
if match = scan(/ \s+ | \\\n /x)
|
||||||
|
if in_preproc_line && match != "\\\n" && match.index(?\n)
|
||||||
|
in_preproc_line = false
|
||||||
|
label_expected = label_expected_before_preproc_line
|
||||||
|
end
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif match = scan(/ \# \s* if \s* 0 /x)
|
||||||
|
match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
|
||||||
|
label_expected = match =~ /[;\{\}]/
|
||||||
|
if case_expected
|
||||||
|
label_expected = true if match == ':'
|
||||||
|
case_expected = false
|
||||||
|
end
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
|
||||||
|
kind = IDENT_KIND[match]
|
||||||
|
if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
|
||||||
|
kind = :label
|
||||||
|
match << matched
|
||||||
|
else
|
||||||
|
label_expected = false
|
||||||
|
if kind == :reserved
|
||||||
|
case match
|
||||||
|
when 'class'
|
||||||
|
state = :class_name_expected
|
||||||
|
when 'case', 'default'
|
||||||
|
case_expected = true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/\$/)
|
||||||
|
kind = :ident
|
||||||
|
|
||||||
|
elsif match = scan(/L?"/)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
if match[0] == ?L
|
||||||
|
tokens << ['L', :modifier]
|
||||||
|
match = '"'
|
||||||
|
end
|
||||||
|
state = :string
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif scan(/#[ \t]*(\w*)/)
|
||||||
|
kind = :preprocessor
|
||||||
|
in_preproc_line = true
|
||||||
|
label_expected_before_preproc_line = label_expected
|
||||||
|
state = :include_expected if self[1] == 'include'
|
||||||
|
|
||||||
|
elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
|
||||||
|
label_expected = false
|
||||||
|
kind = :char
|
||||||
|
|
||||||
|
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :hex
|
||||||
|
|
||||||
|
elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :oct
|
||||||
|
|
||||||
|
elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :integer
|
||||||
|
|
||||||
|
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :float
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
when :string
|
||||||
|
if scan(/[^\\"]+/)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/"/)
|
||||||
|
tokens << ['"', :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
state = :initial
|
||||||
|
label_expected = false
|
||||||
|
next
|
||||||
|
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/ \\ | $ /x)
|
||||||
|
tokens << [:close, :string]
|
||||||
|
kind = :error
|
||||||
|
state = :initial
|
||||||
|
label_expected = false
|
||||||
|
else
|
||||||
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
when :include_expected
|
||||||
|
if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
|
||||||
|
kind = :include
|
||||||
|
state = :initial
|
||||||
|
|
||||||
|
elsif match = scan(/\s+/)
|
||||||
|
kind = :space
|
||||||
|
state = :initial if match.index ?\n
|
||||||
|
|
||||||
|
else
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
when :class_name_expected
|
||||||
|
if scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
|
||||||
|
kind = :class
|
||||||
|
state = :initial
|
||||||
|
|
||||||
|
elsif match = scan(/\s+/)
|
||||||
|
kind = :space
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
state = :initial
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
if state == :string
|
||||||
|
tokens << [:close, :string]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,210 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
class CSS < Scanner
|
||||||
|
|
||||||
|
register_for :css
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = [
|
||||||
|
:comment,
|
||||||
|
:class, :pseudo_class, :type,
|
||||||
|
:constant, :directive,
|
||||||
|
:key, :value, :operator, :color, :float,
|
||||||
|
:error, :important,
|
||||||
|
]
|
||||||
|
|
||||||
|
module RE
|
||||||
|
NonASCII = /[\x80-\xFF]/
|
||||||
|
Hex = /[0-9a-fA-F]/
|
||||||
|
Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
|
||||||
|
Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
|
||||||
|
NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
|
||||||
|
NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
|
||||||
|
NL = /\r\n|\r|\n|\f/
|
||||||
|
String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # FIXME: buggy regexp
|
||||||
|
String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # FIXME: buggy regexp
|
||||||
|
String = /#{String1}|#{String2}/
|
||||||
|
|
||||||
|
HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
|
||||||
|
Color = /#{HexColor}/
|
||||||
|
|
||||||
|
Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
|
||||||
|
Name = /#{NMChar}+/
|
||||||
|
Ident = /-?#{NMStart}#{NMChar}*/
|
||||||
|
AtKeyword = /@#{Ident}/
|
||||||
|
Percentage = /#{Num}%/
|
||||||
|
|
||||||
|
reldimensions = %w[em ex px]
|
||||||
|
absdimensions = %w[in cm mm pt pc]
|
||||||
|
Unit = Regexp.union(*(reldimensions + absdimensions))
|
||||||
|
|
||||||
|
Dimension = /#{Num}#{Unit}/
|
||||||
|
|
||||||
|
Comment = %r! /\* (?: .*? \*/ | .* ) !mx
|
||||||
|
Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
|
||||||
|
|
||||||
|
Id = /##{Name}/
|
||||||
|
Class = /\.#{Name}/
|
||||||
|
PseudoClass = /:#{Name}/
|
||||||
|
AttributeSelector = /\[[^\]]*\]?/
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
value_expected = nil
|
||||||
|
states = [:initial]
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
if scan(/\s+/)
|
||||||
|
kind = :space
|
||||||
|
|
||||||
|
elsif case states.last
|
||||||
|
when :initial, :media
|
||||||
|
if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
|
||||||
|
kind = :type
|
||||||
|
elsif scan RE::Class
|
||||||
|
kind = :class
|
||||||
|
elsif scan RE::Id
|
||||||
|
kind = :constant
|
||||||
|
elsif scan RE::PseudoClass
|
||||||
|
kind = :pseudo_class
|
||||||
|
elsif match = scan(RE::AttributeSelector)
|
||||||
|
# TODO: Improve highlighting inside of attribute selectors.
|
||||||
|
tokens << [:open, :string]
|
||||||
|
tokens << [match[0,1], :delimiter]
|
||||||
|
tokens << [match[1..-2], :content] if match.size > 2
|
||||||
|
tokens << [match[-1,1], :delimiter] if match[-1] == ?]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
next
|
||||||
|
elsif match = scan(/@media/)
|
||||||
|
kind = :directive
|
||||||
|
states.push :media_before_name
|
||||||
|
end
|
||||||
|
|
||||||
|
when :block
|
||||||
|
if scan(/(?>#{RE::Ident})(?!\()/ox)
|
||||||
|
if value_expected
|
||||||
|
kind = :value
|
||||||
|
else
|
||||||
|
kind = :key
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
when :media_before_name
|
||||||
|
if scan RE::Ident
|
||||||
|
kind = :type
|
||||||
|
states[-1] = :media_after_name
|
||||||
|
end
|
||||||
|
|
||||||
|
when :media_after_name
|
||||||
|
if scan(/\{/)
|
||||||
|
kind = :operator
|
||||||
|
states[-1] = :media
|
||||||
|
end
|
||||||
|
|
||||||
|
when :comment
|
||||||
|
if scan(/(?:[^*\s]|\*(?!\/))+/)
|
||||||
|
kind = :comment
|
||||||
|
elsif scan(/\*\//)
|
||||||
|
kind = :comment
|
||||||
|
states.pop
|
||||||
|
elsif scan(/\s+/)
|
||||||
|
kind = :space
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/\/\*/)
|
||||||
|
kind = :comment
|
||||||
|
states.push :comment
|
||||||
|
|
||||||
|
elsif scan(/\{/)
|
||||||
|
value_expected = false
|
||||||
|
kind = :operator
|
||||||
|
states.push :block
|
||||||
|
|
||||||
|
elsif scan(/\}/)
|
||||||
|
value_expected = false
|
||||||
|
if states.last == :block || states.last == :media
|
||||||
|
kind = :operator
|
||||||
|
states.pop
|
||||||
|
else
|
||||||
|
kind = :error
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif match = scan(/#{RE::String}/o)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
tokens << [match[0, 1], :delimiter]
|
||||||
|
tokens << [match[1..-2], :content] if match.size > 2
|
||||||
|
tokens << [match[-1, 1], :delimiter] if match.size >= 2
|
||||||
|
tokens << [:close, :string]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif match = scan(/#{RE::Function}/o)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
start = match[/^\w+\(/]
|
||||||
|
tokens << [start, :delimiter]
|
||||||
|
if match[-1] == ?)
|
||||||
|
tokens << [match[start.size..-2], :content]
|
||||||
|
tokens << [')', :delimiter]
|
||||||
|
else
|
||||||
|
tokens << [match[start.size..-1], :content]
|
||||||
|
end
|
||||||
|
tokens << [:close, :string]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
|
||||||
|
kind = :float
|
||||||
|
|
||||||
|
elsif scan(/#{RE::Color}/o)
|
||||||
|
kind = :color
|
||||||
|
|
||||||
|
elsif scan(/! *important/)
|
||||||
|
kind = :important
|
||||||
|
|
||||||
|
elsif scan(/rgb\([^()\n]*\)?/)
|
||||||
|
kind = :color
|
||||||
|
|
||||||
|
elsif scan(/#{RE::AtKeyword}/o)
|
||||||
|
kind = :directive
|
||||||
|
|
||||||
|
elsif match = scan(/ [+>:;,.=()\/] /x)
|
||||||
|
if match == ':'
|
||||||
|
value_expected = true
|
||||||
|
elsif match == ';'
|
||||||
|
value_expected = false
|
||||||
|
end
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -6,6 +6,8 @@ module Scanners
|
||||||
|
|
||||||
include Streamable
|
include Streamable
|
||||||
register_for :debug
|
register_for :debug
|
||||||
|
file_extension 'raydebug'
|
||||||
|
title 'CodeRay Token Dump'
|
||||||
|
|
||||||
protected
|
protected
|
||||||
def scan_tokens tokens, options
|
def scan_tokens tokens, options
|
||||||
|
@ -30,8 +32,8 @@ module Scanners
|
||||||
opened_tokens << kind
|
opened_tokens << kind
|
||||||
match = :open
|
match = :open
|
||||||
|
|
||||||
elsif scan(/ > /x)
|
elsif !opened_tokens.empty? && scan(/ > /x)
|
||||||
kind = opened_tokens.pop
|
kind = opened_tokens.pop || :error
|
||||||
match = :close
|
match = :close
|
||||||
|
|
||||||
else
|
else
|
||||||
|
@ -41,7 +43,7 @@ module Scanners
|
||||||
end
|
end
|
||||||
|
|
||||||
match ||= matched
|
match ||= matched
|
||||||
if $DEBUG and not kind
|
if $CODERAY_DEBUG and not kind
|
||||||
raise_inspect 'Error token %p in line %d' %
|
raise_inspect 'Error token %p in line %d' %
|
||||||
[[match, kind], line], tokens
|
[[match, kind], line], tokens
|
||||||
end
|
end
|
|
@ -4,6 +4,7 @@ module Scanners
|
||||||
class Delphi < Scanner
|
class Delphi < Scanner
|
||||||
|
|
||||||
register_for :delphi
|
register_for :delphi
|
||||||
|
file_extension 'pas'
|
||||||
|
|
||||||
RESERVED_WORDS = [
|
RESERVED_WORDS = [
|
||||||
'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
|
'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
|
||||||
|
@ -29,11 +30,11 @@ module Scanners
|
||||||
'virtual', 'write', 'writeonly'
|
'virtual', 'write', 'writeonly'
|
||||||
]
|
]
|
||||||
|
|
||||||
IDENT_KIND = CaseIgnoringWordList.new(:ident, caching=true).
|
IDENT_KIND = CaseIgnoringWordList.new(:ident).
|
||||||
add(RESERVED_WORDS, :reserved).
|
add(RESERVED_WORDS, :reserved).
|
||||||
add(DIRECTIVES, :directive)
|
add(DIRECTIVES, :directive)
|
||||||
|
|
||||||
NAME_FOLLOWS = CaseIgnoringWordList.new(false, caching=true).
|
NAME_FOLLOWS = CaseIgnoringWordList.new(false).
|
||||||
add(%w(procedure function .))
|
add(%w(procedure function .))
|
||||||
|
|
||||||
private
|
private
|
||||||
|
@ -129,7 +130,7 @@ module Scanners
|
||||||
end
|
end
|
||||||
|
|
||||||
match ||= matched
|
match ||= matched
|
||||||
if $DEBUG and not kind
|
if $CODERAY_DEBUG and not kind
|
||||||
raise_inspect 'Error token %p in line %d' %
|
raise_inspect 'Error token %p in line %d' %
|
||||||
[[match, kind], line], tokens, state
|
[[match, kind], line], tokens, state
|
||||||
end
|
end
|
|
@ -0,0 +1,105 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
class Diff < Scanner
|
||||||
|
|
||||||
|
register_for :diff
|
||||||
|
title 'diff output'
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
line_kind = nil
|
||||||
|
state = :initial
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
kind = match = nil
|
||||||
|
|
||||||
|
if match = scan(/\n/)
|
||||||
|
if line_kind
|
||||||
|
tokens << [:end_line, line_kind]
|
||||||
|
line_kind = nil
|
||||||
|
end
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
case state
|
||||||
|
|
||||||
|
when :initial
|
||||||
|
if match = scan(/--- |\+\+\+ |=+|_+/)
|
||||||
|
tokens << [:begin_line, line_kind = :head]
|
||||||
|
tokens << [match, :head]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
elsif match = scan(/Index: |Property changes on: /)
|
||||||
|
tokens << [:begin_line, line_kind = :head]
|
||||||
|
tokens << [match, :head]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
elsif match = scan(/Added: /)
|
||||||
|
tokens << [:begin_line, line_kind = :head]
|
||||||
|
tokens << [match, :head]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
state = :added
|
||||||
|
elsif match = scan(/\\ /)
|
||||||
|
tokens << [:begin_line, line_kind = :change]
|
||||||
|
tokens << [match, :change]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
elsif scan(/(@@)((?>[^@\n]*))(@@)/)
|
||||||
|
tokens << [:begin_line, line_kind = :change]
|
||||||
|
tokens << [self[1], :change]
|
||||||
|
tokens << [self[2], :plain]
|
||||||
|
tokens << [self[3], :change]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
elsif match = scan(/\+/)
|
||||||
|
tokens << [:begin_line, line_kind = :insert]
|
||||||
|
tokens << [match, :insert]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
elsif match = scan(/-/)
|
||||||
|
tokens << [:begin_line, line_kind = :delete]
|
||||||
|
tokens << [match, :delete]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
elsif scan(/ .*/)
|
||||||
|
kind = :comment
|
||||||
|
elsif scan(/.+/)
|
||||||
|
tokens << [:begin_line, line_kind = :head]
|
||||||
|
kind = :plain
|
||||||
|
else
|
||||||
|
raise_inspect 'else case rached'
|
||||||
|
end
|
||||||
|
|
||||||
|
when :added
|
||||||
|
if match = scan(/ \+/)
|
||||||
|
tokens << [:begin_line, line_kind = :insert]
|
||||||
|
tokens << [match, :insert]
|
||||||
|
next unless match = scan(/.+/)
|
||||||
|
kind = :plain
|
||||||
|
else
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens << [:end_line, line_kind] if line_kind
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,264 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
load :java
|
||||||
|
|
||||||
|
class Groovy < Java
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
register_for :groovy
|
||||||
|
|
||||||
|
# TODO: Check this!
|
||||||
|
GROOVY_KEYWORDS = %w[
|
||||||
|
as assert def in
|
||||||
|
]
|
||||||
|
KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
|
||||||
|
case instanceof new return throw typeof while as assert in
|
||||||
|
]
|
||||||
|
GROOVY_MAGIC_VARIABLES = %w[ it ]
|
||||||
|
|
||||||
|
IDENT_KIND = Java::IDENT_KIND.dup.
|
||||||
|
add(GROOVY_KEYWORDS, :keyword).
|
||||||
|
add(GROOVY_MAGIC_VARIABLES, :local_variable)
|
||||||
|
|
||||||
|
ESCAPE = / [bfnrtv$\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # no 4-byte unicode chars? U[a-fA-F0-9]{8}
|
||||||
|
REGEXP_ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | \d | [bBdDsSwW\/] /x
|
||||||
|
|
||||||
|
# TODO: interpretation inside ', ", /
|
||||||
|
STRING_CONTENT_PATTERN = {
|
||||||
|
"'" => /(?>\\[^\\'\n]+|[^\\'\n]+)+/,
|
||||||
|
'"' => /[^\\$"\n]+/,
|
||||||
|
"'''" => /(?>[^\\']+|'(?!''))+/,
|
||||||
|
'"""' => /(?>[^\\$"]+|"(?!""))+/,
|
||||||
|
'/' => /[^\\$\/\n]+/,
|
||||||
|
}
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
inline_block_stack = []
|
||||||
|
inline_block_paren_depth = nil
|
||||||
|
string_delimiter = nil
|
||||||
|
import_clause = class_name_follows = last_token = after_def = false
|
||||||
|
value_expected = true
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
case state
|
||||||
|
|
||||||
|
when :initial
|
||||||
|
|
||||||
|
if match = scan(/ \s+ | \\\n /x)
|
||||||
|
tokens << [match, :space]
|
||||||
|
if match.index ?\n
|
||||||
|
import_clause = after_def = false
|
||||||
|
value_expected = true unless value_expected
|
||||||
|
end
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
||||||
|
value_expected = true
|
||||||
|
after_def = false
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif bol? && scan(/ \#!.* /x)
|
||||||
|
kind = :doctype
|
||||||
|
|
||||||
|
elsif import_clause && scan(/ (?!as) #{IDENT} (?: \. #{IDENT} )* (?: \.\* )? /ox)
|
||||||
|
after_def = value_expected = false
|
||||||
|
kind = :include
|
||||||
|
|
||||||
|
elsif match = scan(/ #{IDENT} | \[\] /ox)
|
||||||
|
kind = IDENT_KIND[match]
|
||||||
|
value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
|
||||||
|
if last_token == '.'
|
||||||
|
kind = :ident
|
||||||
|
elsif class_name_follows
|
||||||
|
kind = :class
|
||||||
|
class_name_follows = false
|
||||||
|
elsif after_def && check(/\s*[({]/)
|
||||||
|
kind = :method
|
||||||
|
after_def = false
|
||||||
|
elsif kind == :ident && last_token != '?' && check(/:/)
|
||||||
|
kind = :key
|
||||||
|
else
|
||||||
|
class_name_follows = true if match == 'class' || (import_clause && match == 'as')
|
||||||
|
import_clause = match == 'import'
|
||||||
|
after_def = true if match == 'def'
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/;/)
|
||||||
|
import_clause = after_def = false
|
||||||
|
value_expected = true
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif scan(/\{/)
|
||||||
|
class_name_follows = after_def = false
|
||||||
|
value_expected = true
|
||||||
|
kind = :operator
|
||||||
|
if !inline_block_stack.empty?
|
||||||
|
inline_block_paren_depth += 1
|
||||||
|
end
|
||||||
|
|
||||||
|
# TODO: ~'...', ~"..." and ~/.../ style regexps
|
||||||
|
elsif match = scan(/ \.\.<? | \*?\.(?!\d)@? | \.& | \?:? | [,?:(\[] | -[->] | \+\+ |
|
||||||
|
&& | \|\| | \*\*=? | ==?~ | <=?>? | [-+*%^~&|>=!]=? | <<<?=? | >>>?=? /x)
|
||||||
|
value_expected = true
|
||||||
|
value_expected = :regexp if match == '~'
|
||||||
|
after_def = false
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif match = scan(/ [)\]}] /x)
|
||||||
|
value_expected = after_def = false
|
||||||
|
if !inline_block_stack.empty? && match == '}'
|
||||||
|
inline_block_paren_depth -= 1
|
||||||
|
if inline_block_paren_depth == 0 # closing brace of inline block reached
|
||||||
|
tokens << [match, :inline_delimiter]
|
||||||
|
tokens << [:close, :inline]
|
||||||
|
state, string_delimiter, inline_block_paren_depth = inline_block_stack.pop
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif check(/[\d.]/)
|
||||||
|
after_def = value_expected = false
|
||||||
|
if scan(/0[xX][0-9A-Fa-f]+/)
|
||||||
|
kind = :hex
|
||||||
|
elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
|
||||||
|
kind = :oct
|
||||||
|
elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
|
||||||
|
kind = :float
|
||||||
|
elsif scan(/\d+[lLgG]?/)
|
||||||
|
kind = :integer
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif match = scan(/'''|"""/)
|
||||||
|
after_def = value_expected = false
|
||||||
|
state = :multiline_string
|
||||||
|
tokens << [:open, :string]
|
||||||
|
string_delimiter = match
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
# TODO: record.'name'
|
||||||
|
elsif match = scan(/["']/)
|
||||||
|
after_def = value_expected = false
|
||||||
|
state = match == '/' ? :regexp : :string
|
||||||
|
tokens << [:open, state]
|
||||||
|
string_delimiter = match
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif value_expected && (match = scan(/\//))
|
||||||
|
after_def = value_expected = false
|
||||||
|
tokens << [:open, :regexp]
|
||||||
|
state = :regexp
|
||||||
|
string_delimiter = '/'
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif scan(/ @ #{IDENT} /ox)
|
||||||
|
after_def = value_expected = false
|
||||||
|
kind = :annotation
|
||||||
|
|
||||||
|
elsif scan(/\//)
|
||||||
|
after_def = false
|
||||||
|
value_expected = true
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
when :string, :regexp, :multiline_string
|
||||||
|
if scan(STRING_CONTENT_PATTERN[string_delimiter])
|
||||||
|
kind = :content
|
||||||
|
|
||||||
|
elsif match = scan(state == :multiline_string ? /'''|"""/ : /["'\/]/)
|
||||||
|
tokens << [match, :delimiter]
|
||||||
|
if state == :regexp
|
||||||
|
# TODO: regexp modifiers? s, m, x, i?
|
||||||
|
modifiers = scan(/[ix]+/)
|
||||||
|
tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
|
||||||
|
end
|
||||||
|
state = :string if state == :multiline_string
|
||||||
|
tokens << [:close, state]
|
||||||
|
string_delimiter = nil
|
||||||
|
after_def = value_expected = false
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif (state == :string || state == :multiline_string) &&
|
||||||
|
(match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
|
||||||
|
if string_delimiter[0] == ?' && !(match == "\\\\" || match == "\\'")
|
||||||
|
kind = :content
|
||||||
|
else
|
||||||
|
kind = :char
|
||||||
|
end
|
||||||
|
elsif state == :regexp && scan(/ \\ (?: #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||||
|
kind = :char
|
||||||
|
|
||||||
|
elsif match = scan(/ \$ #{IDENT} /mox)
|
||||||
|
tokens << [:open, :inline]
|
||||||
|
tokens << ['$', :inline_delimiter]
|
||||||
|
match = match[1..-1]
|
||||||
|
tokens << [match, IDENT_KIND[match]]
|
||||||
|
tokens << [:close, :inline]
|
||||||
|
next
|
||||||
|
elsif match = scan(/ \$ \{ /x)
|
||||||
|
tokens << [:open, :inline]
|
||||||
|
tokens << ['${', :inline_delimiter]
|
||||||
|
inline_block_stack << [state, string_delimiter, inline_block_paren_depth]
|
||||||
|
inline_block_paren_depth = 1
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif scan(/ \$ /mx)
|
||||||
|
kind = :content
|
||||||
|
|
||||||
|
elsif scan(/ \\. /mx)
|
||||||
|
kind = :content
|
||||||
|
|
||||||
|
elsif scan(/ \\ | \n /x)
|
||||||
|
tokens << [:close, state]
|
||||||
|
kind = :error
|
||||||
|
after_def = value_expected = false
|
||||||
|
state = :initial
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
last_token = match unless [:space, :comment, :doctype].include? kind
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
if [:multiline_string, :string, :regexp].include? state
|
||||||
|
tokens << [:close, state]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -2,12 +2,17 @@ module CodeRay
|
||||||
module Scanners
|
module Scanners
|
||||||
|
|
||||||
# HTML Scanner
|
# HTML Scanner
|
||||||
#
|
|
||||||
# $Id$
|
|
||||||
class HTML < Scanner
|
class HTML < Scanner
|
||||||
|
|
||||||
include Streamable
|
include Streamable
|
||||||
register_for :html
|
register_for :html
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = [
|
||||||
|
:comment, :doctype, :preprocessor,
|
||||||
|
:tag, :attribute_name, :operator,
|
||||||
|
:attribute_value, :delimiter, :content,
|
||||||
|
:plain, :entity, :error
|
||||||
|
]
|
||||||
|
|
||||||
ATTR_NAME = /[\w.:-]+/
|
ATTR_NAME = /[\w.:-]+/
|
||||||
ATTR_VALUE_UNQUOTED = ATTR_NAME
|
ATTR_VALUE_UNQUOTED = ATTR_NAME
|
||||||
|
@ -65,14 +70,14 @@ module Scanners
|
||||||
if scan(/<!--.*?-->/m)
|
if scan(/<!--.*?-->/m)
|
||||||
kind = :comment
|
kind = :comment
|
||||||
elsif scan(/<!DOCTYPE.*?>/m)
|
elsif scan(/<!DOCTYPE.*?>/m)
|
||||||
kind = :preprocessor
|
kind = :doctype
|
||||||
elsif scan(/<\?xml.*?\?>/m)
|
elsif scan(/<\?xml.*?\?>/m)
|
||||||
kind = :preprocessor
|
kind = :preprocessor
|
||||||
elsif scan(/<\?.*?\?>|<%.*?%>/m)
|
elsif scan(/<\?.*?\?>|<%.*?%>/m)
|
||||||
kind = :comment
|
kind = :comment
|
||||||
elsif scan(/<\/[-\w_.:]*>/m)
|
elsif scan(/<\/[-\w.:]*>/m)
|
||||||
kind = :tag
|
kind = :tag
|
||||||
elsif match = scan(/<[-\w_.:]+>?/m)
|
elsif match = scan(/<[-\w.:]+>?/m)
|
||||||
kind = :tag
|
kind = :tag
|
||||||
state = :attribute unless match[-1] == ?>
|
state = :attribute unless match[-1] == ?>
|
||||||
elsif scan(/[^<>&]+/)
|
elsif scan(/[^<>&]+/)
|
||||||
|
@ -154,7 +159,7 @@ module Scanners
|
||||||
end
|
end
|
||||||
|
|
||||||
match ||= matched
|
match ||= matched
|
||||||
if $DEBUG and not kind
|
if $CODERAY_DEBUG and not kind
|
||||||
raise_inspect 'Error token %p in line %d' %
|
raise_inspect 'Error token %p in line %d' %
|
||||||
[[match, kind], line], tokens, state
|
[[match, kind], line], tokens, state
|
||||||
end
|
end
|
|
@ -0,0 +1,176 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
class Java < Scanner
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
register_for :java
|
||||||
|
helper :builtin_types
|
||||||
|
|
||||||
|
# http://java.sun.com/docs/books/tutorial/java/nutsandbolts/_keywords.html
|
||||||
|
KEYWORDS = %w[
|
||||||
|
assert break case catch continue default do else
|
||||||
|
finally for if instanceof import new package
|
||||||
|
return switch throw try typeof while
|
||||||
|
debugger export
|
||||||
|
]
|
||||||
|
RESERVED = %w[ const goto ]
|
||||||
|
CONSTANTS = %w[ false null true ]
|
||||||
|
MAGIC_VARIABLES = %w[ this super ]
|
||||||
|
TYPES = %w[
|
||||||
|
boolean byte char class double enum float int interface long
|
||||||
|
short void
|
||||||
|
] << '[]' # because int[] should be highlighted as a type
|
||||||
|
DIRECTIVES = %w[
|
||||||
|
abstract extends final implements native private protected public
|
||||||
|
static strictfp synchronized throws transient volatile
|
||||||
|
]
|
||||||
|
|
||||||
|
IDENT_KIND = WordList.new(:ident).
|
||||||
|
add(KEYWORDS, :keyword).
|
||||||
|
add(RESERVED, :reserved).
|
||||||
|
add(CONSTANTS, :pre_constant).
|
||||||
|
add(MAGIC_VARIABLES, :local_variable).
|
||||||
|
add(TYPES, :type).
|
||||||
|
add(BuiltinTypes::List, :pre_type).
|
||||||
|
add(BuiltinTypes::List.select { |builtin| builtin[/(Error|Exception)$/] }, :exception).
|
||||||
|
add(DIRECTIVES, :directive)
|
||||||
|
|
||||||
|
ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||||
|
STRING_CONTENT_PATTERN = {
|
||||||
|
"'" => /[^\\']+/,
|
||||||
|
'"' => /[^\\"]+/,
|
||||||
|
'/' => /[^\\\/]+/,
|
||||||
|
}
|
||||||
|
IDENT = /[a-zA-Z_][A-Za-z_0-9]*/
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
string_delimiter = nil
|
||||||
|
import_clause = class_name_follows = last_token_dot = false
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
case state
|
||||||
|
|
||||||
|
when :initial
|
||||||
|
|
||||||
|
if match = scan(/ \s+ | \\\n /x)
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif match = scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
||||||
|
tokens << [match, :comment]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif import_clause && scan(/ #{IDENT} (?: \. #{IDENT} )* /ox)
|
||||||
|
kind = :include
|
||||||
|
|
||||||
|
elsif match = scan(/ #{IDENT} | \[\] /ox)
|
||||||
|
kind = IDENT_KIND[match]
|
||||||
|
if last_token_dot
|
||||||
|
kind = :ident
|
||||||
|
elsif class_name_follows
|
||||||
|
kind = :class
|
||||||
|
class_name_follows = false
|
||||||
|
else
|
||||||
|
import_clause = true if match == 'import'
|
||||||
|
class_name_follows = true if match == 'class' || match == 'interface'
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/ \.(?!\d) | [,?:()\[\]}] | -- | \+\+ | && | \|\| | \*\*=? | [-+*\/%^~&|<>=!]=? | <<<?=? | >>>?=? /x)
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif scan(/;/)
|
||||||
|
import_clause = false
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif scan(/\{/)
|
||||||
|
class_name_follows = false
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif check(/[\d.]/)
|
||||||
|
if scan(/0[xX][0-9A-Fa-f]+/)
|
||||||
|
kind = :hex
|
||||||
|
elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
|
||||||
|
kind = :oct
|
||||||
|
elsif scan(/\d+[fFdD]|\d*\.\d+(?:[eE][+-]?\d+)?[fFdD]?|\d+[eE][+-]?\d+[fFdD]?/)
|
||||||
|
kind = :float
|
||||||
|
elsif scan(/\d+[lL]?/)
|
||||||
|
kind = :integer
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif match = scan(/["']/)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
state = :string
|
||||||
|
string_delimiter = match
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif scan(/ @ #{IDENT} /ox)
|
||||||
|
kind = :annotation
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
when :string
|
||||||
|
if scan(STRING_CONTENT_PATTERN[string_delimiter])
|
||||||
|
kind = :content
|
||||||
|
elsif match = scan(/["'\/]/)
|
||||||
|
tokens << [match, :delimiter]
|
||||||
|
tokens << [:close, state]
|
||||||
|
string_delimiter = nil
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
elsif state == :string && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
|
||||||
|
if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
|
||||||
|
kind = :content
|
||||||
|
else
|
||||||
|
kind = :char
|
||||||
|
end
|
||||||
|
elsif scan(/\\./m)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/ \\ | $ /x)
|
||||||
|
tokens << [:close, :delimiter]
|
||||||
|
kind = :error
|
||||||
|
state = :initial
|
||||||
|
else
|
||||||
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
last_token_dot = match == '.'
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
if state == :string
|
||||||
|
tokens << [:close, state]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,419 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
module Java::BuiltinTypes # :nodoc:
|
||||||
|
|
||||||
|
List = %w[
|
||||||
|
AbstractAction AbstractBorder AbstractButton AbstractCellEditor AbstractCollection
|
||||||
|
AbstractColorChooserPanel AbstractDocument AbstractExecutorService AbstractInterruptibleChannel
|
||||||
|
AbstractLayoutCache AbstractList AbstractListModel AbstractMap AbstractMethodError AbstractPreferences
|
||||||
|
AbstractQueue AbstractQueuedSynchronizer AbstractSelectableChannel AbstractSelectionKey AbstractSelector
|
||||||
|
AbstractSequentialList AbstractSet AbstractSpinnerModel AbstractTableModel AbstractUndoableEdit
|
||||||
|
AbstractWriter AccessControlContext AccessControlException AccessController AccessException Accessible
|
||||||
|
AccessibleAction AccessibleAttributeSequence AccessibleBundle AccessibleComponent AccessibleContext
|
||||||
|
AccessibleEditableText AccessibleExtendedComponent AccessibleExtendedTable AccessibleExtendedText
|
||||||
|
AccessibleHyperlink AccessibleHypertext AccessibleIcon AccessibleKeyBinding AccessibleObject
|
||||||
|
AccessibleRelation AccessibleRelationSet AccessibleResourceBundle AccessibleRole AccessibleSelection
|
||||||
|
AccessibleState AccessibleStateSet AccessibleStreamable AccessibleTable AccessibleTableModelChange
|
||||||
|
AccessibleText AccessibleTextSequence AccessibleValue AccountException AccountExpiredException
|
||||||
|
AccountLockedException AccountNotFoundException Acl AclEntry AclNotFoundException Action ActionEvent
|
||||||
|
ActionListener ActionMap ActionMapUIResource Activatable ActivateFailedException ActivationDesc
|
||||||
|
ActivationException ActivationGroup ActivationGroupDesc ActivationGroupID ActivationGroup_Stub
|
||||||
|
ActivationID ActivationInstantiator ActivationMonitor ActivationSystem Activator ActiveEvent
|
||||||
|
ActivityCompletedException ActivityRequiredException Adjustable AdjustmentEvent AdjustmentListener
|
||||||
|
Adler32 AffineTransform AffineTransformOp AlgorithmParameterGenerator AlgorithmParameterGeneratorSpi
|
||||||
|
AlgorithmParameters AlgorithmParameterSpec AlgorithmParametersSpi AllPermission AlphaComposite
|
||||||
|
AlreadyBoundException AlreadyConnectedException AncestorEvent AncestorListener AnnotatedElement
|
||||||
|
Annotation AnnotationFormatError AnnotationTypeMismatchException AppConfigurationEntry Appendable Applet
|
||||||
|
AppletContext AppletInitializer AppletStub Arc2D Area AreaAveragingScaleFilter ArithmeticException Array
|
||||||
|
ArrayBlockingQueue ArrayIndexOutOfBoundsException ArrayList Arrays ArrayStoreException ArrayType
|
||||||
|
AssertionError AsyncBoxView AsynchronousCloseException AtomicBoolean AtomicInteger AtomicIntegerArray
|
||||||
|
AtomicIntegerFieldUpdater AtomicLong AtomicLongArray AtomicLongFieldUpdater AtomicMarkableReference
|
||||||
|
AtomicReference AtomicReferenceArray AtomicReferenceFieldUpdater AtomicStampedReference Attribute
|
||||||
|
AttributeChangeNotification AttributeChangeNotificationFilter AttributedCharacterIterator
|
||||||
|
AttributedString AttributeException AttributeInUseException AttributeList AttributeModificationException
|
||||||
|
AttributeNotFoundException Attributes AttributeSet AttributeSetUtilities AttributeValueExp AudioClip
|
||||||
|
AudioFileFormat AudioFileReader AudioFileWriter AudioFormat AudioInputStream AudioPermission AudioSystem
|
||||||
|
AuthenticationException AuthenticationNotSupportedException Authenticator AuthorizeCallback
|
||||||
|
AuthPermission AuthProvider Autoscroll AWTError AWTEvent AWTEventListener AWTEventListenerProxy
|
||||||
|
AWTEventMulticaster AWTException AWTKeyStroke AWTPermission BackingStoreException
|
||||||
|
BadAttributeValueExpException BadBinaryOpValueExpException BadLocationException BadPaddingException
|
||||||
|
BadStringOperationException BandCombineOp BandedSampleModel BaseRowSet BasicArrowButton BasicAttribute
|
||||||
|
BasicAttributes BasicBorders BasicButtonListener BasicButtonUI BasicCheckBoxMenuItemUI BasicCheckBoxUI
|
||||||
|
BasicColorChooserUI BasicComboBoxEditor BasicComboBoxRenderer BasicComboBoxUI BasicComboPopup
|
||||||
|
BasicControl BasicDesktopIconUI BasicDesktopPaneUI BasicDirectoryModel BasicEditorPaneUI
|
||||||
|
BasicFileChooserUI BasicFormattedTextFieldUI BasicGraphicsUtils BasicHTML BasicIconFactory
|
||||||
|
BasicInternalFrameTitlePane BasicInternalFrameUI BasicLabelUI BasicListUI BasicLookAndFeel
|
||||||
|
BasicMenuBarUI BasicMenuItemUI BasicMenuUI BasicOptionPaneUI BasicPanelUI BasicPasswordFieldUI
|
||||||
|
BasicPermission BasicPopupMenuSeparatorUI BasicPopupMenuUI BasicProgressBarUI BasicRadioButtonMenuItemUI
|
||||||
|
BasicRadioButtonUI BasicRootPaneUI BasicScrollBarUI BasicScrollPaneUI BasicSeparatorUI BasicSliderUI
|
||||||
|
BasicSpinnerUI BasicSplitPaneDivider BasicSplitPaneUI BasicStroke BasicTabbedPaneUI BasicTableHeaderUI
|
||||||
|
BasicTableUI BasicTextAreaUI BasicTextFieldUI BasicTextPaneUI BasicTextUI BasicToggleButtonUI
|
||||||
|
BasicToolBarSeparatorUI BasicToolBarUI BasicToolTipUI BasicTreeUI BasicViewportUI BatchUpdateException
|
||||||
|
BeanContext BeanContextChild BeanContextChildComponentProxy BeanContextChildSupport
|
||||||
|
BeanContextContainerProxy BeanContextEvent BeanContextMembershipEvent BeanContextMembershipListener
|
||||||
|
BeanContextProxy BeanContextServiceAvailableEvent BeanContextServiceProvider
|
||||||
|
BeanContextServiceProviderBeanInfo BeanContextServiceRevokedEvent BeanContextServiceRevokedListener
|
||||||
|
BeanContextServices BeanContextServicesListener BeanContextServicesSupport BeanContextSupport
|
||||||
|
BeanDescriptor BeanInfo Beans BevelBorder Bidi BigDecimal BigInteger BinaryRefAddr BindException Binding
|
||||||
|
BitSet Blob BlockingQueue BlockView BMPImageWriteParam Book Boolean BooleanControl Border BorderFactory
|
||||||
|
BorderLayout BorderUIResource BoundedRangeModel Box BoxLayout BoxView BreakIterator
|
||||||
|
BrokenBarrierException Buffer BufferCapabilities BufferedImage BufferedImageFilter BufferedImageOp
|
||||||
|
BufferedInputStream BufferedOutputStream BufferedReader BufferedWriter BufferOverflowException
|
||||||
|
BufferStrategy BufferUnderflowException Button ButtonGroup ButtonModel ButtonUI Byte
|
||||||
|
ByteArrayInputStream ByteArrayOutputStream ByteBuffer ByteChannel ByteLookupTable ByteOrder CachedRowSet
|
||||||
|
CacheRequest CacheResponse Calendar Callable CallableStatement Callback CallbackHandler
|
||||||
|
CancelablePrintJob CancellationException CancelledKeyException CannotProceedException
|
||||||
|
CannotRedoException CannotUndoException Canvas CardLayout Caret CaretEvent CaretListener CellEditor
|
||||||
|
CellEditorListener CellRendererPane Certificate CertificateEncodingException CertificateException
|
||||||
|
CertificateExpiredException CertificateFactory CertificateFactorySpi CertificateNotYetValidException
|
||||||
|
CertificateParsingException CertPath CertPathBuilder CertPathBuilderException CertPathBuilderResult
|
||||||
|
CertPathBuilderSpi CertPathParameters CertPathTrustManagerParameters CertPathValidator
|
||||||
|
CertPathValidatorException CertPathValidatorResult CertPathValidatorSpi CertSelector CertStore
|
||||||
|
CertStoreException CertStoreParameters CertStoreSpi ChangedCharSetException ChangeEvent ChangeListener
|
||||||
|
Channel Channels Character CharacterCodingException CharacterIterator CharArrayReader CharArrayWriter
|
||||||
|
CharBuffer CharConversionException CharSequence Charset CharsetDecoder CharsetEncoder CharsetProvider
|
||||||
|
Checkbox CheckboxGroup CheckboxMenuItem CheckedInputStream CheckedOutputStream Checksum Choice
|
||||||
|
ChoiceCallback ChoiceFormat Chromaticity Cipher CipherInputStream CipherOutputStream CipherSpi Class
|
||||||
|
ClassCastException ClassCircularityError ClassDefinition ClassDesc ClassFileTransformer ClassFormatError
|
||||||
|
ClassLoader ClassLoaderRepository ClassLoadingMXBean ClassNotFoundException Clip Clipboard
|
||||||
|
ClipboardOwner Clob Cloneable CloneNotSupportedException Closeable ClosedByInterruptException
|
||||||
|
ClosedChannelException ClosedSelectorException CMMException CoderMalfunctionError CoderResult CodeSigner
|
||||||
|
CodeSource CodingErrorAction CollationElementIterator CollationKey Collator Collection
|
||||||
|
CollectionCertStoreParameters Collections Color ColorChooserComponentFactory ColorChooserUI
|
||||||
|
ColorConvertOp ColorModel ColorSelectionModel ColorSpace ColorSupported ColorType ColorUIResource
|
||||||
|
ComboBoxEditor ComboBoxModel ComboBoxUI ComboPopup CommunicationException Comparable Comparator
|
||||||
|
CompilationMXBean Compiler CompletionService Component ComponentAdapter ComponentColorModel
|
||||||
|
ComponentEvent ComponentInputMap ComponentInputMapUIResource ComponentListener ComponentOrientation
|
||||||
|
ComponentSampleModel ComponentUI ComponentView Composite CompositeContext CompositeData
|
||||||
|
CompositeDataSupport CompositeName CompositeType CompositeView CompoundBorder CompoundControl
|
||||||
|
CompoundEdit CompoundName Compression ConcurrentHashMap ConcurrentLinkedQueue ConcurrentMap
|
||||||
|
ConcurrentModificationException Condition Configuration ConfigurationException ConfirmationCallback
|
||||||
|
ConnectException ConnectIOException Connection ConnectionEvent ConnectionEventListener
|
||||||
|
ConnectionPendingException ConnectionPoolDataSource ConsoleHandler Constructor Container
|
||||||
|
ContainerAdapter ContainerEvent ContainerListener ContainerOrderFocusTraversalPolicy ContentHandler
|
||||||
|
ContentHandlerFactory ContentModel Context ContextNotEmptyException ContextualRenderedImageFactory
|
||||||
|
Control ControlFactory ControllerEventListener ConvolveOp CookieHandler Copies CopiesSupported
|
||||||
|
CopyOnWriteArrayList CopyOnWriteArraySet CountDownLatch CounterMonitor CounterMonitorMBean CRC32
|
||||||
|
CredentialException CredentialExpiredException CredentialNotFoundException CRL CRLException CRLSelector
|
||||||
|
CropImageFilter CSS CubicCurve2D Currency Cursor Customizer CyclicBarrier DatabaseMetaData DataBuffer
|
||||||
|
DataBufferByte DataBufferDouble DataBufferFloat DataBufferInt DataBufferShort DataBufferUShort
|
||||||
|
DataFlavor DataFormatException DatagramChannel DatagramPacket DatagramSocket DatagramSocketImpl
|
||||||
|
DatagramSocketImplFactory DataInput DataInputStream DataLine DataOutput DataOutputStream DataSource
|
||||||
|
DataTruncation DatatypeConfigurationException DatatypeConstants DatatypeFactory Date DateFormat
|
||||||
|
DateFormatSymbols DateFormatter DateTimeAtCompleted DateTimeAtCreation DateTimeAtProcessing
|
||||||
|
DateTimeSyntax DebugGraphics DecimalFormat DecimalFormatSymbols DefaultBoundedRangeModel
|
||||||
|
DefaultButtonModel DefaultCaret DefaultCellEditor DefaultColorSelectionModel DefaultComboBoxModel
|
||||||
|
DefaultDesktopManager DefaultEditorKit DefaultFocusManager DefaultFocusTraversalPolicy DefaultFormatter
|
||||||
|
DefaultFormatterFactory DefaultHighlighter DefaultKeyboardFocusManager DefaultListCellRenderer
|
||||||
|
DefaultListModel DefaultListSelectionModel DefaultLoaderRepository DefaultMenuLayout DefaultMetalTheme
|
||||||
|
DefaultMutableTreeNode DefaultPersistenceDelegate DefaultSingleSelectionModel DefaultStyledDocument
|
||||||
|
DefaultTableCellRenderer DefaultTableColumnModel DefaultTableModel DefaultTextUI DefaultTreeCellEditor
|
||||||
|
DefaultTreeCellRenderer DefaultTreeModel DefaultTreeSelectionModel Deflater DeflaterOutputStream Delayed
|
||||||
|
DelayQueue DelegationPermission Deprecated Descriptor DescriptorAccess DescriptorSupport DESedeKeySpec
|
||||||
|
DesignMode DESKeySpec DesktopIconUI DesktopManager DesktopPaneUI Destination Destroyable
|
||||||
|
DestroyFailedException DGC DHGenParameterSpec DHKey DHParameterSpec DHPrivateKey DHPrivateKeySpec
|
||||||
|
DHPublicKey DHPublicKeySpec Dialog Dictionary DigestException DigestInputStream DigestOutputStream
|
||||||
|
Dimension Dimension2D DimensionUIResource DirContext DirectColorModel DirectoryManager DirObjectFactory
|
||||||
|
DirStateFactory DisplayMode DnDConstants Doc DocAttribute DocAttributeSet DocFlavor DocPrintJob Document
|
||||||
|
DocumentBuilder DocumentBuilderFactory Documented DocumentEvent DocumentFilter DocumentListener
|
||||||
|
DocumentName DocumentParser DomainCombiner DOMLocator DOMResult DOMSource Double DoubleBuffer
|
||||||
|
DragGestureEvent DragGestureListener DragGestureRecognizer DragSource DragSourceAdapter
|
||||||
|
DragSourceContext DragSourceDragEvent DragSourceDropEvent DragSourceEvent DragSourceListener
|
||||||
|
DragSourceMotionListener Driver DriverManager DriverPropertyInfo DropTarget DropTargetAdapter
|
||||||
|
DropTargetContext DropTargetDragEvent DropTargetDropEvent DropTargetEvent DropTargetListener DSAKey
|
||||||
|
DSAKeyPairGenerator DSAParameterSpec DSAParams DSAPrivateKey DSAPrivateKeySpec DSAPublicKey
|
||||||
|
DSAPublicKeySpec DTD DTDConstants DuplicateFormatFlagsException Duration DynamicMBean ECField ECFieldF2m
|
||||||
|
ECFieldFp ECGenParameterSpec ECKey ECParameterSpec ECPoint ECPrivateKey ECPrivateKeySpec ECPublicKey
|
||||||
|
ECPublicKeySpec EditorKit Element ElementIterator ElementType Ellipse2D EllipticCurve EmptyBorder
|
||||||
|
EmptyStackException EncodedKeySpec Encoder EncryptedPrivateKeyInfo Entity Enum
|
||||||
|
EnumConstantNotPresentException EnumControl Enumeration EnumMap EnumSet EnumSyntax EOFException Error
|
||||||
|
ErrorListener ErrorManager EtchedBorder Event EventContext EventDirContext EventHandler EventListener
|
||||||
|
EventListenerList EventListenerProxy EventObject EventQueue EventSetDescriptor Exception
|
||||||
|
ExceptionInInitializerError ExceptionListener Exchanger ExecutionException Executor
|
||||||
|
ExecutorCompletionService Executors ExecutorService ExemptionMechanism ExemptionMechanismException
|
||||||
|
ExemptionMechanismSpi ExpandVetoException ExportException Expression ExtendedRequest ExtendedResponse
|
||||||
|
Externalizable FactoryConfigurationError FailedLoginException FeatureDescriptor Fidelity Field
|
||||||
|
FieldPosition FieldView File FileCacheImageInputStream FileCacheImageOutputStream FileChannel
|
||||||
|
FileChooserUI FileDescriptor FileDialog FileFilter FileHandler FileImageInputStream
|
||||||
|
FileImageOutputStream FileInputStream FileLock FileLockInterruptionException FilenameFilter FileNameMap
|
||||||
|
FileNotFoundException FileOutputStream FilePermission FileReader FileSystemView FileView FileWriter
|
||||||
|
Filter FilteredImageSource FilteredRowSet FilterInputStream FilterOutputStream FilterReader FilterWriter
|
||||||
|
Finishings FixedHeightLayoutCache FlatteningPathIterator FlavorEvent FlavorException FlavorListener
|
||||||
|
FlavorMap FlavorTable Float FloatBuffer FloatControl FlowLayout FlowView Flushable FocusAdapter
|
||||||
|
FocusEvent FocusListener FocusManager FocusTraversalPolicy Font FontFormatException FontMetrics
|
||||||
|
FontRenderContext FontUIResource Format FormatConversionProvider FormatFlagsConversionMismatchException
|
||||||
|
Formattable FormattableFlags Formatter FormatterClosedException FormSubmitEvent FormView Frame Future
|
||||||
|
FutureTask GapContent GarbageCollectorMXBean GatheringByteChannel GaugeMonitor GaugeMonitorMBean
|
||||||
|
GeneralPath GeneralSecurityException GenericArrayType GenericDeclaration GenericSignatureFormatError
|
||||||
|
GlyphJustificationInfo GlyphMetrics GlyphVector GlyphView GradientPaint GraphicAttribute Graphics
|
||||||
|
Graphics2D GraphicsConfigTemplate GraphicsConfiguration GraphicsDevice GraphicsEnvironment GrayFilter
|
||||||
|
GregorianCalendar GridBagConstraints GridBagLayout GridLayout Group Guard GuardedObject GZIPInputStream
|
||||||
|
GZIPOutputStream Handler HandshakeCompletedEvent HandshakeCompletedListener HasControls HashAttributeSet
|
||||||
|
HashDocAttributeSet HashMap HashPrintJobAttributeSet HashPrintRequestAttributeSet
|
||||||
|
HashPrintServiceAttributeSet HashSet Hashtable HeadlessException HierarchyBoundsAdapter
|
||||||
|
HierarchyBoundsListener HierarchyEvent HierarchyListener Highlighter HostnameVerifier HTML HTMLDocument
|
||||||
|
HTMLEditorKit HTMLFrameHyperlinkEvent HTMLWriter HttpRetryException HttpsURLConnection HttpURLConnection
|
||||||
|
HyperlinkEvent HyperlinkListener ICC_ColorSpace ICC_Profile ICC_ProfileGray ICC_ProfileRGB Icon
|
||||||
|
IconUIResource IconView Identity IdentityHashMap IdentityScope IIOByteBuffer IIOException IIOImage
|
||||||
|
IIOInvalidTreeException IIOMetadata IIOMetadataController IIOMetadataFormat IIOMetadataFormatImpl
|
||||||
|
IIOMetadataNode IIOParam IIOParamController IIOReadProgressListener IIOReadUpdateListener
|
||||||
|
IIOReadWarningListener IIORegistry IIOServiceProvider IIOWriteProgressListener IIOWriteWarningListener
|
||||||
|
IllegalAccessError IllegalAccessException IllegalArgumentException IllegalBlockingModeException
|
||||||
|
IllegalBlockSizeException IllegalCharsetNameException IllegalClassFormatException
|
||||||
|
IllegalComponentStateException IllegalFormatCodePointException IllegalFormatConversionException
|
||||||
|
IllegalFormatException IllegalFormatFlagsException IllegalFormatPrecisionException
|
||||||
|
IllegalFormatWidthException IllegalMonitorStateException IllegalPathStateException
|
||||||
|
IllegalSelectorException IllegalStateException IllegalThreadStateException Image ImageCapabilities
|
||||||
|
ImageConsumer ImageFilter ImageGraphicAttribute ImageIcon ImageInputStream ImageInputStreamImpl
|
||||||
|
ImageInputStreamSpi ImageIO ImageObserver ImageOutputStream ImageOutputStreamImpl ImageOutputStreamSpi
|
||||||
|
ImageProducer ImageReader ImageReaderSpi ImageReaderWriterSpi ImageReadParam ImageTranscoder
|
||||||
|
ImageTranscoderSpi ImageTypeSpecifier ImageView ImageWriteParam ImageWriter ImageWriterSpi
|
||||||
|
ImagingOpException IncompatibleClassChangeError IncompleteAnnotationException IndexColorModel
|
||||||
|
IndexedPropertyChangeEvent IndexedPropertyDescriptor IndexOutOfBoundsException Inet4Address Inet6Address
|
||||||
|
InetAddress InetSocketAddress Inflater InflaterInputStream InheritableThreadLocal Inherited
|
||||||
|
InitialContext InitialContextFactory InitialContextFactoryBuilder InitialDirContext InitialLdapContext
|
||||||
|
InlineView InputContext InputEvent InputMap InputMapUIResource InputMethod InputMethodContext
|
||||||
|
InputMethodDescriptor InputMethodEvent InputMethodHighlight InputMethodListener InputMethodRequests
|
||||||
|
InputMismatchException InputStream InputStreamReader InputSubset InputVerifier Insets InsetsUIResource
|
||||||
|
InstanceAlreadyExistsException InstanceNotFoundException InstantiationError InstantiationException
|
||||||
|
Instrument Instrumentation InsufficientResourcesException IntBuffer Integer IntegerSyntax InternalError
|
||||||
|
InternalFrameAdapter InternalFrameEvent InternalFrameFocusTraversalPolicy InternalFrameListener
|
||||||
|
InternalFrameUI InternationalFormatter InterruptedException InterruptedIOException
|
||||||
|
InterruptedNamingException InterruptibleChannel IntrospectionException Introspector
|
||||||
|
InvalidActivityException InvalidAlgorithmParameterException InvalidApplicationException
|
||||||
|
InvalidAttributeIdentifierException InvalidAttributesException InvalidAttributeValueException
|
||||||
|
InvalidClassException InvalidDnDOperationException InvalidKeyException InvalidKeySpecException
|
||||||
|
InvalidMarkException InvalidMidiDataException InvalidNameException InvalidObjectException
|
||||||
|
InvalidOpenTypeException InvalidParameterException InvalidParameterSpecException
|
||||||
|
InvalidPreferencesFormatException InvalidPropertiesFormatException InvalidRelationIdException
|
||||||
|
InvalidRelationServiceException InvalidRelationTypeException InvalidRoleInfoException
|
||||||
|
InvalidRoleValueException InvalidSearchControlsException InvalidSearchFilterException
|
||||||
|
InvalidTargetObjectTypeException InvalidTransactionException InvocationEvent InvocationHandler
|
||||||
|
InvocationTargetException IOException ItemEvent ItemListener ItemSelectable Iterable Iterator
|
||||||
|
IvParameterSpec JApplet JarEntry JarException JarFile JarInputStream JarOutputStream JarURLConnection
|
||||||
|
JButton JCheckBox JCheckBoxMenuItem JColorChooser JComboBox JComponent JdbcRowSet JDesktopPane JDialog
|
||||||
|
JEditorPane JFileChooser JFormattedTextField JFrame JInternalFrame JLabel JLayeredPane JList JMenu
|
||||||
|
JMenuBar JMenuItem JMException JMRuntimeException JMXAuthenticator JMXConnectionNotification
|
||||||
|
JMXConnector JMXConnectorFactory JMXConnectorProvider JMXConnectorServer JMXConnectorServerFactory
|
||||||
|
JMXConnectorServerMBean JMXConnectorServerProvider JMXPrincipal JMXProviderException
|
||||||
|
JMXServerErrorException JMXServiceURL JobAttributes JobHoldUntil JobImpressions JobImpressionsCompleted
|
||||||
|
JobImpressionsSupported JobKOctets JobKOctetsProcessed JobKOctetsSupported JobMediaSheets
|
||||||
|
JobMediaSheetsCompleted JobMediaSheetsSupported JobMessageFromOperator JobName JobOriginatingUserName
|
||||||
|
JobPriority JobPrioritySupported JobSheets JobState JobStateReason JobStateReasons Joinable JoinRowSet
|
||||||
|
JOptionPane JPanel JPasswordField JPEGHuffmanTable JPEGImageReadParam JPEGImageWriteParam JPEGQTable
|
||||||
|
JPopupMenu JProgressBar JRadioButton JRadioButtonMenuItem JRootPane JScrollBar JScrollPane JSeparator
|
||||||
|
JSlider JSpinner JSplitPane JTabbedPane JTable JTableHeader JTextArea JTextComponent JTextField
|
||||||
|
JTextPane JToggleButton JToolBar JToolTip JTree JViewport JWindow KerberosKey KerberosPrincipal
|
||||||
|
KerberosTicket Kernel Key KeyAdapter KeyAgreement KeyAgreementSpi KeyAlreadyExistsException
|
||||||
|
KeyboardFocusManager KeyEvent KeyEventDispatcher KeyEventPostProcessor KeyException KeyFactory
|
||||||
|
KeyFactorySpi KeyGenerator KeyGeneratorSpi KeyListener KeyManagementException KeyManager
|
||||||
|
KeyManagerFactory KeyManagerFactorySpi Keymap KeyPair KeyPairGenerator KeyPairGeneratorSpi KeyRep
|
||||||
|
KeySpec KeyStore KeyStoreBuilderParameters KeyStoreException KeyStoreSpi KeyStroke Label LabelUI
|
||||||
|
LabelView LanguageCallback LastOwnerException LayeredHighlighter LayoutFocusTraversalPolicy
|
||||||
|
LayoutManager LayoutManager2 LayoutQueue LDAPCertStoreParameters LdapContext LdapName
|
||||||
|
LdapReferralException Lease Level LimitExceededException Line Line2D LineBorder LineBreakMeasurer
|
||||||
|
LineEvent LineListener LineMetrics LineNumberInputStream LineNumberReader LineUnavailableException
|
||||||
|
LinkageError LinkedBlockingQueue LinkedHashMap LinkedHashSet LinkedList LinkException LinkLoopException
|
||||||
|
LinkRef List ListCellRenderer ListDataEvent ListDataListener ListenerNotFoundException ListIterator
|
||||||
|
ListModel ListResourceBundle ListSelectionEvent ListSelectionListener ListSelectionModel ListUI ListView
|
||||||
|
LoaderHandler Locale LocateRegistry Lock LockSupport Logger LoggingMXBean LoggingPermission LoginContext
|
||||||
|
LoginException LoginModule LogManager LogRecord LogStream Long LongBuffer LookAndFeel LookupOp
|
||||||
|
LookupTable Mac MacSpi MalformedInputException MalformedLinkException MalformedObjectNameException
|
||||||
|
MalformedParameterizedTypeException MalformedURLException ManagementFactory ManagementPermission
|
||||||
|
ManageReferralControl ManagerFactoryParameters Manifest Map MappedByteBuffer MarshalException
|
||||||
|
MarshalledObject MaskFormatter Matcher MatchResult Math MathContext MatteBorder MBeanAttributeInfo
|
||||||
|
MBeanConstructorInfo MBeanException MBeanFeatureInfo MBeanInfo MBeanNotificationInfo MBeanOperationInfo
|
||||||
|
MBeanParameterInfo MBeanPermission MBeanRegistration MBeanRegistrationException MBeanServer
|
||||||
|
MBeanServerBuilder MBeanServerConnection MBeanServerDelegate MBeanServerDelegateMBean MBeanServerFactory
|
||||||
|
MBeanServerForwarder MBeanServerInvocationHandler MBeanServerNotification MBeanServerNotificationFilter
|
||||||
|
MBeanServerPermission MBeanTrustPermission Media MediaName MediaPrintableArea MediaSize MediaSizeName
|
||||||
|
MediaTracker MediaTray Member MemoryCacheImageInputStream MemoryCacheImageOutputStream MemoryHandler
|
||||||
|
MemoryImageSource MemoryManagerMXBean MemoryMXBean MemoryNotificationInfo MemoryPoolMXBean MemoryType
|
||||||
|
MemoryUsage Menu MenuBar MenuBarUI MenuComponent MenuContainer MenuDragMouseEvent MenuDragMouseListener
|
||||||
|
MenuElement MenuEvent MenuItem MenuItemUI MenuKeyEvent MenuKeyListener MenuListener MenuSelectionManager
|
||||||
|
MenuShortcut MessageDigest MessageDigestSpi MessageFormat MetaEventListener MetalBorders MetalButtonUI
|
||||||
|
MetalCheckBoxIcon MetalCheckBoxUI MetalComboBoxButton MetalComboBoxEditor MetalComboBoxIcon
|
||||||
|
MetalComboBoxUI MetalDesktopIconUI MetalFileChooserUI MetalIconFactory MetalInternalFrameTitlePane
|
||||||
|
MetalInternalFrameUI MetalLabelUI MetalLookAndFeel MetalMenuBarUI MetalPopupMenuSeparatorUI
|
||||||
|
MetalProgressBarUI MetalRadioButtonUI MetalRootPaneUI MetalScrollBarUI MetalScrollButton
|
||||||
|
MetalScrollPaneUI MetalSeparatorUI MetalSliderUI MetalSplitPaneUI MetalTabbedPaneUI MetalTextFieldUI
|
||||||
|
MetalTheme MetalToggleButtonUI MetalToolBarUI MetalToolTipUI MetalTreeUI MetaMessage Method
|
||||||
|
MethodDescriptor MGF1ParameterSpec MidiChannel MidiDevice MidiDeviceProvider MidiEvent MidiFileFormat
|
||||||
|
MidiFileReader MidiFileWriter MidiMessage MidiSystem MidiUnavailableException MimeTypeParseException
|
||||||
|
MinimalHTMLWriter MissingFormatArgumentException MissingFormatWidthException MissingResourceException
|
||||||
|
Mixer MixerProvider MLet MLetMBean ModelMBean ModelMBeanAttributeInfo ModelMBeanConstructorInfo
|
||||||
|
ModelMBeanInfo ModelMBeanInfoSupport ModelMBeanNotificationBroadcaster ModelMBeanNotificationInfo
|
||||||
|
ModelMBeanOperationInfo ModificationItem Modifier Monitor MonitorMBean MonitorNotification
|
||||||
|
MonitorSettingException MouseAdapter MouseDragGestureRecognizer MouseEvent MouseInfo MouseInputAdapter
|
||||||
|
MouseInputListener MouseListener MouseMotionAdapter MouseMotionListener MouseWheelEvent
|
||||||
|
MouseWheelListener MultiButtonUI MulticastSocket MultiColorChooserUI MultiComboBoxUI MultiDesktopIconUI
|
||||||
|
MultiDesktopPaneUI MultiDoc MultiDocPrintJob MultiDocPrintService MultiFileChooserUI
|
||||||
|
MultiInternalFrameUI MultiLabelUI MultiListUI MultiLookAndFeel MultiMenuBarUI MultiMenuItemUI
|
||||||
|
MultiOptionPaneUI MultiPanelUI MultiPixelPackedSampleModel MultipleDocumentHandling MultipleMaster
|
||||||
|
MultiPopupMenuUI MultiProgressBarUI MultiRootPaneUI MultiScrollBarUI MultiScrollPaneUI MultiSeparatorUI
|
||||||
|
MultiSliderUI MultiSpinnerUI MultiSplitPaneUI MultiTabbedPaneUI MultiTableHeaderUI MultiTableUI
|
||||||
|
MultiTextUI MultiToolBarUI MultiToolTipUI MultiTreeUI MultiViewportUI MutableAttributeSet
|
||||||
|
MutableComboBoxModel MutableTreeNode Name NameAlreadyBoundException NameCallback NameClassPair
|
||||||
|
NameNotFoundException NameParser NamespaceChangeListener NamespaceContext Naming NamingEnumeration
|
||||||
|
NamingEvent NamingException NamingExceptionEvent NamingListener NamingManager NamingSecurityException
|
||||||
|
NavigationFilter NegativeArraySizeException NetPermission NetworkInterface NoClassDefFoundError
|
||||||
|
NoConnectionPendingException NodeChangeEvent NodeChangeListener NoInitialContextException
|
||||||
|
NoninvertibleTransformException NonReadableChannelException NonWritableChannelException
|
||||||
|
NoPermissionException NoRouteToHostException NoSuchAlgorithmException NoSuchAttributeException
|
||||||
|
NoSuchElementException NoSuchFieldError NoSuchFieldException NoSuchMethodError NoSuchMethodException
|
||||||
|
NoSuchObjectException NoSuchPaddingException NoSuchProviderException NotActiveException
|
||||||
|
NotBoundException NotCompliantMBeanException NotContextException Notification NotificationBroadcaster
|
||||||
|
NotificationBroadcasterSupport NotificationEmitter NotificationFilter NotificationFilterSupport
|
||||||
|
NotificationListener NotificationResult NotOwnerException NotSerializableException NotYetBoundException
|
||||||
|
NotYetConnectedException NullCipher NullPointerException Number NumberFormat NumberFormatException
|
||||||
|
NumberFormatter NumberOfDocuments NumberOfInterveningJobs NumberUp NumberUpSupported NumericShaper
|
||||||
|
OAEPParameterSpec Object ObjectChangeListener ObjectFactory ObjectFactoryBuilder ObjectInput
|
||||||
|
ObjectInputStream ObjectInputValidation ObjectInstance ObjectName ObjectOutput ObjectOutputStream
|
||||||
|
ObjectStreamClass ObjectStreamConstants ObjectStreamException ObjectStreamField ObjectView ObjID
|
||||||
|
Observable Observer OceanTheme OpenDataException OpenMBeanAttributeInfo OpenMBeanAttributeInfoSupport
|
||||||
|
OpenMBeanConstructorInfo OpenMBeanConstructorInfoSupport OpenMBeanInfo OpenMBeanInfoSupport
|
||||||
|
OpenMBeanOperationInfo OpenMBeanOperationInfoSupport OpenMBeanParameterInfo
|
||||||
|
OpenMBeanParameterInfoSupport OpenType OperatingSystemMXBean Operation OperationNotSupportedException
|
||||||
|
OperationsException Option OptionalDataException OptionPaneUI OrientationRequested OutOfMemoryError
|
||||||
|
OutputDeviceAssigned OutputKeys OutputStream OutputStreamWriter OverlappingFileLockException
|
||||||
|
OverlayLayout Override Owner Pack200 Package PackedColorModel Pageable PageAttributes
|
||||||
|
PagedResultsControl PagedResultsResponseControl PageFormat PageRanges PagesPerMinute PagesPerMinuteColor
|
||||||
|
Paint PaintContext PaintEvent Panel PanelUI Paper ParagraphView ParameterBlock ParameterDescriptor
|
||||||
|
ParameterizedType ParameterMetaData ParseException ParsePosition Parser ParserConfigurationException
|
||||||
|
ParserDelegator PartialResultException PasswordAuthentication PasswordCallback PasswordView Patch
|
||||||
|
PathIterator Pattern PatternSyntaxException PBEKey PBEKeySpec PBEParameterSpec PDLOverrideSupported
|
||||||
|
Permission PermissionCollection Permissions PersistenceDelegate PersistentMBean PhantomReference Pipe
|
||||||
|
PipedInputStream PipedOutputStream PipedReader PipedWriter PixelGrabber PixelInterleavedSampleModel
|
||||||
|
PKCS8EncodedKeySpec PKIXBuilderParameters PKIXCertPathBuilderResult PKIXCertPathChecker
|
||||||
|
PKIXCertPathValidatorResult PKIXParameters PlainDocument PlainView Point Point2D PointerInfo Policy
|
||||||
|
PolicyNode PolicyQualifierInfo Polygon PooledConnection Popup PopupFactory PopupMenu PopupMenuEvent
|
||||||
|
PopupMenuListener PopupMenuUI Port PortableRemoteObject PortableRemoteObjectDelegate
|
||||||
|
PortUnreachableException Position Predicate PreferenceChangeEvent PreferenceChangeListener Preferences
|
||||||
|
PreferencesFactory PreparedStatement PresentationDirection Principal Printable PrinterAbortException
|
||||||
|
PrinterException PrinterGraphics PrinterInfo PrinterIOException PrinterIsAcceptingJobs PrinterJob
|
||||||
|
PrinterLocation PrinterMakeAndModel PrinterMessageFromOperator PrinterMoreInfo
|
||||||
|
PrinterMoreInfoManufacturer PrinterName PrinterResolution PrinterState PrinterStateReason
|
||||||
|
PrinterStateReasons PrinterURI PrintEvent PrintException PrintGraphics PrintJob PrintJobAdapter
|
||||||
|
PrintJobAttribute PrintJobAttributeEvent PrintJobAttributeListener PrintJobAttributeSet PrintJobEvent
|
||||||
|
PrintJobListener PrintQuality PrintRequestAttribute PrintRequestAttributeSet PrintService
|
||||||
|
PrintServiceAttribute PrintServiceAttributeEvent PrintServiceAttributeListener PrintServiceAttributeSet
|
||||||
|
PrintServiceLookup PrintStream PrintWriter PriorityBlockingQueue PriorityQueue PrivateClassLoader
|
||||||
|
PrivateCredentialPermission PrivateKey PrivateMLet PrivilegedAction PrivilegedActionException
|
||||||
|
PrivilegedExceptionAction Process ProcessBuilder ProfileDataException ProgressBarUI ProgressMonitor
|
||||||
|
ProgressMonitorInputStream Properties PropertyChangeEvent PropertyChangeListener
|
||||||
|
PropertyChangeListenerProxy PropertyChangeSupport PropertyDescriptor PropertyEditor
|
||||||
|
PropertyEditorManager PropertyEditorSupport PropertyPermission PropertyResourceBundle
|
||||||
|
PropertyVetoException ProtectionDomain ProtocolException Provider ProviderException Proxy ProxySelector
|
||||||
|
PSource PSSParameterSpec PublicKey PushbackInputStream PushbackReader QName QuadCurve2D Query QueryEval
|
||||||
|
QueryExp Queue QueuedJobCount Random RandomAccess RandomAccessFile Raster RasterFormatException RasterOp
|
||||||
|
RC2ParameterSpec RC5ParameterSpec Rdn Readable ReadableByteChannel Reader ReadOnlyBufferException
|
||||||
|
ReadWriteLock RealmCallback RealmChoiceCallback Receiver Rectangle Rectangle2D RectangularShape
|
||||||
|
ReentrantLock ReentrantReadWriteLock Ref RefAddr Reference Referenceable ReferenceQueue
|
||||||
|
ReferenceUriSchemesSupported ReferralException ReflectionException ReflectPermission Refreshable
|
||||||
|
RefreshFailedException Region RegisterableService Registry RegistryHandler RejectedExecutionException
|
||||||
|
RejectedExecutionHandler Relation RelationException RelationNotFoundException RelationNotification
|
||||||
|
RelationService RelationServiceMBean RelationServiceNotRegisteredException RelationSupport
|
||||||
|
RelationSupportMBean RelationType RelationTypeNotFoundException RelationTypeSupport Remote RemoteCall
|
||||||
|
RemoteException RemoteObject RemoteObjectInvocationHandler RemoteRef RemoteServer RemoteStub
|
||||||
|
RenderableImage RenderableImageOp RenderableImageProducer RenderContext RenderedImage
|
||||||
|
RenderedImageFactory Renderer RenderingHints RepaintManager ReplicateScaleFilter RequestingUserName
|
||||||
|
RequiredModelMBean RescaleOp ResolutionSyntax Resolver ResolveResult ResourceBundle ResponseCache Result
|
||||||
|
ResultSet ResultSetMetaData Retention RetentionPolicy ReverbType RGBImageFilter RMIClassLoader
|
||||||
|
RMIClassLoaderSpi RMIClientSocketFactory RMIConnection RMIConnectionImpl RMIConnectionImpl_Stub
|
||||||
|
RMIConnector RMIConnectorServer RMIFailureHandler RMIIIOPServerImpl RMIJRMPServerImpl
|
||||||
|
RMISecurityException RMISecurityManager RMIServer RMIServerImpl RMIServerImpl_Stub
|
||||||
|
RMIServerSocketFactory RMISocketFactory Robot Role RoleInfo RoleInfoNotFoundException RoleList
|
||||||
|
RoleNotFoundException RoleResult RoleStatus RoleUnresolved RoleUnresolvedList RootPaneContainer
|
||||||
|
RootPaneUI RoundingMode RoundRectangle2D RowMapper RowSet RowSetEvent RowSetInternal RowSetListener
|
||||||
|
RowSetMetaData RowSetMetaDataImpl RowSetReader RowSetWarning RowSetWriter RSAKey RSAKeyGenParameterSpec
|
||||||
|
RSAMultiPrimePrivateCrtKey RSAMultiPrimePrivateCrtKeySpec RSAOtherPrimeInfo RSAPrivateCrtKey
|
||||||
|
RSAPrivateCrtKeySpec RSAPrivateKey RSAPrivateKeySpec RSAPublicKey RSAPublicKeySpec RTFEditorKit
|
||||||
|
RuleBasedCollator Runnable Runtime RuntimeErrorException RuntimeException RuntimeMBeanException
|
||||||
|
RuntimeMXBean RuntimeOperationsException RuntimePermission SampleModel Sasl SaslClient SaslClientFactory
|
||||||
|
SaslException SaslServer SaslServerFactory Savepoint SAXParser SAXParserFactory SAXResult SAXSource
|
||||||
|
SAXTransformerFactory Scanner ScatteringByteChannel ScheduledExecutorService ScheduledFuture
|
||||||
|
ScheduledThreadPoolExecutor Schema SchemaFactory SchemaFactoryLoader SchemaViolationException Scrollable
|
||||||
|
Scrollbar ScrollBarUI ScrollPane ScrollPaneAdjustable ScrollPaneConstants ScrollPaneLayout ScrollPaneUI
|
||||||
|
SealedObject SearchControls SearchResult SecretKey SecretKeyFactory SecretKeyFactorySpi SecretKeySpec
|
||||||
|
SecureCacheResponse SecureClassLoader SecureRandom SecureRandomSpi Security SecurityException
|
||||||
|
SecurityManager SecurityPermission Segment SelectableChannel SelectionKey Selector SelectorProvider
|
||||||
|
Semaphore SeparatorUI Sequence SequenceInputStream Sequencer SerialArray SerialBlob SerialClob
|
||||||
|
SerialDatalink SerialException Serializable SerializablePermission SerialJavaObject SerialRef
|
||||||
|
SerialStruct ServerCloneException ServerError ServerException ServerNotActiveException ServerRef
|
||||||
|
ServerRuntimeException ServerSocket ServerSocketChannel ServerSocketFactory ServiceNotFoundException
|
||||||
|
ServicePermission ServiceRegistry ServiceUI ServiceUIFactory ServiceUnavailableException Set
|
||||||
|
SetOfIntegerSyntax Severity Shape ShapeGraphicAttribute SheetCollate Short ShortBuffer
|
||||||
|
ShortBufferException ShortLookupTable ShortMessage Sides Signature SignatureException SignatureSpi
|
||||||
|
SignedObject Signer SimpleAttributeSet SimpleBeanInfo SimpleDateFormat SimpleDoc SimpleFormatter
|
||||||
|
SimpleTimeZone SimpleType SinglePixelPackedSampleModel SingleSelectionModel Size2DSyntax
|
||||||
|
SizeLimitExceededException SizeRequirements SizeSequence Skeleton SkeletonMismatchException
|
||||||
|
SkeletonNotFoundException SliderUI Socket SocketAddress SocketChannel SocketException SocketFactory
|
||||||
|
SocketHandler SocketImpl SocketImplFactory SocketOptions SocketPermission SocketSecurityException
|
||||||
|
SocketTimeoutException SoftBevelBorder SoftReference SortControl SortedMap SortedSet
|
||||||
|
SortingFocusTraversalPolicy SortKey SortResponseControl Soundbank SoundbankReader SoundbankResource
|
||||||
|
Source SourceDataLine SourceLocator SpinnerDateModel SpinnerListModel SpinnerModel SpinnerNumberModel
|
||||||
|
SpinnerUI SplitPaneUI Spring SpringLayout SQLData SQLException SQLInput SQLInputImpl SQLOutput
|
||||||
|
SQLOutputImpl SQLPermission SQLWarning SSLContext SSLContextSpi SSLEngine SSLEngineResult SSLException
|
||||||
|
SSLHandshakeException SSLKeyException SSLPeerUnverifiedException SSLPermission SSLProtocolException
|
||||||
|
SslRMIClientSocketFactory SslRMIServerSocketFactory SSLServerSocket SSLServerSocketFactory SSLSession
|
||||||
|
SSLSessionBindingEvent SSLSessionBindingListener SSLSessionContext SSLSocket SSLSocketFactory Stack
|
||||||
|
StackOverflowError StackTraceElement StandardMBean StartTlsRequest StartTlsResponse StateEdit
|
||||||
|
StateEditable StateFactory Statement StreamCorruptedException StreamHandler StreamPrintService
|
||||||
|
StreamPrintServiceFactory StreamResult StreamSource StreamTokenizer StrictMath String StringBuffer
|
||||||
|
StringBufferInputStream StringBuilder StringCharacterIterator StringContent
|
||||||
|
StringIndexOutOfBoundsException StringMonitor StringMonitorMBean StringReader StringRefAddr
|
||||||
|
StringSelection StringTokenizer StringValueExp StringWriter Stroke Struct Stub StubDelegate
|
||||||
|
StubNotFoundException Style StyleConstants StyleContext StyledDocument StyledEditorKit StyleSheet
|
||||||
|
Subject SubjectDelegationPermission SubjectDomainCombiner SupportedValuesAttribute SuppressWarnings
|
||||||
|
SwingConstants SwingPropertyChangeSupport SwingUtilities SyncFactory SyncFactoryException
|
||||||
|
SyncFailedException SynchronousQueue SyncProvider SyncProviderException SyncResolver SynthConstants
|
||||||
|
SynthContext Synthesizer SynthGraphicsUtils SynthLookAndFeel SynthPainter SynthStyle SynthStyleFactory
|
||||||
|
SysexMessage System SystemColor SystemFlavorMap TabableView TabbedPaneUI TabExpander TableCellEditor
|
||||||
|
TableCellRenderer TableColumn TableColumnModel TableColumnModelEvent TableColumnModelListener
|
||||||
|
TableHeaderUI TableModel TableModelEvent TableModelListener TableUI TableView TabSet TabStop TabularData
|
||||||
|
TabularDataSupport TabularType TagElement Target TargetDataLine TargetedNotification Templates
|
||||||
|
TemplatesHandler TextAction TextArea TextAttribute TextComponent TextEvent TextField TextHitInfo
|
||||||
|
TextInputCallback TextLayout TextListener TextMeasurer TextOutputCallback TextSyntax TextUI TexturePaint
|
||||||
|
Thread ThreadDeath ThreadFactory ThreadGroup ThreadInfo ThreadLocal ThreadMXBean ThreadPoolExecutor
|
||||||
|
Throwable Tie TileObserver Time TimeLimitExceededException TimeoutException Timer
|
||||||
|
TimerAlarmClockNotification TimerMBean TimerNotification TimerTask Timestamp TimeUnit TimeZone
|
||||||
|
TitledBorder ToolBarUI Toolkit ToolTipManager ToolTipUI TooManyListenersException Track
|
||||||
|
TransactionalWriter TransactionRequiredException TransactionRolledbackException Transferable
|
||||||
|
TransferHandler TransformAttribute Transformer TransformerConfigurationException TransformerException
|
||||||
|
TransformerFactory TransformerFactoryConfigurationError TransformerHandler Transmitter Transparency
|
||||||
|
TreeCellEditor TreeCellRenderer TreeExpansionEvent TreeExpansionListener TreeMap TreeModel
|
||||||
|
TreeModelEvent TreeModelListener TreeNode TreePath TreeSelectionEvent TreeSelectionListener
|
||||||
|
TreeSelectionModel TreeSet TreeUI TreeWillExpandListener TrustAnchor TrustManager TrustManagerFactory
|
||||||
|
TrustManagerFactorySpi Type TypeInfoProvider TypeNotPresentException Types TypeVariable UID UIDefaults
|
||||||
|
UIManager UIResource UndeclaredThrowableException UndoableEdit UndoableEditEvent UndoableEditListener
|
||||||
|
UndoableEditSupport UndoManager UnexpectedException UnicastRemoteObject UnknownError
|
||||||
|
UnknownFormatConversionException UnknownFormatFlagsException UnknownGroupException UnknownHostException
|
||||||
|
UnknownObjectException UnknownServiceException UnmappableCharacterException UnmarshalException
|
||||||
|
UnmodifiableClassException UnmodifiableSetException UnrecoverableEntryException
|
||||||
|
UnrecoverableKeyException Unreferenced UnresolvedAddressException UnresolvedPermission
|
||||||
|
UnsatisfiedLinkError UnsolicitedNotification UnsolicitedNotificationEvent
|
||||||
|
UnsolicitedNotificationListener UnsupportedAddressTypeException UnsupportedAudioFileException
|
||||||
|
UnsupportedCallbackException UnsupportedCharsetException UnsupportedClassVersionError
|
||||||
|
UnsupportedEncodingException UnsupportedFlavorException UnsupportedLookAndFeelException
|
||||||
|
UnsupportedOperationException URI URIException URIResolver URISyntax URISyntaxException URL
|
||||||
|
URLClassLoader URLConnection URLDecoder URLEncoder URLStreamHandler URLStreamHandlerFactory
|
||||||
|
UTFDataFormatException Util UtilDelegate Utilities UUID Validator ValidatorHandler ValueExp ValueHandler
|
||||||
|
ValueHandlerMultiFormat VariableHeightLayoutCache Vector VerifyError VetoableChangeListener
|
||||||
|
VetoableChangeListenerProxy VetoableChangeSupport View ViewFactory ViewportLayout ViewportUI
|
||||||
|
VirtualMachineError Visibility VMID VoiceStatus Void VolatileImage WeakHashMap WeakReference WebRowSet
|
||||||
|
WildcardType Window WindowAdapter WindowConstants WindowEvent WindowFocusListener WindowListener
|
||||||
|
WindowStateListener WrappedPlainView WritableByteChannel WritableRaster WritableRenderedImage
|
||||||
|
WriteAbortedException Writer X500Principal X500PrivateCredential X509Certificate X509CertSelector
|
||||||
|
X509CRL X509CRLEntry X509CRLSelector X509EncodedKeySpec X509ExtendedKeyManager X509Extension
|
||||||
|
X509KeyManager X509TrustManager XAConnection XADataSource XAException XAResource Xid XMLConstants
|
||||||
|
XMLDecoder XMLEncoder XMLFormatter XMLGregorianCalendar XMLParseException XmlReader XmlWriter XPath
|
||||||
|
XPathConstants XPathException XPathExpression XPathExpressionException XPathFactory
|
||||||
|
XPathFactoryConfigurationException XPathFunction XPathFunctionException XPathFunctionResolver
|
||||||
|
XPathVariableResolver ZipEntry ZipException ZipFile ZipInputStream ZipOutputStream ZoneView
|
||||||
|
]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,224 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
class JavaScript < Scanner
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
|
||||||
|
register_for :java_script
|
||||||
|
file_extension 'js'
|
||||||
|
|
||||||
|
# The actual JavaScript keywords.
|
||||||
|
KEYWORDS = %w[
|
||||||
|
break case catch continue default delete do else
|
||||||
|
finally for function if in instanceof new
|
||||||
|
return switch throw try typeof var void while with
|
||||||
|
]
|
||||||
|
PREDEFINED_CONSTANTS = %w[
|
||||||
|
false null true undefined
|
||||||
|
]
|
||||||
|
|
||||||
|
MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
|
||||||
|
|
||||||
|
KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
|
||||||
|
case delete in instanceof new return throw typeof with
|
||||||
|
]
|
||||||
|
|
||||||
|
# Reserved for future use.
|
||||||
|
RESERVED_WORDS = %w[
|
||||||
|
abstract boolean byte char class debugger double enum export extends
|
||||||
|
final float goto implements import int interface long native package
|
||||||
|
private protected public short static super synchronized throws transient
|
||||||
|
volatile
|
||||||
|
]
|
||||||
|
|
||||||
|
IDENT_KIND = WordList.new(:ident).
|
||||||
|
add(RESERVED_WORDS, :reserved).
|
||||||
|
add(PREDEFINED_CONSTANTS, :pre_constant).
|
||||||
|
add(MAGIC_VARIABLES, :local_variable).
|
||||||
|
add(KEYWORDS, :keyword)
|
||||||
|
|
||||||
|
ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||||
|
REGEXP_ESCAPE = / [bBdDsSwW] /x
|
||||||
|
STRING_CONTENT_PATTERN = {
|
||||||
|
"'" => /[^\\']+/,
|
||||||
|
'"' => /[^\\"]+/,
|
||||||
|
'/' => /[^\\\/]+/,
|
||||||
|
}
|
||||||
|
KEY_CHECK_PATTERN = {
|
||||||
|
"'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
|
||||||
|
'"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
|
||||||
|
}
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
string_delimiter = nil
|
||||||
|
value_expected = true
|
||||||
|
key_expected = false
|
||||||
|
function_expected = false
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
case state
|
||||||
|
|
||||||
|
when :initial
|
||||||
|
|
||||||
|
if match = scan(/ \s+ | \\\n /x)
|
||||||
|
value_expected = true if !value_expected && match.index(?\n)
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
||||||
|
value_expected = true
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif check(/\.?\d/)
|
||||||
|
key_expected = value_expected = false
|
||||||
|
if scan(/0[xX][0-9A-Fa-f]+/)
|
||||||
|
kind = :hex
|
||||||
|
elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
|
||||||
|
kind = :oct
|
||||||
|
elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
||||||
|
kind = :float
|
||||||
|
elsif scan(/\d+/)
|
||||||
|
kind = :integer
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
|
||||||
|
# FIXME: scan over nested tags
|
||||||
|
xml_scanner.tokenize match
|
||||||
|
value_expected = false
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
|
||||||
|
value_expected = true
|
||||||
|
last_operator = match[-1]
|
||||||
|
key_expected = (last_operator == ?{) || (last_operator == ?,)
|
||||||
|
function_expected = false
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif scan(/ [)\]}]+ /x)
|
||||||
|
function_expected = key_expected = value_expected = false
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
|
||||||
|
kind = IDENT_KIND[match]
|
||||||
|
value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
|
||||||
|
# TODO: labels
|
||||||
|
if kind == :ident
|
||||||
|
if match.index(?$) # $ allowed inside an identifier
|
||||||
|
kind = :predefined
|
||||||
|
elsif function_expected
|
||||||
|
kind = :function
|
||||||
|
elsif check(/\s*[=:]\s*function\b/)
|
||||||
|
kind = :function
|
||||||
|
elsif key_expected && check(/\s*:/)
|
||||||
|
kind = :key
|
||||||
|
end
|
||||||
|
end
|
||||||
|
function_expected = (kind == :keyword) && (match == 'function')
|
||||||
|
key_expected = false
|
||||||
|
|
||||||
|
elsif match = scan(/["']/)
|
||||||
|
if key_expected && check(KEY_CHECK_PATTERN[match])
|
||||||
|
state = :key
|
||||||
|
else
|
||||||
|
state = :string
|
||||||
|
end
|
||||||
|
tokens << [:open, state]
|
||||||
|
string_delimiter = match
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif value_expected && (match = scan(/\/(?=\S)/))
|
||||||
|
tokens << [:open, :regexp]
|
||||||
|
state = :regexp
|
||||||
|
string_delimiter = '/'
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif scan(/ \/ /x)
|
||||||
|
value_expected = true
|
||||||
|
key_expected = false
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
when :string, :regexp, :key
|
||||||
|
if scan(STRING_CONTENT_PATTERN[string_delimiter])
|
||||||
|
kind = :content
|
||||||
|
elsif match = scan(/["'\/]/)
|
||||||
|
tokens << [match, :delimiter]
|
||||||
|
if state == :regexp
|
||||||
|
modifiers = scan(/[gim]+/)
|
||||||
|
tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
|
||||||
|
end
|
||||||
|
tokens << [:close, state]
|
||||||
|
string_delimiter = nil
|
||||||
|
key_expected = value_expected = false
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
|
||||||
|
if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
|
||||||
|
kind = :content
|
||||||
|
else
|
||||||
|
kind = :char
|
||||||
|
end
|
||||||
|
elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/\\./m)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/ \\ | $ /x)
|
||||||
|
tokens << [:close, state]
|
||||||
|
kind = :error
|
||||||
|
key_expected = value_expected = false
|
||||||
|
state = :initial
|
||||||
|
else
|
||||||
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
if [:string, :regexp].include? state
|
||||||
|
tokens << [:close, state]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
protected
|
||||||
|
|
||||||
|
def reset_instance
|
||||||
|
super
|
||||||
|
@xml_scanner.reset if defined? @xml_scanner
|
||||||
|
end
|
||||||
|
|
||||||
|
def xml_scanner
|
||||||
|
@xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,112 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
class JSON < Scanner
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
|
||||||
|
register_for :json
|
||||||
|
file_extension 'json'
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = [
|
||||||
|
:float, :char, :content, :delimiter,
|
||||||
|
:error, :integer, :operator, :value,
|
||||||
|
]
|
||||||
|
|
||||||
|
CONSTANTS = %w( true false null )
|
||||||
|
IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
|
||||||
|
|
||||||
|
ESCAPE = / [bfnrt\\"\/] /x
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
stack = []
|
||||||
|
string_delimiter = nil
|
||||||
|
key_expected = false
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
case state
|
||||||
|
|
||||||
|
when :initial
|
||||||
|
if match = scan(/ \s+ | \\\n /x)
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
elsif match = scan(/ [:,\[{\]}] /x)
|
||||||
|
kind = :operator
|
||||||
|
case match
|
||||||
|
when '{' then stack << :object; key_expected = true
|
||||||
|
when '[' then stack << :array
|
||||||
|
when ':' then key_expected = false
|
||||||
|
when ',' then key_expected = true if stack.last == :object
|
||||||
|
when '}', ']' then stack.pop # no error recovery, but works for valid JSON
|
||||||
|
end
|
||||||
|
elsif match = scan(/ true | false | null /x)
|
||||||
|
kind = IDENT_KIND[match]
|
||||||
|
elsif match = scan(/-?(?:0|[1-9]\d*)/)
|
||||||
|
kind = :integer
|
||||||
|
if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
|
||||||
|
match << matched
|
||||||
|
kind = :float
|
||||||
|
end
|
||||||
|
elsif match = scan(/"/)
|
||||||
|
state = key_expected ? :key : :string
|
||||||
|
tokens << [:open, state]
|
||||||
|
kind = :delimiter
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
end
|
||||||
|
|
||||||
|
when :string, :key
|
||||||
|
if scan(/[^\\"]+/)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/"/)
|
||||||
|
tokens << ['"', :delimiter]
|
||||||
|
tokens << [:close, state]
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/\\./m)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/ \\ | $ /x)
|
||||||
|
tokens << [:close, :delimiter]
|
||||||
|
kind = :error
|
||||||
|
state = :initial
|
||||||
|
else
|
||||||
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
if [:string, :key].include? state
|
||||||
|
tokens << [:close, state]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -5,13 +5,15 @@ module Scanners
|
||||||
load :ruby
|
load :ruby
|
||||||
|
|
||||||
# Nitro XHTML Scanner
|
# Nitro XHTML Scanner
|
||||||
#
|
|
||||||
# $Id$
|
|
||||||
class NitroXHTML < Scanner
|
class NitroXHTML < Scanner
|
||||||
|
|
||||||
include Streamable
|
include Streamable
|
||||||
register_for :nitro_xhtml
|
register_for :nitro_xhtml
|
||||||
|
file_extension :xhtml
|
||||||
|
title 'Nitro XHTML'
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
|
||||||
|
|
||||||
NITRO_RUBY_BLOCK = /
|
NITRO_RUBY_BLOCK = /
|
||||||
<\?r
|
<\?r
|
||||||
(?>
|
(?>
|
||||||
|
@ -119,6 +121,7 @@ module Scanners
|
||||||
|
|
||||||
else
|
else
|
||||||
raise_inspect 'else-case reached!', tokens
|
raise_inspect 'else-case reached!', tokens
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
|
@ -0,0 +1,527 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
load :html
|
||||||
|
|
||||||
|
# Original by Stefan Walk.
|
||||||
|
class PHP < Scanner
|
||||||
|
|
||||||
|
register_for :php
|
||||||
|
file_extension 'php'
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
|
||||||
|
|
||||||
|
def setup
|
||||||
|
@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
||||||
|
end
|
||||||
|
|
||||||
|
def reset_instance
|
||||||
|
super
|
||||||
|
@html_scanner.reset
|
||||||
|
end
|
||||||
|
|
||||||
|
module Words
|
||||||
|
|
||||||
|
# according to http://www.php.net/manual/en/reserved.keywords.php
|
||||||
|
KEYWORDS = %w[
|
||||||
|
abstract and array as break case catch class clone const continue declare default do else elseif
|
||||||
|
enddeclare endfor endforeach endif endswitch endwhile extends final for foreach function global
|
||||||
|
goto if implements interface instanceof namespace new or private protected public static switch
|
||||||
|
throw try use var while xor
|
||||||
|
cfunction old_function
|
||||||
|
]
|
||||||
|
|
||||||
|
TYPES = %w[ int integer float double bool boolean string array object resource ]
|
||||||
|
|
||||||
|
LANGUAGE_CONSTRUCTS = %w[
|
||||||
|
die echo empty exit eval include include_once isset list
|
||||||
|
require require_once return print unset
|
||||||
|
]
|
||||||
|
|
||||||
|
CLASSES = %w[ Directory stdClass __PHP_Incomplete_Class exception php_user_filter Closure ]
|
||||||
|
|
||||||
|
# according to http://php.net/quickref.php on 2009-04-21;
|
||||||
|
# all functions with _ excluded (module functions) and selected additional functions
|
||||||
|
BUILTIN_FUNCTIONS = %w[
|
||||||
|
abs acos acosh addcslashes addslashes aggregate array arsort ascii2ebcdic asin asinh asort assert atan atan2
|
||||||
|
atanh basename bcadd bccomp bcdiv bcmod bcmul bcpow bcpowmod bcscale bcsqrt bcsub bin2hex bindec
|
||||||
|
bindtextdomain bzclose bzcompress bzdecompress bzerrno bzerror bzerrstr bzflush bzopen bzread bzwrite
|
||||||
|
calculhmac ceil chdir checkdate checkdnsrr chgrp chmod chop chown chr chroot clearstatcache closedir closelog
|
||||||
|
compact constant copy cos cosh count crc32 crypt current date dcgettext dcngettext deaggregate decbin dechex
|
||||||
|
decoct define defined deg2rad delete dgettext die dirname diskfreespace dl dngettext doubleval each
|
||||||
|
ebcdic2ascii echo empty end ereg eregi escapeshellarg escapeshellcmd eval exec exit exp explode expm1 extract
|
||||||
|
fclose feof fflush fgetc fgetcsv fgets fgetss file fileatime filectime filegroup fileinode filemtime fileowner
|
||||||
|
fileperms filepro filesize filetype floatval flock floor flush fmod fnmatch fopen fpassthru fprintf fputcsv
|
||||||
|
fputs fread frenchtojd fscanf fseek fsockopen fstat ftell ftok ftruncate fwrite getallheaders getcwd getdate
|
||||||
|
getenv gethostbyaddr gethostbyname gethostbynamel getimagesize getlastmod getmxrr getmygid getmyinode getmypid
|
||||||
|
getmyuid getopt getprotobyname getprotobynumber getrandmax getrusage getservbyname getservbyport gettext
|
||||||
|
gettimeofday gettype glob gmdate gmmktime gmstrftime gregoriantojd gzclose gzcompress gzdecode gzdeflate
|
||||||
|
gzencode gzeof gzfile gzgetc gzgets gzgetss gzinflate gzopen gzpassthru gzputs gzread gzrewind gzseek gztell
|
||||||
|
gzuncompress gzwrite hash header hebrev hebrevc hexdec htmlentities htmlspecialchars hypot iconv idate
|
||||||
|
implode include intval ip2long iptcembed iptcparse isset
|
||||||
|
jddayofweek jdmonthname jdtofrench jdtogregorian jdtojewish jdtojulian jdtounix jewishtojd join jpeg2wbmp
|
||||||
|
juliantojd key krsort ksort lcfirst lchgrp lchown levenshtein link linkinfo list localeconv localtime log
|
||||||
|
log10 log1p long2ip lstat ltrim mail main max md5 metaphone mhash microtime min mkdir mktime msql natcasesort
|
||||||
|
natsort next ngettext nl2br nthmac octdec opendir openlog
|
||||||
|
ord overload pack passthru pathinfo pclose pfsockopen phpcredits phpinfo phpversion pi png2wbmp popen pos pow
|
||||||
|
prev print printf putenv quotemeta rad2deg rand range rawurldecode rawurlencode readdir readfile readgzfile
|
||||||
|
readline readlink realpath recode rename require reset rewind rewinddir rmdir round rsort rtrim scandir
|
||||||
|
serialize setcookie setlocale setrawcookie settype sha1 shuffle signeurlpaiement sin sinh sizeof sleep snmpget
|
||||||
|
snmpgetnext snmprealwalk snmpset snmpwalk snmpwalkoid sort soundex split spliti sprintf sqrt srand sscanf stat
|
||||||
|
strcasecmp strchr strcmp strcoll strcspn strftime stripcslashes stripos stripslashes stristr strlen
|
||||||
|
strnatcasecmp strnatcmp strncasecmp strncmp strpbrk strpos strptime strrchr strrev strripos strrpos strspn
|
||||||
|
strstr strtok strtolower strtotime strtoupper strtr strval substr symlink syslog system tan tanh tempnam
|
||||||
|
textdomain time tmpfile touch trim uasort ucfirst ucwords uksort umask uniqid unixtojd unlink unpack
|
||||||
|
unserialize unset urldecode urlencode usleep usort vfprintf virtual vprintf vsprintf wordwrap
|
||||||
|
array_change_key_case array_chunk array_combine array_count_values array_diff array_diff_assoc
|
||||||
|
array_diff_key array_diff_uassoc array_diff_ukey array_fill array_fill_keys array_filter array_flip
|
||||||
|
array_intersect array_intersect_assoc array_intersect_key array_intersect_uassoc array_intersect_ukey
|
||||||
|
array_key_exists array_keys array_map array_merge array_merge_recursive array_multisort array_pad
|
||||||
|
array_pop array_product array_push array_rand array_reduce array_reverse array_search array_shift
|
||||||
|
array_slice array_splice array_sum array_udiff array_udiff_assoc array_udiff_uassoc array_uintersect
|
||||||
|
array_uintersect_assoc array_uintersect_uassoc array_unique array_unshift array_values array_walk
|
||||||
|
array_walk_recursive
|
||||||
|
assert_options base_convert base64_decode base64_encode
|
||||||
|
chunk_split class_exists class_implements class_parents
|
||||||
|
count_chars debug_backtrace debug_print_backtrace debug_zval_dump
|
||||||
|
error_get_last error_log error_reporting extension_loaded
|
||||||
|
file_exists file_get_contents file_put_contents load_file
|
||||||
|
func_get_arg func_get_args func_num_args function_exists
|
||||||
|
get_browser get_called_class get_cfg_var get_class get_class_methods get_class_vars
|
||||||
|
get_current_user get_declared_classes get_declared_interfaces get_defined_constants
|
||||||
|
get_defined_functions get_defined_vars get_extension_funcs get_headers get_html_translation_table
|
||||||
|
get_include_path get_included_files get_loaded_extensions get_magic_quotes_gpc get_magic_quotes_runtime
|
||||||
|
get_meta_tags get_object_vars get_parent_class get_required_filesget_resource_type
|
||||||
|
gc_collect_cycles gc_disable gc_enable gc_enabled
|
||||||
|
halt_compiler headers_list headers_sent highlight_file highlight_string
|
||||||
|
html_entity_decode htmlspecialchars_decode
|
||||||
|
in_array include_once inclued_get_data
|
||||||
|
is_a is_array is_binary is_bool is_buffer is_callable is_dir is_double is_executable is_file is_finite
|
||||||
|
is_float is_infinite is_int is_integer is_link is_long is_nan is_null is_numeric is_object is_readable
|
||||||
|
is_real is_resource is_scalar is_soap_fault is_string is_subclass_of is_unicode is_uploaded_file
|
||||||
|
is_writable is_writeable
|
||||||
|
locale_get_default locale_set_default
|
||||||
|
number_format override_function parse_str parse_url
|
||||||
|
php_check_syntax php_ini_loaded_file php_ini_scanned_files php_logo_guid php_sapi_name
|
||||||
|
php_strip_whitespace php_uname
|
||||||
|
preg_filter preg_grep preg_last_error preg_match preg_match_all preg_quote preg_replace
|
||||||
|
preg_replace_callback preg_split print_r
|
||||||
|
require_once register_shutdown_function register_tick_function
|
||||||
|
set_error_handler set_exception_handler set_file_buffer set_include_path
|
||||||
|
set_magic_quotes_runtime set_time_limit shell_exec
|
||||||
|
str_getcsv str_ireplace str_pad str_repeat str_replace str_rot13 str_shuffle str_split str_word_count
|
||||||
|
strip_tags substr_compare substr_count substr_replace
|
||||||
|
time_nanosleep time_sleep_until
|
||||||
|
token_get_all token_name trigger_error
|
||||||
|
unregister_tick_function use_soap_error_handler user_error
|
||||||
|
utf8_decode utf8_encode var_dump var_export
|
||||||
|
version_compare
|
||||||
|
zend_logo_guid zend_thread_id zend_version
|
||||||
|
create_function call_user_func_array
|
||||||
|
posix_access posix_ctermid posix_get_last_error posix_getcwd posix_getegid
|
||||||
|
posix_geteuid posix_getgid posix_getgrgid posix_getgrnam posix_getgroups
|
||||||
|
posix_getlogin posix_getpgid posix_getpgrp posix_getpid posix_getppid
|
||||||
|
posix_getpwnam posix_getpwuid posix_getrlimit posix_getsid posix_getuid
|
||||||
|
posix_initgroups posix_isatty posix_kill posix_mkfifo posix_mknod
|
||||||
|
posix_setegid posix_seteuid posix_setgid posix_setpgid posix_setsid
|
||||||
|
posix_setuid posix_strerror posix_times posix_ttyname posix_uname
|
||||||
|
pcntl_alarm pcntl_exec pcntl_fork pcntl_getpriority pcntl_setpriority
|
||||||
|
pcntl_signal pcntl_signal_dispatch pcntl_sigprocmask pcntl_sigtimedwait
|
||||||
|
pcntl_sigwaitinfo pcntl_wait pcntl_waitpid pcntl_wexitstatus pcntl_wifexited
|
||||||
|
pcntl_wifsignaled pcntl_wifstopped pcntl_wstopsig pcntl_wtermsig
|
||||||
|
]
|
||||||
|
# TODO: more built-in PHP functions?
|
||||||
|
|
||||||
|
EXCEPTIONS = %w[
|
||||||
|
E_ERROR E_WARNING E_PARSE E_NOTICE E_CORE_ERROR E_CORE_WARNING E_COMPILE_ERROR E_COMPILE_WARNING
|
||||||
|
E_USER_ERROR E_USER_WARNING E_USER_NOTICE E_DEPRECATED E_USER_DEPRECATED E_ALL E_STRICT
|
||||||
|
]
|
||||||
|
|
||||||
|
CONSTANTS = %w[
|
||||||
|
null true false self parent
|
||||||
|
__LINE__ __DIR__ __FILE__ __LINE__
|
||||||
|
__CLASS__ __NAMESPACE__ __METHOD__ __FUNCTION__
|
||||||
|
PHP_VERSION PHP_MAJOR_VERSION PHP_MINOR_VERSION PHP_RELEASE_VERSION PHP_VERSION_ID PHP_EXTRA_VERSION PHP_ZTS
|
||||||
|
PHP_DEBUG PHP_MAXPATHLEN PHP_OS PHP_SAPI PHP_EOL PHP_INT_MAX PHP_INT_SIZE DEFAULT_INCLUDE_PATH
|
||||||
|
PEAR_INSTALL_DIR PEAR_EXTENSION_DIR PHP_EXTENSION_DIR PHP_PREFIX PHP_BINDIR PHP_LIBDIR PHP_DATADIR
|
||||||
|
PHP_SYSCONFDIR PHP_LOCALSTATEDIR PHP_CONFIG_FILE_PATH PHP_CONFIG_FILE_SCAN_DIR PHP_SHLIB_SUFFIX
|
||||||
|
PHP_OUTPUT_HANDLER_START PHP_OUTPUT_HANDLER_CONT PHP_OUTPUT_HANDLER_END
|
||||||
|
__COMPILER_HALT_OFFSET__
|
||||||
|
EXTR_OVERWRITE EXTR_SKIP EXTR_PREFIX_SAME EXTR_PREFIX_ALL EXTR_PREFIX_INVALID EXTR_PREFIX_IF_EXISTS
|
||||||
|
EXTR_IF_EXISTS SORT_ASC SORT_DESC SORT_REGULAR SORT_NUMERIC SORT_STRING CASE_LOWER CASE_UPPER COUNT_NORMAL
|
||||||
|
COUNT_RECURSIVE ASSERT_ACTIVE ASSERT_CALLBACK ASSERT_BAIL ASSERT_WARNING ASSERT_QUIET_EVAL CONNECTION_ABORTED
|
||||||
|
CONNECTION_NORMAL CONNECTION_TIMEOUT INI_USER INI_PERDIR INI_SYSTEM INI_ALL M_E M_LOG2E M_LOG10E M_LN2 M_LN10
|
||||||
|
M_PI M_PI_2 M_PI_4 M_1_PI M_2_PI M_2_SQRTPI M_SQRT2 M_SQRT1_2 CRYPT_SALT_LENGTH CRYPT_STD_DES CRYPT_EXT_DES
|
||||||
|
CRYPT_MD5 CRYPT_BLOWFISH DIRECTORY_SEPARATOR SEEK_SET SEEK_CUR SEEK_END LOCK_SH LOCK_EX LOCK_UN LOCK_NB
|
||||||
|
HTML_SPECIALCHARS HTML_ENTITIES ENT_COMPAT ENT_QUOTES ENT_NOQUOTES INFO_GENERAL INFO_CREDITS
|
||||||
|
INFO_CONFIGURATION INFO_MODULES INFO_ENVIRONMENT INFO_VARIABLES INFO_LICENSE INFO_ALL CREDITS_GROUP
|
||||||
|
CREDITS_GENERAL CREDITS_SAPI CREDITS_MODULES CREDITS_DOCS CREDITS_FULLPAGE CREDITS_QA CREDITS_ALL STR_PAD_LEFT
|
||||||
|
STR_PAD_RIGHT STR_PAD_BOTH PATHINFO_DIRNAME PATHINFO_BASENAME PATHINFO_EXTENSION PATH_SEPARATOR CHAR_MAX
|
||||||
|
LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_ALL LC_MESSAGES ABDAY_1 ABDAY_2 ABDAY_3 ABDAY_4 ABDAY_5
|
||||||
|
ABDAY_6 ABDAY_7 DAY_1 DAY_2 DAY_3 DAY_4 DAY_5 DAY_6 DAY_7 ABMON_1 ABMON_2 ABMON_3 ABMON_4 ABMON_5 ABMON_6
|
||||||
|
ABMON_7 ABMON_8 ABMON_9 ABMON_10 ABMON_11 ABMON_12 MON_1 MON_2 MON_3 MON_4 MON_5 MON_6 MON_7 MON_8 MON_9
|
||||||
|
MON_10 MON_11 MON_12 AM_STR PM_STR D_T_FMT D_FMT T_FMT T_FMT_AMPM ERA ERA_YEAR ERA_D_T_FMT ERA_D_FMT ERA_T_FMT
|
||||||
|
ALT_DIGITS INT_CURR_SYMBOL CURRENCY_SYMBOL CRNCYSTR MON_DECIMAL_POINT MON_THOUSANDS_SEP MON_GROUPING
|
||||||
|
POSITIVE_SIGN NEGATIVE_SIGN INT_FRAC_DIGITS FRAC_DIGITS P_CS_PRECEDES P_SEP_BY_SPACE N_CS_PRECEDES
|
||||||
|
N_SEP_BY_SPACE P_SIGN_POSN N_SIGN_POSN DECIMAL_POINT RADIXCHAR THOUSANDS_SEP THOUSEP GROUPING YESEXPR NOEXPR
|
||||||
|
YESSTR NOSTR CODESET LOG_EMERG LOG_ALERT LOG_CRIT LOG_ERR LOG_WARNING LOG_NOTICE LOG_INFO LOG_DEBUG LOG_KERN
|
||||||
|
LOG_USER LOG_MAIL LOG_DAEMON LOG_AUTH LOG_SYSLOG LOG_LPR LOG_NEWS LOG_UUCP LOG_CRON LOG_AUTHPRIV LOG_LOCAL0
|
||||||
|
LOG_LOCAL1 LOG_LOCAL2 LOG_LOCAL3 LOG_LOCAL4 LOG_LOCAL5 LOG_LOCAL6 LOG_LOCAL7 LOG_PID LOG_CONS LOG_ODELAY
|
||||||
|
LOG_NDELAY LOG_NOWAIT LOG_PERROR
|
||||||
|
]
|
||||||
|
|
||||||
|
PREDEFINED = %w[
|
||||||
|
$GLOBALS $_SERVER $_GET $_POST $_FILES $_REQUEST $_SESSION $_ENV
|
||||||
|
$_COOKIE $php_errormsg $HTTP_RAW_POST_DATA $http_response_header
|
||||||
|
$argc $argv
|
||||||
|
]
|
||||||
|
|
||||||
|
IDENT_KIND = CaseIgnoringWordList.new(:ident).
|
||||||
|
add(KEYWORDS, :reserved).
|
||||||
|
add(TYPES, :pre_type).
|
||||||
|
add(LANGUAGE_CONSTRUCTS, :reserved).
|
||||||
|
add(BUILTIN_FUNCTIONS, :predefined).
|
||||||
|
add(CLASSES, :pre_constant).
|
||||||
|
add(EXCEPTIONS, :exception).
|
||||||
|
add(CONSTANTS, :pre_constant)
|
||||||
|
|
||||||
|
VARIABLE_KIND = WordList.new(:local_variable).
|
||||||
|
add(PREDEFINED, :predefined)
|
||||||
|
end
|
||||||
|
|
||||||
|
module RE
|
||||||
|
|
||||||
|
PHP_START = /
|
||||||
|
<script\s+[^>]*?language\s*=\s*"php"[^>]*?> |
|
||||||
|
<script\s+[^>]*?language\s*=\s*'php'[^>]*?> |
|
||||||
|
<\?php\d? |
|
||||||
|
<\?(?!xml)
|
||||||
|
/xi
|
||||||
|
|
||||||
|
PHP_END = %r!
|
||||||
|
</script> |
|
||||||
|
\?>
|
||||||
|
!xi
|
||||||
|
|
||||||
|
HTML_INDICATOR = /<!DOCTYPE html|<(?:html|body|div|p)[> ]/i
|
||||||
|
|
||||||
|
IDENTIFIER = /[a-z_\x7f-\xFF][a-z0-9_\x7f-\xFF]*/i
|
||||||
|
VARIABLE = /\$#{IDENTIFIER}/
|
||||||
|
|
||||||
|
OPERATOR = /
|
||||||
|
\.(?!\d)=? | # dot that is not decimal point, string concatenation
|
||||||
|
&& | \|\| | # logic
|
||||||
|
:: | -> | => | # scope, member, dictionary
|
||||||
|
\\(?!\n) | # namespace
|
||||||
|
\+\+ | -- | # increment, decrement
|
||||||
|
[,;?:()\[\]{}] | # simple delimiters
|
||||||
|
[-+*\/%&|^]=? | # ordinary math, binary logic, assignment shortcuts
|
||||||
|
[~$] | # whatever
|
||||||
|
=& | # reference assignment
|
||||||
|
[=!]=?=? | <> | # comparison and assignment
|
||||||
|
<<=? | >>=? | [<>]=? # comparison and shift
|
||||||
|
/x
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
if check(RE::PHP_START) || # starts with <?
|
||||||
|
(match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?
|
||||||
|
exist?(RE::HTML_INDICATOR) ||
|
||||||
|
check(/.{1,100}#{RE::PHP_START}/om) # PHP start after max 100 chars
|
||||||
|
# is HTML with embedded PHP, so start with HTML
|
||||||
|
states = [:initial]
|
||||||
|
else
|
||||||
|
# is just PHP, so start with PHP surrounded by HTML
|
||||||
|
states = [:initial, :php]
|
||||||
|
end
|
||||||
|
|
||||||
|
label_expected = true
|
||||||
|
case_expected = false
|
||||||
|
|
||||||
|
heredoc_delimiter = nil
|
||||||
|
delimiter = nil
|
||||||
|
modifier = nil
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
match = nil
|
||||||
|
kind = nil
|
||||||
|
|
||||||
|
case states.last
|
||||||
|
|
||||||
|
when :initial # HTML
|
||||||
|
if scan RE::PHP_START
|
||||||
|
kind = :inline_delimiter
|
||||||
|
label_expected = true
|
||||||
|
states << :php
|
||||||
|
else
|
||||||
|
match = scan_until(/(?=#{RE::PHP_START})/o) || scan_until(/\z/)
|
||||||
|
@html_scanner.tokenize match unless match.empty?
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
when :php
|
||||||
|
if match = scan(/\s+/)
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif scan(%r! (?m: \/\* (?: .*? \*\/ | .* ) ) | (?://|\#) .*? (?=#{RE::PHP_END}|$) !xo)
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif match = scan(RE::IDENTIFIER)
|
||||||
|
kind = Words::IDENT_KIND[match]
|
||||||
|
if kind == :ident && label_expected && check(/:(?!:)/)
|
||||||
|
kind = :label
|
||||||
|
label_expected = true
|
||||||
|
else
|
||||||
|
label_expected = false
|
||||||
|
if kind == :ident && match =~ /^[A-Z]/
|
||||||
|
kind = :constant
|
||||||
|
elsif kind == :reserved
|
||||||
|
case match
|
||||||
|
when 'class'
|
||||||
|
states << :class_expected
|
||||||
|
when 'function'
|
||||||
|
states << :function_expected
|
||||||
|
when 'case', 'default'
|
||||||
|
case_expected = true
|
||||||
|
end
|
||||||
|
elsif match == 'b' && check(/['"]/) # binary string literal
|
||||||
|
modifier = match
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/(?:\d+\.\d*|\d*\.\d+)(?:e[-+]?\d+)?|\d+e[-+]?\d+/i)
|
||||||
|
label_expected = false
|
||||||
|
kind = :float
|
||||||
|
|
||||||
|
elsif scan(/0x[0-9a-fA-F]+/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :hex
|
||||||
|
|
||||||
|
elsif scan(/\d+/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :integer
|
||||||
|
|
||||||
|
elsif scan(/'/)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
if modifier
|
||||||
|
tokens << [modifier, :modifier]
|
||||||
|
modifier = nil
|
||||||
|
end
|
||||||
|
kind = :delimiter
|
||||||
|
states.push :sqstring
|
||||||
|
|
||||||
|
elsif match = scan(/["`]/)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
if modifier
|
||||||
|
tokens << [modifier, :modifier]
|
||||||
|
modifier = nil
|
||||||
|
end
|
||||||
|
delimiter = match
|
||||||
|
kind = :delimiter
|
||||||
|
states.push :dqstring
|
||||||
|
|
||||||
|
elsif match = scan(RE::VARIABLE)
|
||||||
|
label_expected = false
|
||||||
|
kind = Words::VARIABLE_KIND[match]
|
||||||
|
|
||||||
|
elsif scan(/\{/)
|
||||||
|
kind = :operator
|
||||||
|
label_expected = true
|
||||||
|
states.push :php
|
||||||
|
|
||||||
|
elsif scan(/\}/)
|
||||||
|
if states.size == 1
|
||||||
|
kind = :error
|
||||||
|
else
|
||||||
|
states.pop
|
||||||
|
if states.last.is_a?(::Array)
|
||||||
|
delimiter = states.last[1]
|
||||||
|
states[-1] = states.last[0]
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :inline]
|
||||||
|
next
|
||||||
|
else
|
||||||
|
kind = :operator
|
||||||
|
label_expected = true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/@/)
|
||||||
|
label_expected = false
|
||||||
|
kind = :exception
|
||||||
|
|
||||||
|
elsif scan RE::PHP_END
|
||||||
|
kind = :inline_delimiter
|
||||||
|
states = [:initial]
|
||||||
|
|
||||||
|
elsif match = scan(/<<<(?:(#{RE::IDENTIFIER})|"(#{RE::IDENTIFIER})"|'(#{RE::IDENTIFIER})')/o)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
warn 'heredoc in heredoc?' if heredoc_delimiter
|
||||||
|
heredoc_delimiter = Regexp.escape(self[1] || self[2] || self[3])
|
||||||
|
kind = :delimiter
|
||||||
|
states.push self[3] ? :sqstring : :dqstring
|
||||||
|
heredoc_delimiter = /#{heredoc_delimiter}(?=;?$)/
|
||||||
|
|
||||||
|
elsif match = scan(/#{RE::OPERATOR}/o)
|
||||||
|
label_expected = match == ';'
|
||||||
|
if case_expected
|
||||||
|
label_expected = true if match == ':'
|
||||||
|
case_expected = false
|
||||||
|
end
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
when :sqstring
|
||||||
|
if scan(heredoc_delimiter ? /[^\\\n]+/ : /[^'\\]+/)
|
||||||
|
kind = :content
|
||||||
|
elsif !heredoc_delimiter && scan(/'/)
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
delimiter = nil
|
||||||
|
label_expected = false
|
||||||
|
states.pop
|
||||||
|
next
|
||||||
|
elsif heredoc_delimiter && match = scan(/\n/)
|
||||||
|
kind = :content
|
||||||
|
if scan heredoc_delimiter
|
||||||
|
tokens << ["\n", :content]
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
heredoc_delimiter = nil
|
||||||
|
label_expected = false
|
||||||
|
states.pop
|
||||||
|
next
|
||||||
|
end
|
||||||
|
elsif scan(heredoc_delimiter ? /\\\\/ : /\\[\\'\n]/)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/\\./m)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/\\/)
|
||||||
|
kind = :error
|
||||||
|
end
|
||||||
|
|
||||||
|
when :dqstring
|
||||||
|
if scan(heredoc_delimiter ? /[^${\\\n]+/ : (delimiter == '"' ? /[^"${\\]+/ : /[^`${\\]+/))
|
||||||
|
kind = :content
|
||||||
|
elsif !heredoc_delimiter && scan(delimiter == '"' ? /"/ : /`/)
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
delimiter = nil
|
||||||
|
label_expected = false
|
||||||
|
states.pop
|
||||||
|
next
|
||||||
|
elsif heredoc_delimiter && match = scan(/\n/)
|
||||||
|
kind = :content
|
||||||
|
if scan heredoc_delimiter
|
||||||
|
tokens << ["\n", :content]
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
heredoc_delimiter = nil
|
||||||
|
label_expected = false
|
||||||
|
states.pop
|
||||||
|
next
|
||||||
|
end
|
||||||
|
elsif scan(/\\(?:x[0-9A-Fa-f]{1,2}|[0-7]{1,3})/)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(heredoc_delimiter ? /\\[nrtvf\\$]/ : (delimiter == '"' ? /\\[nrtvf\\$"]/ : /\\[nrtvf\\$`]/))
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/\\./m)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/\\/)
|
||||||
|
kind = :error
|
||||||
|
elsif match = scan(/#{RE::VARIABLE}/o)
|
||||||
|
kind = :local_variable
|
||||||
|
if check(/\[#{RE::IDENTIFIER}\]/o)
|
||||||
|
tokens << [:open, :inline]
|
||||||
|
tokens << [match, :local_variable]
|
||||||
|
tokens << [scan(/\[/), :operator]
|
||||||
|
tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
|
||||||
|
tokens << [scan(/\]/), :operator]
|
||||||
|
tokens << [:close, :inline]
|
||||||
|
next
|
||||||
|
elsif check(/\[/)
|
||||||
|
match << scan(/\[['"]?#{RE::IDENTIFIER}?['"]?\]?/o)
|
||||||
|
kind = :error
|
||||||
|
elsif check(/->#{RE::IDENTIFIER}/o)
|
||||||
|
tokens << [:open, :inline]
|
||||||
|
tokens << [match, :local_variable]
|
||||||
|
tokens << [scan(/->/), :operator]
|
||||||
|
tokens << [scan(/#{RE::IDENTIFIER}/o), :ident]
|
||||||
|
tokens << [:close, :inline]
|
||||||
|
next
|
||||||
|
elsif check(/->/)
|
||||||
|
match << scan(/->/)
|
||||||
|
kind = :error
|
||||||
|
end
|
||||||
|
elsif match = scan(/\{/)
|
||||||
|
if check(/\$/)
|
||||||
|
kind = :delimiter
|
||||||
|
states[-1] = [states.last, delimiter]
|
||||||
|
delimiter = nil
|
||||||
|
states.push :php
|
||||||
|
tokens << [:open, :inline]
|
||||||
|
else
|
||||||
|
kind = :string
|
||||||
|
end
|
||||||
|
elsif scan(/\$\{#{RE::IDENTIFIER}\}/o)
|
||||||
|
kind = :local_variable
|
||||||
|
elsif scan(/\$/)
|
||||||
|
kind = :content
|
||||||
|
end
|
||||||
|
|
||||||
|
when :class_expected
|
||||||
|
if scan(/\s+/)
|
||||||
|
kind = :space
|
||||||
|
elsif match = scan(/#{RE::IDENTIFIER}/o)
|
||||||
|
kind = :class
|
||||||
|
states.pop
|
||||||
|
else
|
||||||
|
states.pop
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
when :function_expected
|
||||||
|
if scan(/\s+/)
|
||||||
|
kind = :space
|
||||||
|
elsif scan(/&/)
|
||||||
|
kind = :operator
|
||||||
|
elsif match = scan(/#{RE::IDENTIFIER}/o)
|
||||||
|
kind = :function
|
||||||
|
states.pop
|
||||||
|
else
|
||||||
|
states.pop
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state!', tokens, states
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens, states
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens, states unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -4,9 +4,12 @@ module Scanners
|
||||||
class Plaintext < Scanner
|
class Plaintext < Scanner
|
||||||
|
|
||||||
register_for :plaintext, :plain
|
register_for :plaintext, :plain
|
||||||
|
title 'Plain text'
|
||||||
|
|
||||||
include Streamable
|
include Streamable
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = [:plain]
|
||||||
|
|
||||||
def scan_tokens tokens, options
|
def scan_tokens tokens, options
|
||||||
text = (scan_until(/\z/) || '')
|
text = (scan_until(/\z/) || '')
|
||||||
tokens << [text, :plain]
|
tokens << [text, :plain]
|
|
@ -0,0 +1,285 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
# Bases on pygments' PythonLexer, see
|
||||||
|
# http://dev.pocoo.org/projects/pygments/browser/pygments/lexers/agile.py.
|
||||||
|
class Python < Scanner
|
||||||
|
|
||||||
|
include Streamable
|
||||||
|
|
||||||
|
register_for :python
|
||||||
|
file_extension 'py'
|
||||||
|
|
||||||
|
KEYWORDS = [
|
||||||
|
'and', 'as', 'assert', 'break', 'class', 'continue', 'def',
|
||||||
|
'del', 'elif', 'else', 'except', 'finally', 'for',
|
||||||
|
'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'not',
|
||||||
|
'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
|
||||||
|
'nonlocal', # new in Python 3
|
||||||
|
]
|
||||||
|
|
||||||
|
OLD_KEYWORDS = [
|
||||||
|
'exec', 'print', # gone in Python 3
|
||||||
|
]
|
||||||
|
|
||||||
|
PREDEFINED_METHODS_AND_TYPES = %w[
|
||||||
|
__import__ abs all any apply basestring bin bool buffer
|
||||||
|
bytearray bytes callable chr classmethod cmp coerce compile
|
||||||
|
complex delattr dict dir divmod enumerate eval execfile exit
|
||||||
|
file filter float frozenset getattr globals hasattr hash hex id
|
||||||
|
input int intern isinstance issubclass iter len list locals
|
||||||
|
long map max min next object oct open ord pow property range
|
||||||
|
raw_input reduce reload repr reversed round set setattr slice
|
||||||
|
sorted staticmethod str sum super tuple type unichr unicode
|
||||||
|
vars xrange zip
|
||||||
|
]
|
||||||
|
|
||||||
|
PREDEFINED_EXCEPTIONS = %w[
|
||||||
|
ArithmeticError AssertionError AttributeError
|
||||||
|
BaseException DeprecationWarning EOFError EnvironmentError
|
||||||
|
Exception FloatingPointError FutureWarning GeneratorExit IOError
|
||||||
|
ImportError ImportWarning IndentationError IndexError KeyError
|
||||||
|
KeyboardInterrupt LookupError MemoryError NameError
|
||||||
|
NotImplemented NotImplementedError OSError OverflowError
|
||||||
|
OverflowWarning PendingDeprecationWarning ReferenceError
|
||||||
|
RuntimeError RuntimeWarning StandardError StopIteration
|
||||||
|
SyntaxError SyntaxWarning SystemError SystemExit TabError
|
||||||
|
TypeError UnboundLocalError UnicodeDecodeError
|
||||||
|
UnicodeEncodeError UnicodeError UnicodeTranslateError
|
||||||
|
UnicodeWarning UserWarning ValueError Warning ZeroDivisionError
|
||||||
|
]
|
||||||
|
|
||||||
|
PREDEFINED_VARIABLES_AND_CONSTANTS = [
|
||||||
|
'False', 'True', 'None', # "keywords" since Python 3
|
||||||
|
'self', 'Ellipsis', 'NotImplemented',
|
||||||
|
]
|
||||||
|
|
||||||
|
IDENT_KIND = WordList.new(:ident).
|
||||||
|
add(KEYWORDS, :keyword).
|
||||||
|
add(OLD_KEYWORDS, :old_keyword).
|
||||||
|
add(PREDEFINED_METHODS_AND_TYPES, :predefined).
|
||||||
|
add(PREDEFINED_VARIABLES_AND_CONSTANTS, :pre_constant).
|
||||||
|
add(PREDEFINED_EXCEPTIONS, :exception)
|
||||||
|
|
||||||
|
NAME = / [^\W\d] \w* /x
|
||||||
|
ESCAPE = / [abfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} | N\{[-\w ]+\} /x
|
||||||
|
|
||||||
|
OPERATOR = /
|
||||||
|
\.\.\. | # ellipsis
|
||||||
|
\.(?!\d) | # dot but not decimal point
|
||||||
|
[,;:()\[\]{}] | # simple delimiters
|
||||||
|
\/\/=? | \*\*=? | # special math
|
||||||
|
[-+*\/%&|^]=? | # ordinary math and binary logic
|
||||||
|
[~`] | # binary complement and inspection
|
||||||
|
<<=? | >>=? | [<>=]=? | != # comparison and assignment
|
||||||
|
/x
|
||||||
|
|
||||||
|
STRING_DELIMITER_REGEXP = Hash.new do |h, delimiter|
|
||||||
|
h[delimiter] = Regexp.union delimiter
|
||||||
|
end
|
||||||
|
|
||||||
|
STRING_CONTENT_REGEXP = Hash.new do |h, delimiter|
|
||||||
|
h[delimiter] = / [^\\\n]+? (?= \\ | $ | #{Regexp.escape(delimiter)} ) /x
|
||||||
|
end
|
||||||
|
|
||||||
|
DEF_NEW_STATE = WordList.new(:initial).
|
||||||
|
add(%w(def), :def_expected).
|
||||||
|
add(%w(import from), :include_expected).
|
||||||
|
add(%w(class), :class_expected)
|
||||||
|
|
||||||
|
DESCRIPTOR = /
|
||||||
|
#{NAME}
|
||||||
|
(?: \. #{NAME} )*
|
||||||
|
| \*
|
||||||
|
/x
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
string_delimiter = nil
|
||||||
|
string_raw = false
|
||||||
|
import_clause = class_name_follows = last_token_dot = false
|
||||||
|
unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
|
||||||
|
from_import_state = []
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
if state == :string
|
||||||
|
if scan(STRING_DELIMITER_REGEXP[string_delimiter])
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
elsif string_delimiter.size == 3 && scan(/\n/)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(STRING_CONTENT_REGEXP[string_delimiter])
|
||||||
|
kind = :content
|
||||||
|
elsif !string_raw && scan(/ \\ #{ESCAPE} /ox)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/ \\ #{UNICODE_ESCAPE} /ox)
|
||||||
|
kind = :char
|
||||||
|
elsif scan(/ \\ . /x)
|
||||||
|
kind = :content
|
||||||
|
elsif scan(/ \\ | $ /x)
|
||||||
|
tokens << [:close, :string]
|
||||||
|
kind = :error
|
||||||
|
state = :initial
|
||||||
|
else
|
||||||
|
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens, state
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif match = scan(/ [ \t]+ | \\\n /x)
|
||||||
|
tokens << [match, :space]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif match = scan(/\n/)
|
||||||
|
tokens << [match, :space]
|
||||||
|
state = :initial if state == :include_expected
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif match = scan(/ \# [^\n]* /mx)
|
||||||
|
tokens << [match, :comment]
|
||||||
|
next
|
||||||
|
|
||||||
|
elsif state == :initial
|
||||||
|
|
||||||
|
if scan(/#{OPERATOR}/o)
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif match = scan(/(u?r?|b)?("""|"|'''|')/i)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
string_delimiter = self[2]
|
||||||
|
string_raw = false
|
||||||
|
modifiers = self[1]
|
||||||
|
unless modifiers.empty?
|
||||||
|
string_raw = !!modifiers.index(?r)
|
||||||
|
tokens << [modifiers, :modifier]
|
||||||
|
match = string_delimiter
|
||||||
|
end
|
||||||
|
state = :string
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
# TODO: backticks
|
||||||
|
|
||||||
|
elsif match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
|
||||||
|
kind = IDENT_KIND[match]
|
||||||
|
# TODO: keyword arguments
|
||||||
|
kind = :ident if last_token_dot
|
||||||
|
if kind == :old_keyword
|
||||||
|
kind = check(/\(/) ? :ident : :keyword
|
||||||
|
elsif kind == :predefined && check(/ *=/)
|
||||||
|
kind = :ident
|
||||||
|
elsif kind == :keyword
|
||||||
|
state = DEF_NEW_STATE[match]
|
||||||
|
from_import_state << match.to_sym if state == :include_expected
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/@[a-zA-Z0-9_.]+[lL]?/)
|
||||||
|
kind = :decorator
|
||||||
|
|
||||||
|
elsif scan(/0[xX][0-9A-Fa-f]+[lL]?/)
|
||||||
|
kind = :hex
|
||||||
|
|
||||||
|
elsif scan(/0[bB][01]+[lL]?/)
|
||||||
|
kind = :bin
|
||||||
|
|
||||||
|
elsif match = scan(/(?:\d*\.\d+|\d+\.\d*)(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
|
||||||
|
kind = :float
|
||||||
|
if scan(/[jJ]/)
|
||||||
|
match << matched
|
||||||
|
kind = :imaginary
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif scan(/0[oO][0-7]+|0[0-7]+(?![89.eE])[lL]?/)
|
||||||
|
kind = :oct
|
||||||
|
|
||||||
|
elsif match = scan(/\d+([lL])?/)
|
||||||
|
kind = :integer
|
||||||
|
if self[1] == nil && scan(/[jJ]/)
|
||||||
|
match << matched
|
||||||
|
kind = :imaginary
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif state == :def_expected
|
||||||
|
state = :initial
|
||||||
|
if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
|
||||||
|
kind = :method
|
||||||
|
else
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif state == :class_expected
|
||||||
|
state = :initial
|
||||||
|
if match = scan(unicode ? /#{NAME}/uo : /#{NAME}/o)
|
||||||
|
kind = :class
|
||||||
|
else
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif state == :include_expected
|
||||||
|
if match = scan(unicode ? /#{DESCRIPTOR}/uo : /#{DESCRIPTOR}/o)
|
||||||
|
kind = :include
|
||||||
|
if match == 'as'
|
||||||
|
kind = :keyword
|
||||||
|
from_import_state << :as
|
||||||
|
elsif from_import_state.first == :from && match == 'import'
|
||||||
|
kind = :keyword
|
||||||
|
from_import_state << :import
|
||||||
|
elsif from_import_state.last == :as
|
||||||
|
# kind = match[0,1][unicode ? /[[:upper:]]/u : /[[:upper:]]/] ? :class : :method
|
||||||
|
kind = :ident
|
||||||
|
from_import_state.pop
|
||||||
|
elsif IDENT_KIND[match] == :keyword
|
||||||
|
unscan
|
||||||
|
match = nil
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
end
|
||||||
|
elsif match = scan(/,/)
|
||||||
|
from_import_state.pop if from_import_state.last == :as
|
||||||
|
kind = :operator
|
||||||
|
else
|
||||||
|
from_import_state = []
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise_inspect 'Unknown state', tokens, state
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens, state
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens, state unless match
|
||||||
|
|
||||||
|
last_token_dot = match == '.'
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
if state == :string
|
||||||
|
tokens << [:close, :string]
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -5,12 +5,13 @@ module Scanners
|
||||||
load :ruby
|
load :ruby
|
||||||
|
|
||||||
# RHTML Scanner
|
# RHTML Scanner
|
||||||
#
|
|
||||||
# $Id$
|
|
||||||
class RHTML < Scanner
|
class RHTML < Scanner
|
||||||
|
|
||||||
include Streamable
|
include Streamable
|
||||||
register_for :rhtml
|
register_for :rhtml
|
||||||
|
title 'HTML ERB Template'
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
|
||||||
|
|
||||||
ERB_RUBY_BLOCK = /
|
ERB_RUBY_BLOCK = /
|
||||||
<%(?!%)[=-]?
|
<%(?!%)[=-]?
|
|
@ -21,6 +21,10 @@ module Scanners
|
||||||
file_extension 'rb'
|
file_extension 'rb'
|
||||||
|
|
||||||
helper :patterns
|
helper :patterns
|
||||||
|
|
||||||
|
if not defined? EncodingError
|
||||||
|
EncodingError = Class.new Exception
|
||||||
|
end
|
||||||
|
|
||||||
private
|
private
|
||||||
def scan_tokens tokens, options
|
def scan_tokens tokens, options
|
||||||
|
@ -31,9 +35,10 @@ module Scanners
|
||||||
state = :initial
|
state = :initial
|
||||||
depth = nil
|
depth = nil
|
||||||
inline_block_stack = []
|
inline_block_stack = []
|
||||||
|
unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
|
||||||
|
|
||||||
patterns = Patterns # avoid constant lookup
|
patterns = Patterns # avoid constant lookup
|
||||||
|
|
||||||
until eos?
|
until eos?
|
||||||
match = nil
|
match = nil
|
||||||
kind = nil
|
kind = nil
|
||||||
|
@ -124,14 +129,15 @@ module Scanners
|
||||||
# {{{
|
# {{{
|
||||||
if match = scan(/[ \t\f]+/)
|
if match = scan(/[ \t\f]+/)
|
||||||
kind = :space
|
kind = :space
|
||||||
match << scan(/\s*/) unless eos? or heredocs
|
match << scan(/\s*/) unless eos? || heredocs
|
||||||
|
value_expected = true if match.index(?\n)
|
||||||
tokens << [match, kind]
|
tokens << [match, kind]
|
||||||
next
|
next
|
||||||
|
|
||||||
elsif match = scan(/\\?\n/)
|
elsif match = scan(/\\?\n/)
|
||||||
kind = :space
|
kind = :space
|
||||||
if match == "\n"
|
if match == "\n"
|
||||||
value_expected = true # FIXME not quite true
|
value_expected = true
|
||||||
state = :initial if state == :undef_comma_expected
|
state = :initial if state == :undef_comma_expected
|
||||||
end
|
end
|
||||||
if heredocs
|
if heredocs
|
||||||
|
@ -146,17 +152,21 @@ module Scanners
|
||||||
tokens << [match, kind]
|
tokens << [match, kind]
|
||||||
next
|
next
|
||||||
|
|
||||||
|
elsif bol? && match = scan(/\#!.*/)
|
||||||
|
tokens << [match, :doctype]
|
||||||
|
next
|
||||||
|
|
||||||
elsif match = scan(/\#.*/) or
|
elsif match = scan(/\#.*/) or
|
||||||
( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
|
( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
|
||||||
kind = :comment
|
kind = :comment
|
||||||
value_expected = true
|
|
||||||
tokens << [match, kind]
|
tokens << [match, kind]
|
||||||
next
|
next
|
||||||
|
|
||||||
elsif state == :initial
|
elsif state == :initial
|
||||||
|
|
||||||
# IDENTS #
|
# IDENTS #
|
||||||
if match = scan(/#{patterns::METHOD_NAME}/o)
|
if match = scan(unicode ? /#{patterns::METHOD_NAME}/uo :
|
||||||
|
/#{patterns::METHOD_NAME}/o)
|
||||||
if last_token_dot
|
if last_token_dot
|
||||||
kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
|
kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
|
||||||
else
|
else
|
||||||
|
@ -165,13 +175,12 @@ module Scanners
|
||||||
kind = :constant
|
kind = :constant
|
||||||
elsif kind == :reserved
|
elsif kind == :reserved
|
||||||
state = patterns::DEF_NEW_STATE[match]
|
state = patterns::DEF_NEW_STATE[match]
|
||||||
|
value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
## experimental!
|
value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
|
||||||
value_expected = :set if
|
|
||||||
patterns::REGEXP_ALLOWED[match] or check(/#{patterns::VALUE_FOLLOWS}/o)
|
|
||||||
|
|
||||||
elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o)
|
elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
|
||||||
kind = :ident
|
kind = :ident
|
||||||
value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
|
value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
|
||||||
|
|
||||||
|
@ -190,6 +199,7 @@ module Scanners
|
||||||
depth -= 1
|
depth -= 1
|
||||||
if depth == 0 # closing brace of inline block reached
|
if depth == 0 # closing brace of inline block reached
|
||||||
state, depth, heredocs = inline_block_stack.pop
|
state, depth, heredocs = inline_block_stack.pop
|
||||||
|
heredocs = nil if heredocs && heredocs.empty?
|
||||||
tokens << [match, :inline_delimiter]
|
tokens << [match, :inline_delimiter]
|
||||||
kind = :inline
|
kind = :inline
|
||||||
match = :close
|
match = :close
|
||||||
|
@ -211,8 +221,9 @@ module Scanners
|
||||||
interpreted = true
|
interpreted = true
|
||||||
state = patterns::StringState.new :regexp, interpreted, match
|
state = patterns::StringState.new :regexp, interpreted, match
|
||||||
|
|
||||||
elsif match = scan(/#{patterns::NUMERIC}/o)
|
# elsif match = scan(/[-+]?#{patterns::NUMERIC}/o)
|
||||||
kind = if self[1] then :float else :integer end
|
elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
|
||||||
|
kind = self[1] ? :float : :integer
|
||||||
|
|
||||||
elsif match = scan(/#{patterns::SYMBOL}/o)
|
elsif match = scan(/#{patterns::SYMBOL}/o)
|
||||||
case delim = match[1]
|
case delim = match[1]
|
||||||
|
@ -273,6 +284,21 @@ module Scanners
|
||||||
kind = :class_variable
|
kind = :class_variable
|
||||||
|
|
||||||
else
|
else
|
||||||
|
if !unicode
|
||||||
|
# check for unicode
|
||||||
|
debug, $DEBUG = $DEBUG, false
|
||||||
|
begin
|
||||||
|
if check(/./mu).size > 1
|
||||||
|
# seems like we should try again with unicode
|
||||||
|
unicode = true
|
||||||
|
end
|
||||||
|
rescue
|
||||||
|
# bad unicode char; use getch
|
||||||
|
ensure
|
||||||
|
$DEBUG = debug
|
||||||
|
end
|
||||||
|
next if unicode
|
||||||
|
end
|
||||||
kind = :error
|
kind = :error
|
||||||
match = getch
|
match = getch
|
||||||
|
|
||||||
|
@ -280,12 +306,29 @@ module Scanners
|
||||||
|
|
||||||
elsif state == :def_expected
|
elsif state == :def_expected
|
||||||
state = :initial
|
state = :initial
|
||||||
if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
|
if scan(/self\./)
|
||||||
|
tokens << ['self', :pre_constant]
|
||||||
|
tokens << ['.', :operator]
|
||||||
|
end
|
||||||
|
if match = scan(unicode ? /(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/uo :
|
||||||
|
/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
|
||||||
kind = :method
|
kind = :method
|
||||||
else
|
else
|
||||||
next
|
next
|
||||||
end
|
end
|
||||||
|
|
||||||
|
elsif state == :module_expected
|
||||||
|
if match = scan(/<</)
|
||||||
|
kind = :operator
|
||||||
|
else
|
||||||
|
state = :initial
|
||||||
|
if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
|
||||||
|
kind = :class
|
||||||
|
else
|
||||||
|
next
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
elsif state == :undef_expected
|
elsif state == :undef_expected
|
||||||
state = :undef_comma_expected
|
state = :undef_comma_expected
|
||||||
if match = scan(/#{patterns::METHOD_NAME_EX}/o)
|
if match = scan(/#{patterns::METHOD_NAME_EX}/o)
|
||||||
|
@ -307,6 +350,18 @@ module Scanners
|
||||||
next
|
next
|
||||||
end
|
end
|
||||||
|
|
||||||
|
elsif state == :alias_expected
|
||||||
|
match = scan(unicode ? /(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/uo :
|
||||||
|
/(#{patterns::METHOD_NAME_OR_SYMBOL})([ \t]+)(#{patterns::METHOD_NAME_OR_SYMBOL})/o)
|
||||||
|
|
||||||
|
if match
|
||||||
|
tokens << [self[1], (self[1][0] == ?: ? :symbol : :method)]
|
||||||
|
tokens << [self[2], :space]
|
||||||
|
tokens << [self[3], (self[3][0] == ?: ? :symbol : :method)]
|
||||||
|
end
|
||||||
|
state = :initial
|
||||||
|
next
|
||||||
|
|
||||||
elsif state == :undef_comma_expected
|
elsif state == :undef_comma_expected
|
||||||
if match = scan(/,/)
|
if match = scan(/,/)
|
||||||
kind = :operator
|
kind = :operator
|
||||||
|
@ -316,25 +371,15 @@ module Scanners
|
||||||
next
|
next
|
||||||
end
|
end
|
||||||
|
|
||||||
elsif state == :module_expected
|
|
||||||
if match = scan(/<</)
|
|
||||||
kind = :operator
|
|
||||||
else
|
|
||||||
state = :initial
|
|
||||||
if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
|
|
||||||
kind = :class
|
|
||||||
else
|
|
||||||
next
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
end
|
||||||
# }}}
|
# }}}
|
||||||
|
|
||||||
value_expected = value_expected == :set
|
unless kind == :error
|
||||||
last_token_dot = last_token_dot == :set
|
value_expected = value_expected == :set
|
||||||
|
last_token_dot = last_token_dot == :set
|
||||||
if $DEBUG and not kind
|
end
|
||||||
|
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
raise_inspect 'Error token %p in line %d' %
|
raise_inspect 'Error token %p in line %d' %
|
||||||
[[match, kind], line], tokens, state
|
[[match, kind], line], tokens, state
|
||||||
end
|
end
|
|
@ -1,3 +1,4 @@
|
||||||
|
# encoding: utf-8
|
||||||
module CodeRay
|
module CodeRay
|
||||||
module Scanners
|
module Scanners
|
||||||
|
|
||||||
|
@ -14,19 +15,14 @@ module Scanners
|
||||||
|
|
||||||
DEF_KEYWORDS = %w[ def ]
|
DEF_KEYWORDS = %w[ def ]
|
||||||
UNDEF_KEYWORDS = %w[ undef ]
|
UNDEF_KEYWORDS = %w[ undef ]
|
||||||
|
ALIAS_KEYWORDS = %w[ alias ]
|
||||||
MODULE_KEYWORDS = %w[class module]
|
MODULE_KEYWORDS = %w[class module]
|
||||||
DEF_NEW_STATE = WordList.new(:initial).
|
DEF_NEW_STATE = WordList.new(:initial).
|
||||||
add(DEF_KEYWORDS, :def_expected).
|
add(DEF_KEYWORDS, :def_expected).
|
||||||
add(UNDEF_KEYWORDS, :undef_expected).
|
add(UNDEF_KEYWORDS, :undef_expected).
|
||||||
|
add(ALIAS_KEYWORDS, :alias_expected).
|
||||||
add(MODULE_KEYWORDS, :module_expected)
|
add(MODULE_KEYWORDS, :module_expected)
|
||||||
|
|
||||||
IDENTS_ALLOWING_REGEXP = %w[
|
|
||||||
and or not while until unless if then elsif when sub sub! gsub gsub!
|
|
||||||
scan slice slice! split
|
|
||||||
]
|
|
||||||
REGEXP_ALLOWED = WordList.new(false).
|
|
||||||
add(IDENTS_ALLOWING_REGEXP, :set)
|
|
||||||
|
|
||||||
PREDEFINED_CONSTANTS = %w[
|
PREDEFINED_CONSTANTS = %w[
|
||||||
nil true false self
|
nil true false self
|
||||||
DATA ARGV ARGF __FILE__ __LINE__
|
DATA ARGV ARGF __FILE__ __LINE__
|
||||||
|
@ -36,24 +32,25 @@ module Scanners
|
||||||
add(RESERVED_WORDS, :reserved).
|
add(RESERVED_WORDS, :reserved).
|
||||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
add(PREDEFINED_CONSTANTS, :pre_constant)
|
||||||
|
|
||||||
IDENT = /[a-z_][\w_]*/i
|
IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
|
||||||
|
|
||||||
METHOD_NAME = / #{IDENT} [?!]? /ox
|
METHOD_NAME = / #{IDENT} [?!]? /ox
|
||||||
METHOD_NAME_OPERATOR = /
|
METHOD_NAME_OPERATOR = /
|
||||||
\*\*? # multiplication and power
|
\*\*? # multiplication and power
|
||||||
| [-+]@? # plus, minus
|
| [-+~]@? # plus, minus, tilde with and without at sign
|
||||||
| [\/%&|^`~] # division, modulo or format strings, &and, |or, ^xor, `system`, tilde
|
| [\/%&|^`] # division, modulo or format strings, and, or, xor, system
|
||||||
| \[\]=? # array getter and setter
|
| \[\]=? # array getter and setter
|
||||||
| << | >> # append or shift left, shift right
|
| << | >> # append or shift left, shift right
|
||||||
| <=?>? | >=? # comparison, rocket operator
|
| <=?>? | >=? # comparison, rocket operator
|
||||||
| ===? # simple equality and case equality
|
| ===? | =~ # simple equality, case equality, match
|
||||||
|
| ![~=@]? # negation with and without at sign, not-equal and not-match
|
||||||
/ox
|
/ox
|
||||||
METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
|
METHOD_NAME_EX = / #{IDENT} (?:[?!]|=(?!>))? | #{METHOD_NAME_OPERATOR} /ox
|
||||||
INSTANCE_VARIABLE = / @ #{IDENT} /ox
|
INSTANCE_VARIABLE = / @ #{IDENT} /ox
|
||||||
CLASS_VARIABLE = / @@ #{IDENT} /ox
|
CLASS_VARIABLE = / @@ #{IDENT} /ox
|
||||||
OBJECT_VARIABLE = / @@? #{IDENT} /ox
|
OBJECT_VARIABLE = / @@? #{IDENT} /ox
|
||||||
GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
|
GLOBAL_VARIABLE = / \$ (?: #{IDENT} | [1-9]\d* | 0\w* | [~&+`'=\/,;_.<>!@$?*":\\] | -[a-zA-Z_0-9] ) /ox
|
||||||
PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} |#{OBJECT_VARIABLE} /ox
|
PREFIX_VARIABLE = / #{GLOBAL_VARIABLE} | #{OBJECT_VARIABLE} /ox
|
||||||
VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
|
VARIABLE = / @?@? #{IDENT} | #{GLOBAL_VARIABLE} /ox
|
||||||
|
|
||||||
QUOTE_TO_TYPE = {
|
QUOTE_TO_TYPE = {
|
||||||
|
@ -63,7 +60,7 @@ module Scanners
|
||||||
QUOTE_TO_TYPE.default = :string
|
QUOTE_TO_TYPE.default = :string
|
||||||
|
|
||||||
REGEXP_MODIFIERS = /[mixounse]*/
|
REGEXP_MODIFIERS = /[mixounse]*/
|
||||||
REGEXP_SYMBOLS = /[|?*+?(){}\[\].^$]/
|
REGEXP_SYMBOLS = /[|?*+(){}\[\].^$]/
|
||||||
|
|
||||||
DECIMAL = /\d+(?:_\d+)*/
|
DECIMAL = /\d+(?:_\d+)*/
|
||||||
OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
|
OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
|
||||||
|
@ -73,7 +70,7 @@ module Scanners
|
||||||
EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
|
EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
|
||||||
FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
|
FLOAT_SUFFIX = / #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? /ox
|
||||||
FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
|
FLOAT_OR_INT = / #{DECIMAL} (?: #{FLOAT_SUFFIX} () )? /ox
|
||||||
NUMERIC = / [-+]? (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
|
NUMERIC = / (?: (?=0) (?: #{OCTAL} | #{HEXADECIMAL} | #{BINARY} ) | #{FLOAT_OR_INT} ) /ox
|
||||||
|
|
||||||
SYMBOL = /
|
SYMBOL = /
|
||||||
:
|
:
|
||||||
|
@ -83,26 +80,32 @@ module Scanners
|
||||||
| ['"]
|
| ['"]
|
||||||
)
|
)
|
||||||
/ox
|
/ox
|
||||||
|
METHOD_NAME_OR_SYMBOL = / #{METHOD_NAME_EX} | #{SYMBOL} /ox
|
||||||
|
|
||||||
# TODO investigste \M, \c and \C escape sequences
|
SIMPLE_ESCAPE = /
|
||||||
# (?: M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-)? (?: \\ (?: [0-7]{3} | x[0-9A-Fa-f]{2} | . ) )
|
|
||||||
# assert_equal(225, ?\M-a)
|
|
||||||
# assert_equal(129, ?\M-\C-a)
|
|
||||||
ESCAPE = /
|
|
||||||
[abefnrstv]
|
[abefnrstv]
|
||||||
| M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-
|
|
||||||
| [0-7]{1,3}
|
| [0-7]{1,3}
|
||||||
| x[0-9A-Fa-f]{1,2}
|
| x[0-9A-Fa-f]{1,2}
|
||||||
| .
|
| .?
|
||||||
/mx
|
/mx
|
||||||
|
|
||||||
|
CONTROL_META_ESCAPE = /
|
||||||
|
(?: M-|C-|c )
|
||||||
|
(?: \\ (?: M-|C-|c ) )*
|
||||||
|
(?: [^\\] | \\ #{SIMPLE_ESCAPE} )?
|
||||||
|
/mox
|
||||||
|
|
||||||
|
ESCAPE = /
|
||||||
|
#{CONTROL_META_ESCAPE} | #{SIMPLE_ESCAPE}
|
||||||
|
/mox
|
||||||
|
|
||||||
CHARACTER = /
|
CHARACTER = /
|
||||||
\?
|
\?
|
||||||
(?:
|
(?:
|
||||||
[^\s\\]
|
[^\s\\]
|
||||||
| \\ #{ESCAPE}
|
| \\ #{ESCAPE}
|
||||||
)
|
)
|
||||||
/mx
|
/mox
|
||||||
|
|
||||||
# NOTE: This is not completely correct, but
|
# NOTE: This is not completely correct, but
|
||||||
# nobody needs heredoc delimiters ending with \n.
|
# nobody needs heredoc delimiters ending with \n.
|
||||||
|
@ -129,25 +132,29 @@ module Scanners
|
||||||
/mx
|
/mx
|
||||||
|
|
||||||
# Checks for a valid value to follow. This enables
|
# Checks for a valid value to follow. This enables
|
||||||
# fancy_allowed in method calls.
|
# value_expected in method calls without parentheses.
|
||||||
VALUE_FOLLOWS = /
|
VALUE_FOLLOWS = /
|
||||||
\s+
|
(?>[ \t\f\v]+)
|
||||||
(?:
|
(?:
|
||||||
[%\/][^\s=]
|
[%\/][^\s=]
|
||||||
|
|
| <<-?\S
|
||||||
<<-?\S
|
| [-+] \d
|
||||||
|
|
| #{CHARACTER}
|
||||||
#{CHARACTER}
|
|
||||||
)
|
)
|
||||||
/x
|
/x
|
||||||
|
KEYWORDS_EXPECTING_VALUE = WordList.new.add(%w[
|
||||||
|
and end in or unless begin
|
||||||
|
defined? ensure redo super until
|
||||||
|
break do next rescue then
|
||||||
|
when case else for retry
|
||||||
|
while elsif if not return
|
||||||
|
yield
|
||||||
|
])
|
||||||
|
|
||||||
RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
|
RUBYDOC_OR_DATA = / #{RUBYDOC} | #{DATA} /xo
|
||||||
|
|
||||||
RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
|
RDOC_DATA_START = / ^=begin (?!\S) | ^__END__$ /x
|
||||||
|
|
||||||
# FIXME: \s and = are only a workaround, they are still allowed
|
|
||||||
# as delimiters.
|
|
||||||
FANCY_START_SAVE = / % ( [qQwWxsr] | (?![a-zA-Z0-9\s=]) ) ([^a-zA-Z0-9]) /mx
|
|
||||||
FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
|
FANCY_START_CORRECT = / % ( [qQwWxsr] | (?![a-zA-Z0-9]) ) ([^a-zA-Z0-9]) /mx
|
||||||
|
|
||||||
FancyStringType = {
|
FancyStringType = {
|
||||||
|
@ -170,17 +177,18 @@ module Scanners
|
||||||
{ }
|
{ }
|
||||||
] ]
|
] ]
|
||||||
|
|
||||||
CLOSING_PAREN.values.each { |o| o.freeze } # debug, if I try to change it with <<
|
CLOSING_PAREN.each { |k,v| k.freeze; v.freeze } # debug, if I try to change it with <<
|
||||||
OPENING_PAREN = CLOSING_PAREN.invert
|
OPENING_PAREN = CLOSING_PAREN.invert
|
||||||
|
|
||||||
STRING_PATTERN = Hash.new { |h, k|
|
STRING_PATTERN = Hash.new do |h, k|
|
||||||
delim, interpreted = *k
|
delim, interpreted = *k
|
||||||
delim_pattern = Regexp.escape(delim.dup)
|
delim_pattern = Regexp.escape(delim.dup) # dup: workaround for old Ruby
|
||||||
if closing_paren = CLOSING_PAREN[delim]
|
if closing_paren = CLOSING_PAREN[delim]
|
||||||
|
delim_pattern = delim_pattern[0..-1] if defined? JRUBY_VERSION # JRuby fix
|
||||||
delim_pattern << Regexp.escape(closing_paren)
|
delim_pattern << Regexp.escape(closing_paren)
|
||||||
end
|
end
|
||||||
|
delim_pattern << '\\\\' unless delim == '\\'
|
||||||
|
|
||||||
special_escapes =
|
special_escapes =
|
||||||
case interpreted
|
case interpreted
|
||||||
when :regexp_symbols
|
when :regexp_symbols
|
||||||
|
@ -188,18 +196,18 @@ module Scanners
|
||||||
when :words
|
when :words
|
||||||
'| \s'
|
'| \s'
|
||||||
end
|
end
|
||||||
|
|
||||||
h[k] =
|
h[k] =
|
||||||
if interpreted and not delim == '#'
|
if interpreted and not delim == '#'
|
||||||
/ (?= [#{delim_pattern}\\] | \# [{$@] #{special_escapes} ) /mx
|
/ (?= [#{delim_pattern}] | \# [{$@] #{special_escapes} ) /mx
|
||||||
else
|
else
|
||||||
/ (?= [#{delim_pattern}\\] #{special_escapes} ) /mx
|
/ (?= [#{delim_pattern}] #{special_escapes} ) /mx
|
||||||
end
|
end
|
||||||
}
|
end
|
||||||
|
|
||||||
HEREDOC_PATTERN = Hash.new { |h, k|
|
HEREDOC_PATTERN = Hash.new do |h, k|
|
||||||
delim, interpreted, indented = *k
|
delim, interpreted, indented = *k
|
||||||
delim_pattern = Regexp.escape(delim.dup)
|
delim_pattern = Regexp.escape(delim.dup) # dup: workaround for old Ruby
|
||||||
delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
|
delim_pattern = / \n #{ '(?>[\ \t]*)' if indented } #{ Regexp.new delim_pattern } $ /x
|
||||||
h[k] =
|
h[k] =
|
||||||
if interpreted
|
if interpreted
|
||||||
|
@ -207,12 +215,12 @@ module Scanners
|
||||||
else
|
else
|
||||||
/ (?= #{delim_pattern}() | \\ ) /mx
|
/ (?= #{delim_pattern}() | \\ ) /mx
|
||||||
end
|
end
|
||||||
}
|
end
|
||||||
|
|
||||||
def initialize kind, interpreted, delim, heredoc = false
|
def initialize kind, interpreted, delim, heredoc = false
|
||||||
if heredoc
|
if heredoc
|
||||||
pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
|
pattern = HEREDOC_PATTERN[ [delim, interpreted, heredoc == :indented] ]
|
||||||
delim = nil
|
delim = nil
|
||||||
else
|
else
|
||||||
pattern = STRING_PATTERN[ [delim, interpreted] ]
|
pattern = STRING_PATTERN[ [delim, interpreted] ]
|
||||||
if paren = CLOSING_PAREN[delim]
|
if paren = CLOSING_PAREN[delim]
|
|
@ -5,8 +5,11 @@ module CodeRay
|
||||||
# Thanks to murphy for putting CodeRay into public.
|
# Thanks to murphy for putting CodeRay into public.
|
||||||
class Scheme < Scanner
|
class Scheme < Scanner
|
||||||
|
|
||||||
|
# TODO: function defs
|
||||||
|
# TODO: built-in functions
|
||||||
|
|
||||||
register_for :scheme
|
register_for :scheme
|
||||||
file_extension :scm
|
file_extension 'scm'
|
||||||
|
|
||||||
CORE_FORMS = %w[
|
CORE_FORMS = %w[
|
||||||
lambda let let* letrec syntax-case define-syntax let-syntax
|
lambda let let* letrec syntax-case define-syntax let-syntax
|
||||||
|
@ -120,7 +123,7 @@ module CodeRay
|
||||||
end
|
end
|
||||||
|
|
||||||
match ||= matched
|
match ||= matched
|
||||||
if $DEBUG and not kind
|
if $CODERAY_DEBUG and not kind
|
||||||
raise_inspect 'Error token %p in line %d' %
|
raise_inspect 'Error token %p in line %d' %
|
||||||
[[match, kind], line], tokens
|
[[match, kind], line], tokens
|
||||||
end
|
end
|
|
@ -0,0 +1,162 @@
|
||||||
|
module CodeRay module Scanners
|
||||||
|
|
||||||
|
# by Josh Goebel
|
||||||
|
class SQL < Scanner
|
||||||
|
|
||||||
|
register_for :sql
|
||||||
|
|
||||||
|
RESERVED_WORDS = %w(
|
||||||
|
create database table index trigger drop primary key set select
|
||||||
|
insert update delete replace into
|
||||||
|
on from values before and or if exists case when
|
||||||
|
then else as group order by avg where
|
||||||
|
join inner outer union engine not
|
||||||
|
like end using collate show columns begin
|
||||||
|
)
|
||||||
|
|
||||||
|
PREDEFINED_TYPES = %w(
|
||||||
|
char varchar enum binary text tinytext mediumtext
|
||||||
|
longtext blob tinyblob mediumblob longblob timestamp
|
||||||
|
date time datetime year double decimal float int
|
||||||
|
integer tinyint mediumint bigint smallint unsigned bit
|
||||||
|
bool boolean hex bin oct
|
||||||
|
)
|
||||||
|
|
||||||
|
PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
|
||||||
|
|
||||||
|
DIRECTIVES = %w( auto_increment unique default charset )
|
||||||
|
|
||||||
|
PREDEFINED_CONSTANTS = %w( null true false )
|
||||||
|
|
||||||
|
IDENT_KIND = CaseIgnoringWordList.new(:ident).
|
||||||
|
add(RESERVED_WORDS, :reserved).
|
||||||
|
add(PREDEFINED_TYPES, :pre_type).
|
||||||
|
add(PREDEFINED_CONSTANTS, :pre_constant).
|
||||||
|
add(PREDEFINED_FUNCTIONS, :predefined).
|
||||||
|
add(DIRECTIVES, :directive)
|
||||||
|
|
||||||
|
ESCAPE = / [rbfntv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
|
||||||
|
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||||
|
|
||||||
|
STRING_PREFIXES = /[xnb]|_\w+/i
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
state = :initial
|
||||||
|
string_type = nil
|
||||||
|
string_content = ''
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
|
||||||
|
if state == :initial
|
||||||
|
|
||||||
|
if scan(/ \s+ | \\\n /x)
|
||||||
|
kind = :space
|
||||||
|
|
||||||
|
elsif scan(/^(?:--\s?|#).*/)
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
|
||||||
|
kind = :operator
|
||||||
|
|
||||||
|
elsif scan(/(#{STRING_PREFIXES})?([`"'])/o)
|
||||||
|
prefix = self[1]
|
||||||
|
string_type = self[2]
|
||||||
|
tokens << [:open, :string]
|
||||||
|
tokens << [prefix, :modifier] if prefix
|
||||||
|
match = string_type
|
||||||
|
state = :string
|
||||||
|
kind = :delimiter
|
||||||
|
|
||||||
|
elsif match = scan(/ @? [A-Za-z_][A-Za-z_0-9]* /x)
|
||||||
|
kind = match[0] == ?@ ? :variable : IDENT_KIND[match.downcase]
|
||||||
|
|
||||||
|
elsif scan(/0[xX][0-9A-Fa-f]+/)
|
||||||
|
kind = :hex
|
||||||
|
|
||||||
|
elsif scan(/0[0-7]+(?![89.eEfF])/)
|
||||||
|
kind = :oct
|
||||||
|
|
||||||
|
elsif scan(/(?>\d+)(?![.eEfF])/)
|
||||||
|
kind = :integer
|
||||||
|
|
||||||
|
elsif scan(/\d[fF]|\d*\.\d+(?:[eE][+-]?\d+)?|\d+[eE][+-]?\d+/)
|
||||||
|
kind = :float
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif state == :string
|
||||||
|
if match = scan(/[^\\"'`]+/)
|
||||||
|
string_content << match
|
||||||
|
next
|
||||||
|
elsif match = scan(/["'`]/)
|
||||||
|
if string_type == match
|
||||||
|
if peek(1) == string_type # doubling means escape
|
||||||
|
string_content << string_type << getch
|
||||||
|
next
|
||||||
|
end
|
||||||
|
unless string_content.empty?
|
||||||
|
tokens << [string_content, :content]
|
||||||
|
string_content = ''
|
||||||
|
end
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [:close, :string]
|
||||||
|
state = :initial
|
||||||
|
string_type = nil
|
||||||
|
next
|
||||||
|
else
|
||||||
|
string_content << match
|
||||||
|
end
|
||||||
|
next
|
||||||
|
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||||
|
unless string_content.empty?
|
||||||
|
tokens << [string_content, :content]
|
||||||
|
string_content = ''
|
||||||
|
end
|
||||||
|
kind = :char
|
||||||
|
elsif match = scan(/ \\ . /mox)
|
||||||
|
string_content << match
|
||||||
|
next
|
||||||
|
elsif scan(/ \\ | $ /x)
|
||||||
|
unless string_content.empty?
|
||||||
|
tokens << [string_content, :content]
|
||||||
|
string_content = ''
|
||||||
|
end
|
||||||
|
kind = :error
|
||||||
|
state = :initial
|
||||||
|
else
|
||||||
|
raise "else case \" reached; %p not handled." % peek(1), tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
raise 'else-case reached', tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
unless kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens, state
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
tokens
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end end
|
|
@ -5,13 +5,12 @@ module Scanners
|
||||||
|
|
||||||
# XML Scanner
|
# XML Scanner
|
||||||
#
|
#
|
||||||
# $Id$
|
|
||||||
#
|
|
||||||
# Currently this is the same scanner as Scanners::HTML.
|
# Currently this is the same scanner as Scanners::HTML.
|
||||||
class XML < HTML
|
class XML < HTML
|
||||||
|
|
||||||
register_for :xml
|
register_for :xml
|
||||||
|
file_extension 'xml'
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
|
@ -0,0 +1,140 @@
|
||||||
|
module CodeRay
|
||||||
|
module Scanners
|
||||||
|
|
||||||
|
# YAML Scanner
|
||||||
|
#
|
||||||
|
# Based on the YAML scanner from Syntax by Jamis Buck.
|
||||||
|
class YAML < Scanner
|
||||||
|
|
||||||
|
register_for :yaml
|
||||||
|
file_extension 'yml'
|
||||||
|
|
||||||
|
KINDS_NOT_LOC = :all
|
||||||
|
|
||||||
|
def scan_tokens tokens, options
|
||||||
|
|
||||||
|
value_expected = nil
|
||||||
|
state = :initial
|
||||||
|
key_indent = indent = 0
|
||||||
|
|
||||||
|
until eos?
|
||||||
|
|
||||||
|
kind = nil
|
||||||
|
match = nil
|
||||||
|
key_indent = nil if bol?
|
||||||
|
|
||||||
|
if match = scan(/ +[\t ]*/)
|
||||||
|
kind = :space
|
||||||
|
|
||||||
|
elsif match = scan(/\n+/)
|
||||||
|
kind = :space
|
||||||
|
state = :initial if match.index(?\n)
|
||||||
|
|
||||||
|
elsif match = scan(/#.*/)
|
||||||
|
kind = :comment
|
||||||
|
|
||||||
|
elsif bol? and case
|
||||||
|
when match = scan(/---|\.\.\./)
|
||||||
|
tokens << [:open, :head]
|
||||||
|
tokens << [match, :head]
|
||||||
|
tokens << [:close, :head]
|
||||||
|
next
|
||||||
|
when match = scan(/%.*/)
|
||||||
|
tokens << [match, :doctype]
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif state == :value and case
|
||||||
|
when !check(/(?:"[^"]*")(?=: |:$)/) && scan(/"/)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
tokens << [matched, :delimiter]
|
||||||
|
tokens << [matched, :content] if scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)
|
||||||
|
tokens << [matched, :delimiter] if scan(/"/)
|
||||||
|
tokens << [:close, :string]
|
||||||
|
next
|
||||||
|
when match = scan(/[|>][-+]?/)
|
||||||
|
tokens << [:open, :string]
|
||||||
|
tokens << [match, :delimiter]
|
||||||
|
string_indent = key_indent || column(pos - match.size - 1)
|
||||||
|
tokens << [matched, :content] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
|
||||||
|
tokens << [:close, :string]
|
||||||
|
next
|
||||||
|
when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
|
||||||
|
tokens << [match, :string]
|
||||||
|
string_indent = key_indent || column(pos - match.size - 1)
|
||||||
|
tokens << [matched, :string] if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
|
||||||
|
next
|
||||||
|
end
|
||||||
|
|
||||||
|
elsif case
|
||||||
|
when match = scan(/[-:](?= |$)/)
|
||||||
|
state = :value if state == :colon && (match == ':' || match == '-')
|
||||||
|
state = :value if state == :initial && match == '-'
|
||||||
|
kind = :operator
|
||||||
|
when match = scan(/[,{}\[\]]/)
|
||||||
|
kind = :operator
|
||||||
|
when state == :initial && match = scan(/[\w.() ]*\S(?=: |:$)/)
|
||||||
|
kind = :key
|
||||||
|
key_indent = column(pos - match.size - 1)
|
||||||
|
# tokens << [key_indent.inspect, :debug]
|
||||||
|
state = :colon
|
||||||
|
when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?=: |:$)/)
|
||||||
|
tokens << [:open, :key]
|
||||||
|
tokens << [match[0,1], :delimiter]
|
||||||
|
tokens << [match[1..-2], :content]
|
||||||
|
tokens << [match[-1,1], :delimiter]
|
||||||
|
tokens << [:close, :key]
|
||||||
|
key_indent = column(pos - match.size - 1)
|
||||||
|
# tokens << [key_indent.inspect, :debug]
|
||||||
|
state = :colon
|
||||||
|
next
|
||||||
|
when scan(/(![\w\/]+)(:([\w:]+))?/)
|
||||||
|
tokens << [self[1], :type]
|
||||||
|
if self[2]
|
||||||
|
tokens << [':', :operator]
|
||||||
|
tokens << [self[3], :class]
|
||||||
|
end
|
||||||
|
next
|
||||||
|
when scan(/&\S+/)
|
||||||
|
kind = :variable
|
||||||
|
when scan(/\*\w+/)
|
||||||
|
kind = :global_variable
|
||||||
|
when scan(/<</)
|
||||||
|
kind = :class_variable
|
||||||
|
when scan(/\d\d:\d\d:\d\d/)
|
||||||
|
kind = :oct
|
||||||
|
when scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
|
||||||
|
kind = :oct
|
||||||
|
when scan(/:\w+/)
|
||||||
|
kind = :symbol
|
||||||
|
when scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
|
||||||
|
kind = :error
|
||||||
|
when scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
|
||||||
|
kind = :error
|
||||||
|
end
|
||||||
|
|
||||||
|
else
|
||||||
|
getch
|
||||||
|
kind = :error
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
match ||= matched
|
||||||
|
|
||||||
|
if $CODERAY_DEBUG and not kind
|
||||||
|
raise_inspect 'Error token %p in line %d' %
|
||||||
|
[[match, kind], line], tokens, state
|
||||||
|
end
|
||||||
|
raise_inspect 'Empty token', tokens, state unless match
|
||||||
|
|
||||||
|
tokens << [match, kind]
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
||||||
|
end
|
|
@ -8,7 +8,7 @@ module Styles
|
||||||
code_background = '#f8f8f8'
|
code_background = '#f8f8f8'
|
||||||
numbers_background = '#def'
|
numbers_background = '#def'
|
||||||
border_color = 'silver'
|
border_color = 'silver'
|
||||||
normal_color = '#100'
|
normal_color = '#000'
|
||||||
|
|
||||||
CSS_MAIN_STYLES = <<-MAIN
|
CSS_MAIN_STYLES = <<-MAIN
|
||||||
.CodeRay {
|
.CodeRay {
|
||||||
|
@ -32,6 +32,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top }
|
||||||
text-align: right;
|
text-align: right;
|
||||||
}
|
}
|
||||||
.CodeRay .line_numbers tt { font-weight: bold }
|
.CodeRay .line_numbers tt { font-weight: bold }
|
||||||
|
.CodeRay .line_numbers .highlighted { color: red }
|
||||||
.CodeRay .no { padding: 0px 4px }
|
.CodeRay .no { padding: 0px 4px }
|
||||||
.CodeRay .code { width: 100% }
|
.CodeRay .code { width: 100% }
|
||||||
|
|
||||||
|
@ -46,28 +47,32 @@ ol.CodeRay li { white-space: pre }
|
||||||
|
|
||||||
.af { color:#00C }
|
.af { color:#00C }
|
||||||
.an { color:#007 }
|
.an { color:#007 }
|
||||||
|
.at { color:#f08 }
|
||||||
.av { color:#700 }
|
.av { color:#700 }
|
||||||
.aw { color:#C00 }
|
.aw { color:#C00 }
|
||||||
.bi { color:#509; font-weight:bold }
|
.bi { color:#509; font-weight:bold }
|
||||||
.c { color:#666; }
|
.c { color:#888; }
|
||||||
|
|
||||||
.ch { color:#04D }
|
.ch { color:#04D }
|
||||||
.ch .k { color:#04D }
|
.ch .k { color:#04D }
|
||||||
.ch .dl { color:#039 }
|
.ch .dl { color:#039 }
|
||||||
|
|
||||||
.cl { color:#B06; font-weight:bold }
|
.cl { color:#B06; font-weight:bold }
|
||||||
|
.cm { color:#A08; font-weight:bold }
|
||||||
.co { color:#036; font-weight:bold }
|
.co { color:#036; font-weight:bold }
|
||||||
.cr { color:#0A0 }
|
.cr { color:#0A0 }
|
||||||
.cv { color:#369 }
|
.cv { color:#369 }
|
||||||
|
.de { color:#B0B; }
|
||||||
.df { color:#099; font-weight:bold }
|
.df { color:#099; font-weight:bold }
|
||||||
.di { color:#088; font-weight:bold }
|
.di { color:#088; font-weight:bold }
|
||||||
.dl { color:black }
|
.dl { color:black }
|
||||||
.do { color:#970 }
|
.do { color:#970 }
|
||||||
|
.dt { color:#34b }
|
||||||
.ds { color:#D42; font-weight:bold }
|
.ds { color:#D42; font-weight:bold }
|
||||||
.e { color:#666; font-weight:bold }
|
.e { color:#666; font-weight:bold }
|
||||||
.en { color:#800; font-weight:bold }
|
.en { color:#800; font-weight:bold }
|
||||||
.er { color:#F00; background-color:#FAA }
|
.er { color:#F00; background-color:#FAA }
|
||||||
.ex { color:#F00; font-weight:bold }
|
.ex { color:#C00; font-weight:bold }
|
||||||
.fl { color:#60E; font-weight:bold }
|
.fl { color:#60E; font-weight:bold }
|
||||||
.fu { color:#06B; font-weight:bold }
|
.fu { color:#06B; font-weight:bold }
|
||||||
.gv { color:#d70; font-weight:bold }
|
.gv { color:#d70; font-weight:bold }
|
||||||
|
@ -75,11 +80,13 @@ ol.CodeRay li { white-space: pre }
|
||||||
.i { color:#00D; font-weight:bold }
|
.i { color:#00D; font-weight:bold }
|
||||||
.ic { color:#B44; font-weight:bold }
|
.ic { color:#B44; font-weight:bold }
|
||||||
|
|
||||||
.il { background: #eee }
|
.il { background: #ddd; color: black }
|
||||||
.il .il { background: #ddd }
|
.il .il { background: #ccc }
|
||||||
.il .il .il { background: #ccc }
|
.il .il .il { background: #bbb }
|
||||||
.il .idl { font-weight: bold; color: #888 }
|
.il .idl { background: #ddd; font-weight: bold; color: #666 }
|
||||||
|
.idl { background-color: #bbb; font-weight: bold; color: #666; }
|
||||||
|
|
||||||
|
.im { color:#f00; }
|
||||||
.in { color:#B2B; font-weight:bold }
|
.in { color:#B2B; font-weight:bold }
|
||||||
.iv { color:#33B }
|
.iv { color:#33B }
|
||||||
.la { color:#970; font-weight:bold }
|
.la { color:#970; font-weight:bold }
|
||||||
|
@ -89,9 +96,15 @@ ol.CodeRay li { white-space: pre }
|
||||||
.op { }
|
.op { }
|
||||||
.pc { color:#038; font-weight:bold }
|
.pc { color:#038; font-weight:bold }
|
||||||
.pd { color:#369; font-weight:bold }
|
.pd { color:#369; font-weight:bold }
|
||||||
.pp { color:#579 }
|
.pp { color:#579; }
|
||||||
.pt { color:#339; font-weight:bold }
|
.ps { color:#00C; font-weight:bold }
|
||||||
.r { color:#080; font-weight:bold }
|
.pt { color:#074; font-weight:bold }
|
||||||
|
.r, .kw { color:#080; font-weight:bold }
|
||||||
|
|
||||||
|
.ke { color: #808; }
|
||||||
|
.ke .dl { color: #606; }
|
||||||
|
.ke .ch { color: #80f; }
|
||||||
|
.vl { color: #088; }
|
||||||
|
|
||||||
.rx { background-color:#fff0ff }
|
.rx { background-color:#fff0ff }
|
||||||
.rx .k { color:#808 }
|
.rx .k { color:#808 }
|
||||||
|
@ -99,14 +112,15 @@ ol.CodeRay li { white-space: pre }
|
||||||
.rx .mod { color:#C2C }
|
.rx .mod { color:#C2C }
|
||||||
.rx .fu { color:#404; font-weight: bold }
|
.rx .fu { color:#404; font-weight: bold }
|
||||||
|
|
||||||
.s { background-color:#fff0f0 }
|
.s { background-color:#fff0f0; color: #D20; }
|
||||||
.s .s { background-color:#ffe0e0 }
|
.s .s { background-color:#ffe0e0 }
|
||||||
.s .s .s { background-color:#ffd0d0 }
|
.s .s .s { background-color:#ffd0d0 }
|
||||||
.s .k { color:#D20 }
|
.s .k { }
|
||||||
.s .dl { color:#710 }
|
.s .ch { color: #b0b; }
|
||||||
|
.s .dl { color: #710; }
|
||||||
|
|
||||||
.sh { background-color:#f0fff0 }
|
.sh { background-color:#f0fff0; color:#2B2 }
|
||||||
.sh .k { color:#2B2 }
|
.sh .k { }
|
||||||
.sh .dl { color:#161 }
|
.sh .dl { color:#161 }
|
||||||
|
|
||||||
.sy { color:#A60 }
|
.sy { color:#A60 }
|
||||||
|
@ -119,6 +133,16 @@ ol.CodeRay li { white-space: pre }
|
||||||
.ty { color:#339; font-weight:bold }
|
.ty { color:#339; font-weight:bold }
|
||||||
.v { color:#036 }
|
.v { color:#036 }
|
||||||
.xt { color:#444 }
|
.xt { color:#444 }
|
||||||
|
|
||||||
|
.ins { background: #afa; }
|
||||||
|
.del { background: #faa; }
|
||||||
|
.chg { color: #aaf; background: #007; }
|
||||||
|
.head { color: #f8f; background: #505 }
|
||||||
|
|
||||||
|
.ins .ins { color: #080; font-weight:bold }
|
||||||
|
.del .del { color: #800; font-weight:bold }
|
||||||
|
.chg .chg { color: #66f; }
|
||||||
|
.head .head { color: #f4f; }
|
||||||
TOKENS
|
TOKENS
|
||||||
|
|
||||||
end
|
end
|
|
@ -84,6 +84,9 @@ ol.CodeRay li { white-space: pre; }
|
||||||
.pp { color:#579; }
|
.pp { color:#579; }
|
||||||
.pt { color:#66f; font-weight:bold; }
|
.pt { color:#66f; font-weight:bold; }
|
||||||
.r { color:#5de; font-weight:bold; }
|
.r { color:#5de; font-weight:bold; }
|
||||||
|
.r, .kw { color:#5de; font-weight:bold }
|
||||||
|
|
||||||
|
.ke { color: #808; }
|
||||||
|
|
||||||
.rx { background-color:#221133; }
|
.rx { background-color:#221133; }
|
||||||
.rx .k { color:#f8f; }
|
.rx .k { color:#f8f; }
|
||||||
|
@ -111,6 +114,16 @@ ol.CodeRay li { white-space: pre; }
|
||||||
.ty { color:#339; font-weight:bold; }
|
.ty { color:#339; font-weight:bold; }
|
||||||
.v { color:#036; }
|
.v { color:#036; }
|
||||||
.xt { color:#444; }
|
.xt { color:#444; }
|
||||||
|
|
||||||
|
.ins { background: #afa; }
|
||||||
|
.del { background: #faa; }
|
||||||
|
.chg { color: #aaf; background: #007; }
|
||||||
|
.head { color: #f8f; background: #505 }
|
||||||
|
|
||||||
|
.ins .ins { color: #080; font-weight:bold }
|
||||||
|
.del .del { color: #800; font-weight:bold }
|
||||||
|
.chg .chg { color: #66f; }
|
||||||
|
.head .head { color: #f4f; }
|
||||||
TOKENS
|
TOKENS
|
||||||
|
|
||||||
end
|
end
|
|
@ -4,6 +4,7 @@ module CodeRay
|
||||||
h[k] = k.to_s
|
h[k] = k.to_s
|
||||||
end
|
end
|
||||||
ClassOfKind.update with = {
|
ClassOfKind.update with = {
|
||||||
|
:annotation => 'at',
|
||||||
:attribute_name => 'an',
|
:attribute_name => 'an',
|
||||||
:attribute_name_fat => 'af',
|
:attribute_name_fat => 'af',
|
||||||
:attribute_value => 'av',
|
:attribute_value => 'av',
|
||||||
|
@ -14,12 +15,15 @@ module CodeRay
|
||||||
:class_variable => 'cv',
|
:class_variable => 'cv',
|
||||||
:color => 'cr',
|
:color => 'cr',
|
||||||
:comment => 'c',
|
:comment => 'c',
|
||||||
|
:complex => 'cm',
|
||||||
:constant => 'co',
|
:constant => 'co',
|
||||||
:content => 'k',
|
:content => 'k',
|
||||||
|
:decorator => 'de',
|
||||||
:definition => 'df',
|
:definition => 'df',
|
||||||
:delimiter => 'dl',
|
:delimiter => 'dl',
|
||||||
:directive => 'di',
|
:directive => 'di',
|
||||||
:doc => 'do',
|
:doc => 'do',
|
||||||
|
:doctype => 'dt',
|
||||||
:doc_string => 'ds',
|
:doc_string => 'ds',
|
||||||
:entity => 'en',
|
:entity => 'en',
|
||||||
:error => 'er',
|
:error => 'er',
|
||||||
|
@ -29,12 +33,16 @@ module CodeRay
|
||||||
:function => 'fu',
|
:function => 'fu',
|
||||||
:global_variable => 'gv',
|
:global_variable => 'gv',
|
||||||
:hex => 'hx',
|
:hex => 'hx',
|
||||||
|
:imaginary => 'cm',
|
||||||
|
:important => 'im',
|
||||||
:include => 'ic',
|
:include => 'ic',
|
||||||
:inline => 'il',
|
:inline => 'il',
|
||||||
:inline_delimiter => 'idl',
|
:inline_delimiter => 'idl',
|
||||||
:instance_variable => 'iv',
|
:instance_variable => 'iv',
|
||||||
:integer => 'i',
|
:integer => 'i',
|
||||||
:interpreted => 'in',
|
:interpreted => 'in',
|
||||||
|
:keyword => 'kw',
|
||||||
|
:key => 'ke',
|
||||||
:label => 'la',
|
:label => 'la',
|
||||||
:local_variable => 'lv',
|
:local_variable => 'lv',
|
||||||
:modifier => 'mod',
|
:modifier => 'mod',
|
||||||
|
@ -44,6 +52,7 @@ module CodeRay
|
||||||
:pre_type => 'pt',
|
:pre_type => 'pt',
|
||||||
:predefined => 'pd',
|
:predefined => 'pd',
|
||||||
:preprocessor => 'pp',
|
:preprocessor => 'pp',
|
||||||
|
:pseudo_class => 'ps',
|
||||||
:regexp => 'rx',
|
:regexp => 'rx',
|
||||||
:reserved => 'r',
|
:reserved => 'r',
|
||||||
:shell => 'sh',
|
:shell => 'sh',
|
||||||
|
@ -54,7 +63,13 @@ module CodeRay
|
||||||
:tag_special => 'ts',
|
:tag_special => 'ts',
|
||||||
:type => 'ty',
|
:type => 'ty',
|
||||||
:variable => 'v',
|
:variable => 'v',
|
||||||
|
:value => 'vl',
|
||||||
:xml_text => 'xt',
|
:xml_text => 'xt',
|
||||||
|
|
||||||
|
:insert => 'ins',
|
||||||
|
:delete => 'del',
|
||||||
|
:change => 'chg',
|
||||||
|
:head => 'head',
|
||||||
|
|
||||||
:ident => :NO_HIGHLIGHT, # 'id'
|
:ident => :NO_HIGHLIGHT, # 'id'
|
||||||
#:operator => 'op',
|
#:operator => 'op',
|
||||||
|
@ -62,7 +77,7 @@ module CodeRay
|
||||||
:space => :NO_HIGHLIGHT, # 'sp'
|
:space => :NO_HIGHLIGHT, # 'sp'
|
||||||
:plain => :NO_HIGHLIGHT,
|
:plain => :NO_HIGHLIGHT,
|
||||||
}
|
}
|
||||||
ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
|
ClassOfKind[:method] = ClassOfKind[:function]
|
||||||
ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
|
ClassOfKind[:open] = ClassOfKind[:close] = ClassOfKind[:delimiter]
|
||||||
ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
|
ClassOfKind[:nesting_delimiter] = ClassOfKind[:delimiter]
|
||||||
ClassOfKind[:escape] = ClassOfKind[:delimiter]
|
ClassOfKind[:escape] = ClassOfKind[:delimiter]
|
|
@ -14,7 +14,7 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# [:comment, '# It looks like this']
|
# [:comment, '# It looks like this']
|
||||||
# [:float, '3.1415926']
|
# [:float, '3.1415926']
|
||||||
# [:error, 'äöü']
|
# [:error, '$^']
|
||||||
#
|
#
|
||||||
# Some scanners also yield some kind of sub-tokens, represented by special
|
# Some scanners also yield some kind of sub-tokens, represented by special
|
||||||
# token texts, namely :open and :close .
|
# token texts, namely :open and :close .
|
||||||
|
@ -46,47 +46,10 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# Tokens' subclass TokenStream allows streaming to save memory.
|
# Tokens' subclass TokenStream allows streaming to save memory.
|
||||||
class Tokens < Array
|
class Tokens < Array
|
||||||
|
|
||||||
class << self
|
# The Scanner instance that created the tokens.
|
||||||
|
attr_accessor :scanner
|
||||||
# Convert the token to a string.
|
|
||||||
#
|
|
||||||
# This format is used by Encoders.Tokens.
|
|
||||||
# It can be reverted using read_token.
|
|
||||||
def write_token text, type
|
|
||||||
if text.is_a? String
|
|
||||||
"#{type}\t#{escape(text)}\n"
|
|
||||||
else
|
|
||||||
":#{text}\t#{type}\t\n"
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# Read a token from the string.
|
|
||||||
#
|
|
||||||
# Inversion of write_token.
|
|
||||||
#
|
|
||||||
# TODO Test this!
|
|
||||||
def read_token token
|
|
||||||
type, text = token.split("\t", 2)
|
|
||||||
if type[0] == ?:
|
|
||||||
[text.to_sym, type[1..-1].to_sym]
|
|
||||||
else
|
|
||||||
[type.to_sym, unescape(text)]
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
# Escapes a string for use in write_token.
|
|
||||||
def escape text
|
|
||||||
text.gsub(/[\n\\]/, '\\\\\&')
|
|
||||||
end
|
|
||||||
|
|
||||||
# Unescapes a string created by escape.
|
|
||||||
def unescape text
|
|
||||||
text.gsub(/\\[\n\\]/) { |m| m[1,1] }
|
|
||||||
end
|
|
||||||
|
|
||||||
end
|
|
||||||
|
|
||||||
# Whether the object is a TokenStream.
|
# Whether the object is a TokenStream.
|
||||||
#
|
#
|
||||||
# Returns false.
|
# Returns false.
|
||||||
|
@ -146,7 +109,6 @@ module CodeRay
|
||||||
encode :text, options
|
encode :text, options
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
# Redirects unknown methods to encoder calls.
|
# Redirects unknown methods to encoder calls.
|
||||||
#
|
#
|
||||||
# For example, if you call +tokens.html+, the HTML encoder
|
# For example, if you call +tokens.html+, the HTML encoder
|
||||||
|
@ -167,7 +129,6 @@ module CodeRay
|
||||||
# for example, consecutive //-comment lines could already be
|
# for example, consecutive //-comment lines could already be
|
||||||
# joined in one comment token by the Scanner.
|
# joined in one comment token by the Scanner.
|
||||||
def optimize
|
def optimize
|
||||||
print ' Tokens#optimize: before: %d - ' % size if $DEBUG
|
|
||||||
last_kind = last_text = nil
|
last_kind = last_text = nil
|
||||||
new = self.class.new
|
new = self.class.new
|
||||||
for text, kind in self
|
for text, kind in self
|
||||||
|
@ -186,8 +147,6 @@ module CodeRay
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
new << [last_text, last_kind] if last_kind
|
new << [last_text, last_kind] if last_kind
|
||||||
print 'after: %d (%d saved = %2.0f%%)' %
|
|
||||||
[new.size, size - new.size, 1.0 - (new.size.to_f / size)] if $DEBUG
|
|
||||||
new
|
new
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -200,25 +159,29 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# TODO: Test this!
|
# TODO: Test this!
|
||||||
def fix
|
def fix
|
||||||
|
tokens = self.class.new
|
||||||
# Check token nesting using a stack of kinds.
|
# Check token nesting using a stack of kinds.
|
||||||
opened = []
|
opened = []
|
||||||
for token, kind in self
|
for type, kind in self
|
||||||
if token == :open
|
case type
|
||||||
opened.push kind
|
when :open
|
||||||
elsif token == :close
|
opened.push [:close, kind]
|
||||||
|
when :begin_line
|
||||||
|
opened.push [:end_line, kind]
|
||||||
|
when :close, :end_line
|
||||||
expected = opened.pop
|
expected = opened.pop
|
||||||
if kind != expected
|
if [type, kind] != expected
|
||||||
# Unexpected :close; decide what to do based on the kind:
|
# Unexpected :close; decide what to do based on the kind:
|
||||||
# - token was opened earlier: also close tokens in between
|
# - token was never opened: delete the :close (just skip it)
|
||||||
# - token was never opened: delete the :close (skip with next)
|
|
||||||
next unless opened.rindex expected
|
next unless opened.rindex expected
|
||||||
tokens << [:close, kind] until (kind = opened.pop) == expected
|
# - token was opened earlier: also close tokens in between
|
||||||
|
tokens << token until (token = opened.pop) == expected
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
tokens << [token, kind]
|
tokens << [type, kind]
|
||||||
end
|
end
|
||||||
# Close remaining opened tokens
|
# Close remaining opened tokens
|
||||||
tokens << [:close, kind] while kind = opened.pop
|
tokens << token while token = opened.pop
|
||||||
tokens
|
tokens
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -226,6 +189,8 @@ module CodeRay
|
||||||
replace fix
|
replace fix
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# TODO: Scanner#split_into_lines
|
||||||
|
#
|
||||||
# Makes sure that:
|
# Makes sure that:
|
||||||
# - newlines are single tokens
|
# - newlines are single tokens
|
||||||
# (which means all other token are single-line)
|
# (which means all other token are single-line)
|
||||||
|
@ -353,7 +318,7 @@ module CodeRay
|
||||||
#
|
#
|
||||||
# Returns self.
|
# Returns self.
|
||||||
def << token
|
def << token
|
||||||
@callback.call token
|
@callback.call(*token)
|
||||||
@size += 1
|
@size += 1
|
||||||
self
|
self
|
||||||
end
|
end
|
||||||
|
@ -376,8 +341,48 @@ module CodeRay
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
# Token name abbreviations
|
|
||||||
require 'coderay/token_classes'
|
|
||||||
|
|
||||||
end
|
end
|
||||||
|
|
||||||
|
if $0 == __FILE__
|
||||||
|
$VERBOSE = true
|
||||||
|
$: << File.join(File.dirname(__FILE__), '..')
|
||||||
|
eval DATA.read, nil, $0, __LINE__ + 4
|
||||||
|
end
|
||||||
|
|
||||||
|
__END__
|
||||||
|
require 'test/unit'
|
||||||
|
|
||||||
|
class TokensTest < Test::Unit::TestCase
|
||||||
|
|
||||||
|
def test_creation
|
||||||
|
assert CodeRay::Tokens < Array
|
||||||
|
tokens = nil
|
||||||
|
assert_nothing_raised do
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
end
|
||||||
|
assert_kind_of Array, tokens
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_adding_tokens
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
assert_nothing_raised do
|
||||||
|
tokens << ['string', :type]
|
||||||
|
tokens << ['()', :operator]
|
||||||
|
end
|
||||||
|
assert_equal tokens.size, 2
|
||||||
|
end
|
||||||
|
|
||||||
|
def test_dump_undump
|
||||||
|
tokens = CodeRay::Tokens.new
|
||||||
|
assert_nothing_raised do
|
||||||
|
tokens << ['string', :type]
|
||||||
|
tokens << ['()', :operator]
|
||||||
|
end
|
||||||
|
tokens2 = nil
|
||||||
|
assert_nothing_raised do
|
||||||
|
tokens2 = tokens.dump.undump
|
||||||
|
end
|
||||||
|
assert_equal tokens, tokens2
|
||||||
|
end
|
||||||
|
|
||||||
|
end
|
Loading…
Reference in New Issue