Coderay upgraded to 0.9.7 (#5344).
git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@4739 e93f8b46-1217-0410-a6f0-8f06a7374b81
This commit is contained in:
parent
e5c8513dda
commit
b80b789cc6
|
@ -51,6 +51,7 @@ Rails::Initializer.run do |config|
|
|||
config.action_mailer.perform_deliveries = false
|
||||
|
||||
config.gem 'rubytree', :lib => 'tree'
|
||||
config.gem 'coderay', :version => '~>0.9.7'
|
||||
|
||||
# Load any local configuration that is kept out of source control
|
||||
# (e.g. gems, patches).
|
||||
|
|
|
@ -0,0 +1,155 @@
|
|||
--- !ruby/object:Gem::Specification
|
||||
name: coderay
|
||||
version: !ruby/object:Gem::Version
|
||||
hash: 53
|
||||
prerelease: false
|
||||
segments:
|
||||
- 0
|
||||
- 9
|
||||
- 7
|
||||
version: 0.9.7
|
||||
platform: ruby
|
||||
authors:
|
||||
- murphy
|
||||
autorequire:
|
||||
bindir: bin
|
||||
cert_chain: []
|
||||
|
||||
date: 2011-01-15 00:00:00 +01:00
|
||||
default_executable:
|
||||
dependencies: []
|
||||
|
||||
description: |
|
||||
Fast and easy syntax highlighting for selected languages, written in Ruby.
|
||||
Comes with RedCloth integration and LOC counter.
|
||||
|
||||
email: murphy@rubychan.de
|
||||
executables:
|
||||
- coderay
|
||||
- coderay_stylesheet
|
||||
extensions: []
|
||||
|
||||
extra_rdoc_files:
|
||||
- lib/README
|
||||
- FOLDERS
|
||||
files:
|
||||
- ./lib/coderay/duo.rb
|
||||
- ./lib/coderay/encoder.rb
|
||||
- ./lib/coderay/encoders/_map.rb
|
||||
- ./lib/coderay/encoders/comment_filter.rb
|
||||
- ./lib/coderay/encoders/count.rb
|
||||
- ./lib/coderay/encoders/debug.rb
|
||||
- ./lib/coderay/encoders/div.rb
|
||||
- ./lib/coderay/encoders/filter.rb
|
||||
- ./lib/coderay/encoders/html/css.rb
|
||||
- ./lib/coderay/encoders/html/numerization.rb
|
||||
- ./lib/coderay/encoders/html/output.rb
|
||||
- ./lib/coderay/encoders/html.rb
|
||||
- ./lib/coderay/encoders/json.rb
|
||||
- ./lib/coderay/encoders/lines_of_code.rb
|
||||
- ./lib/coderay/encoders/null.rb
|
||||
- ./lib/coderay/encoders/page.rb
|
||||
- ./lib/coderay/encoders/span.rb
|
||||
- ./lib/coderay/encoders/statistic.rb
|
||||
- ./lib/coderay/encoders/term.rb
|
||||
- ./lib/coderay/encoders/text.rb
|
||||
- ./lib/coderay/encoders/token_class_filter.rb
|
||||
- ./lib/coderay/encoders/xml.rb
|
||||
- ./lib/coderay/encoders/yaml.rb
|
||||
- ./lib/coderay/for_redcloth.rb
|
||||
- ./lib/coderay/helpers/file_type.rb
|
||||
- ./lib/coderay/helpers/gzip_simple.rb
|
||||
- ./lib/coderay/helpers/plugin.rb
|
||||
- ./lib/coderay/helpers/word_list.rb
|
||||
- ./lib/coderay/scanner.rb
|
||||
- ./lib/coderay/scanners/_map.rb
|
||||
- ./lib/coderay/scanners/c.rb
|
||||
- ./lib/coderay/scanners/cpp.rb
|
||||
- ./lib/coderay/scanners/css.rb
|
||||
- ./lib/coderay/scanners/debug.rb
|
||||
- ./lib/coderay/scanners/delphi.rb
|
||||
- ./lib/coderay/scanners/diff.rb
|
||||
- ./lib/coderay/scanners/groovy.rb
|
||||
- ./lib/coderay/scanners/html.rb
|
||||
- ./lib/coderay/scanners/java/builtin_types.rb
|
||||
- ./lib/coderay/scanners/java.rb
|
||||
- ./lib/coderay/scanners/java_script-0.9.6.rb
|
||||
- ./lib/coderay/scanners/java_script.rb
|
||||
- ./lib/coderay/scanners/json.rb
|
||||
- ./lib/coderay/scanners/nitro_xhtml.rb
|
||||
- ./lib/coderay/scanners/php.rb
|
||||
- ./lib/coderay/scanners/plaintext.rb
|
||||
- ./lib/coderay/scanners/python.rb
|
||||
- ./lib/coderay/scanners/rhtml.rb
|
||||
- ./lib/coderay/scanners/ruby/patterns.rb
|
||||
- ./lib/coderay/scanners/ruby.rb
|
||||
- ./lib/coderay/scanners/scheme.rb
|
||||
- ./lib/coderay/scanners/sql.rb
|
||||
- ./lib/coderay/scanners/xml.rb
|
||||
- ./lib/coderay/scanners/yaml.rb
|
||||
- ./lib/coderay/style.rb
|
||||
- ./lib/coderay/styles/_map.rb
|
||||
- ./lib/coderay/styles/cycnus.rb
|
||||
- ./lib/coderay/styles/murphy.rb
|
||||
- ./lib/coderay/token_classes.rb
|
||||
- ./lib/coderay/tokens.rb
|
||||
- ./lib/coderay.rb
|
||||
- ./Rakefile
|
||||
- ./test/functional/basic.rb
|
||||
- ./test/functional/basic.rbc
|
||||
- ./test/functional/for_redcloth.rb
|
||||
- ./test/functional/for_redcloth.rbc
|
||||
- ./test/functional/load_plugin_scanner.rb
|
||||
- ./test/functional/load_plugin_scanner.rbc
|
||||
- ./test/functional/suite.rb
|
||||
- ./test/functional/suite.rbc
|
||||
- ./test/functional/vhdl.rb
|
||||
- ./test/functional/vhdl.rbc
|
||||
- ./test/functional/word_list.rb
|
||||
- ./test/functional/word_list.rbc
|
||||
- ./lib/README
|
||||
- ./LICENSE
|
||||
- lib/README
|
||||
- FOLDERS
|
||||
- bin/coderay
|
||||
- bin/coderay_stylesheet
|
||||
has_rdoc: true
|
||||
homepage: http://coderay.rubychan.de
|
||||
licenses: []
|
||||
|
||||
post_install_message:
|
||||
rdoc_options:
|
||||
- -SNw2
|
||||
- -mlib/README
|
||||
- -t CodeRay Documentation
|
||||
require_paths:
|
||||
- lib
|
||||
required_ruby_version: !ruby/object:Gem::Requirement
|
||||
none: false
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
hash: 51
|
||||
segments:
|
||||
- 1
|
||||
- 8
|
||||
- 2
|
||||
version: 1.8.2
|
||||
required_rubygems_version: !ruby/object:Gem::Requirement
|
||||
none: false
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
hash: 3
|
||||
segments:
|
||||
- 0
|
||||
version: "0"
|
||||
requirements: []
|
||||
|
||||
rubyforge_project: coderay
|
||||
rubygems_version: 1.3.7
|
||||
signing_key:
|
||||
specification_version: 3
|
||||
summary: Fast syntax highlighting for selected languages.
|
||||
test_files:
|
||||
- ./test/functional/suite.rb
|
|
@ -0,0 +1,53 @@
|
|||
= CodeRay - Trunk folder structure
|
||||
|
||||
== bench - Benchmarking system
|
||||
|
||||
All benchmarking stuff goes here.
|
||||
|
||||
Test inputs are stored in files named <code>example.<lang></code>.
|
||||
Test outputs go to <code>bench/test.<encoder-default-file-extension></code>.
|
||||
|
||||
Run <code>bench/bench.rb</code> to get a usage description.
|
||||
|
||||
Run <code>rake bench</code> to perform an example benchmark.
|
||||
|
||||
|
||||
== bin - Scripts
|
||||
|
||||
Executional files for CodeRay.
|
||||
|
||||
|
||||
== demo - Demos and functional tests
|
||||
|
||||
Demonstrational scripts to show of CodeRay's features.
|
||||
|
||||
Run them as functional tests with <code>rake test:demos</code>.
|
||||
|
||||
|
||||
== etc - Lots of stuff
|
||||
|
||||
Some addidtional files for CodeRay, mainly graphics and Vim scripts.
|
||||
|
||||
|
||||
== gem_server - Gem output folder
|
||||
|
||||
For <code>rake gem</code>.
|
||||
|
||||
|
||||
== lib - CodeRay library code
|
||||
|
||||
This is the base directory for the CodeRay library.
|
||||
|
||||
|
||||
== rake_helpers - Rake helper libraries
|
||||
|
||||
Some files to enhance Rake, including the Autumnal Rdoc template and some scripts.
|
||||
|
||||
|
||||
== test - Tests
|
||||
|
||||
Tests for the scanners.
|
||||
|
||||
Each language has its own subfolder and sub-suite.
|
||||
|
||||
Run with <code>rake test</code>.
|
|
@ -0,0 +1,35 @@
|
|||
require 'rake/rdoctask'
|
||||
|
||||
ROOT = '.'
|
||||
LIB_ROOT = File.join ROOT, 'lib'
|
||||
EXTRA_RDOC_FILES = %w(lib/README FOLDERS)
|
||||
|
||||
task :default => :test
|
||||
|
||||
if File.directory? 'rake_tasks'
|
||||
|
||||
# load rake tasks from subfolder
|
||||
for task_file in Dir['rake_tasks/*.rake'].sort
|
||||
load task_file
|
||||
end
|
||||
|
||||
else
|
||||
|
||||
# fallback tasks when rake_tasks folder is not present
|
||||
desc 'Run CodeRay tests (basic)'
|
||||
task :test do
|
||||
ruby './test/functional/suite.rb'
|
||||
ruby './test/functional/for_redcloth.rb'
|
||||
end
|
||||
|
||||
desc 'Generate documentation for CodeRay'
|
||||
Rake::RDocTask.new :doc do |rd|
|
||||
rd.title = 'CodeRay Documentation'
|
||||
rd.main = 'lib/README'
|
||||
rd.rdoc_files.add Dir['lib']
|
||||
rd.rdoc_files.add 'lib/README'
|
||||
rd.rdoc_files.add 'FOLDERS'
|
||||
rd.rdoc_dir = 'doc'
|
||||
end
|
||||
|
||||
end
|
|
@ -0,0 +1,86 @@
|
|||
#!/usr/bin/env ruby
|
||||
# CodeRay Executable
|
||||
#
|
||||
# Version: 0.2
|
||||
# Author: murphy
|
||||
|
||||
require 'coderay'
|
||||
|
||||
if ARGV.empty?
|
||||
$stderr.puts <<-USAGE
|
||||
CodeRay #{CodeRay::VERSION} (http://coderay.rubychan.de)
|
||||
|
||||
Usage:
|
||||
coderay file [-<format>]
|
||||
coderay -<lang> [-<format>] [< file] [> output]
|
||||
|
||||
Defaults:
|
||||
lang: based on file extension
|
||||
format: ANSI colorized output for terminal, HTML page for files
|
||||
|
||||
Examples:
|
||||
coderay foo.rb # colorized output to terminal, based on file extension
|
||||
coderay foo.rb -loc # print LOC count, based on file extension and format
|
||||
coderay foo.rb > foo.html # HTML page output to file, based on extension
|
||||
coderay -ruby < foo.rb # colorized output to terminal, based on lang
|
||||
coderay -ruby -loc < foo.rb # print LOC count, based on lang
|
||||
coderay -ruby -page foo.rb # HTML page output to terminal, based on lang and format
|
||||
coderay -ruby -page foo.rb > foo.html # HTML page output to file, based on lang and format
|
||||
USAGE
|
||||
end
|
||||
|
||||
first, second = ARGV
|
||||
|
||||
def read
|
||||
file = ARGV.grep(/^(?!-)/).last
|
||||
if file
|
||||
if File.exist?(file)
|
||||
File.read file
|
||||
else
|
||||
$stderr.puts "No such file: #{file}"
|
||||
end
|
||||
else
|
||||
$stdin.read
|
||||
end
|
||||
end
|
||||
|
||||
if first
|
||||
if first[/-(\w+)/] == first
|
||||
lang = $1
|
||||
input = read
|
||||
tokens = :scan
|
||||
else
|
||||
file = first
|
||||
unless File.exist? file
|
||||
$stderr.puts "No such file: #{file}"
|
||||
exit 2
|
||||
end
|
||||
tokens = CodeRay.scan_file file
|
||||
end
|
||||
else
|
||||
$stderr.puts 'No lang/file given.'
|
||||
exit 1
|
||||
end
|
||||
|
||||
if second
|
||||
if second[/-(\w+)/] == second
|
||||
format = $1.to_sym
|
||||
else
|
||||
raise 'invalid format (must be -xxx)'
|
||||
end
|
||||
else
|
||||
if $stdout.tty?
|
||||
format = :term
|
||||
else
|
||||
$stderr.puts 'No format given; setting to default (HTML Page).'
|
||||
format = :page
|
||||
end
|
||||
end
|
||||
|
||||
if tokens == :scan
|
||||
output = CodeRay::Duo[lang => format].highlight input
|
||||
else
|
||||
output = tokens.encode format
|
||||
end
|
||||
out = $stdout
|
||||
out.puts output
|
|
@ -0,0 +1,4 @@
|
|||
#!/usr/bin/env ruby
|
||||
require 'coderay'
|
||||
|
||||
puts CodeRay::Encoders[:html]::CSS.new.stylesheet
|
|
@ -18,7 +18,7 @@ And with line numbers.
|
|||
* is what everybody should have on their website
|
||||
* solves all your problems and makes the girls run after you
|
||||
|
||||
Version: 0.9.2
|
||||
Version: 0.9.7
|
||||
Author:: murphy (Kornelius Kalnbach)
|
||||
Contact:: murphy rubychan de
|
||||
Website:: coderay.rubychan.de[http://coderay.rubychan.de]
|
||||
|
@ -94,6 +94,7 @@ Please report errors in this documentation to <murphy rubychan de>.
|
|||
* Rob Aldred for the terminal encoder
|
||||
* Trans for pointing out $DEBUG dependencies
|
||||
* Flameeyes for finding that Term::ANSIColor was obsolete
|
||||
* Etienne Massip for reporting a serious bug in JavaScript scanner
|
||||
* matz and all Ruby gods and gurus
|
||||
* The inventors of: the computer, the internet, the true color display, HTML &
|
||||
CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime,
|
|
@ -134,7 +134,7 @@ module CodeRay
|
|||
# Minor: feature milestone
|
||||
# Teeny: development state, 0 for pre-release
|
||||
# Revision: Subversion Revision number (generated on rake gem:make)
|
||||
VERSION = '0.9.2'
|
||||
VERSION = '0.9.7'
|
||||
|
||||
require 'coderay/tokens'
|
||||
require 'coderay/token_classes'
|
|
@ -276,9 +276,13 @@ module Encoders
|
|||
when :begin_line
|
||||
@opened[0] = type
|
||||
if style = @css_style[@opened]
|
||||
@out << style.sub('<span', '<div')
|
||||
if style['class="']
|
||||
@out << style.sub('class="', 'class="line ')
|
||||
else
|
||||
@out << style.sub('>', ' class="line">')
|
||||
end
|
||||
else
|
||||
@out << '<div>'
|
||||
@out << '<span class="line">'
|
||||
end
|
||||
@opened << type
|
||||
when :end_line
|
||||
|
@ -289,7 +293,7 @@ module Encoders
|
|||
raise 'Malformed token stream: Trying to close a line (%p) \
|
||||
that is not open. Open are: %p.' % [type, @opened[1..-1]]
|
||||
end
|
||||
@out << '</div>'
|
||||
@out << '</span>'
|
||||
@opened.pop
|
||||
end
|
||||
|
|
@ -80,8 +80,8 @@ module Encoders
|
|||
line_numbers.gsub!(/\n/) { "<tt>\n</tt>" }
|
||||
|
||||
line_numbers_table_tpl = TABLE.apply('LINE_NUMBERS', line_numbers)
|
||||
gsub!(/<\/div>\n/) { '</div>' }
|
||||
gsub!(/\n/) { "<tt>\n</tt>" }
|
||||
gsub!("</div>\n", '</div>')
|
||||
gsub!("\n", "<tt>\n</tt>")
|
||||
wrap_in! line_numbers_table_tpl
|
||||
@wrapped_in = :div
|
||||
|
|
@ -68,15 +68,16 @@ module CodeRay
|
|||
|
||||
def normify code
|
||||
code = code.to_s
|
||||
if code.respond_to? :force_encoding
|
||||
debug, $DEBUG = $DEBUG, false
|
||||
begin
|
||||
code.force_encoding 'utf-8'
|
||||
code[/\z/] # raises an ArgumentError when code contains a non-UTF-8 char
|
||||
rescue ArgumentError
|
||||
code.force_encoding 'binary'
|
||||
ensure
|
||||
$DEBUG = debug
|
||||
if code.respond_to?(:encoding) && (code.encoding.name != 'UTF-8' || !code.valid_encoding?)
|
||||
code = code.dup
|
||||
original_encoding = code.encoding
|
||||
code.force_encoding 'Windows-1252'
|
||||
unless code.valid_encoding?
|
||||
code.force_encoding original_encoding
|
||||
if code.encoding.name == 'UTF-8'
|
||||
code.encode! 'UTF-16BE', :invalid => :replace, :undef => :replace, :replace => '?'
|
||||
end
|
||||
code.encode! 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '?'
|
||||
end
|
||||
end
|
||||
code.to_unix
|
||||
|
@ -147,7 +148,12 @@ module CodeRay
|
|||
|
||||
def string= code
|
||||
code = Scanner.normify(code)
|
||||
super code
|
||||
if defined?(RUBY_DESCRIPTION) && RUBY_DESCRIPTION['rubinius 1.0.1']
|
||||
reset_state
|
||||
@string = code
|
||||
else
|
||||
super code
|
||||
end
|
||||
reset_instance
|
||||
end
|
||||
|
|
@ -14,12 +14,11 @@ module Scanners
|
|||
]
|
||||
|
||||
module RE
|
||||
NonASCII = /[\x80-\xFF]/
|
||||
Hex = /[0-9a-fA-F]/
|
||||
Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
|
||||
Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
|
||||
NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
|
||||
NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
|
||||
NMChar = /[-_a-zA-Z0-9]|#{Escape}/
|
||||
NMStart = /[_a-zA-Z]|#{Escape}/
|
||||
NL = /\r\n|\r|\n|\f/
|
||||
String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # FIXME: buggy regexp
|
||||
String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # FIXME: buggy regexp
|
|
@ -47,11 +47,16 @@ module Scanners
|
|||
tokens << [match, :change]
|
||||
next unless match = scan(/.+/)
|
||||
kind = :plain
|
||||
elsif scan(/(@@)((?>[^@\n]*))(@@)/)
|
||||
tokens << [:begin_line, line_kind = :change]
|
||||
tokens << [self[1], :change]
|
||||
tokens << [self[2], :plain]
|
||||
tokens << [self[3], :change]
|
||||
elsif match = scan(/@@(?>[^@\n]*)@@/)
|
||||
if check(/\n|$/)
|
||||
tokens << [:begin_line, line_kind = :change]
|
||||
else
|
||||
tokens << [:open, :change]
|
||||
end
|
||||
tokens << [match[0,2], :change]
|
||||
tokens << [match[2...-2], :plain]
|
||||
tokens << [match[-2,2], :change]
|
||||
tokens << [:close, :change] unless line_kind
|
||||
next unless match = scan(/.+/)
|
||||
kind = :plain
|
||||
elsif match = scan(/\+/)
|
||||
|
@ -67,7 +72,7 @@ module Scanners
|
|||
elsif scan(/ .*/)
|
||||
kind = :comment
|
||||
elsif scan(/.+/)
|
||||
tokens << [:begin_line, line_kind = :head]
|
||||
tokens << [:begin_line, line_kind = :comment]
|
||||
kind = :plain
|
||||
else
|
||||
raise_inspect 'else case rached'
|
|
@ -91,7 +91,7 @@ module Scanners
|
|||
end
|
||||
|
||||
when :attribute
|
||||
if scan(/#{TAG_END}/)
|
||||
if scan(/#{TAG_END}/o)
|
||||
kind = :tag
|
||||
state = :initial
|
||||
elsif scan(/#{ATTR_NAME}/o)
|
|
@ -138,7 +138,7 @@ module Scanners
|
|||
elsif scan(/\\./m)
|
||||
kind = :content
|
||||
elsif scan(/ \\ | $ /x)
|
||||
tokens << [:close, :delimiter]
|
||||
tokens << [:close, state]
|
||||
kind = :error
|
||||
state = :initial
|
||||
else
|
|
@ -0,0 +1,224 @@
|
|||
module CodeRay
|
||||
module Scanners
|
||||
|
||||
class JavaScript < Scanner
|
||||
|
||||
include Streamable
|
||||
|
||||
register_for :java_script
|
||||
file_extension 'js'
|
||||
|
||||
# The actual JavaScript keywords.
|
||||
KEYWORDS = %w[
|
||||
break case catch continue default delete do else
|
||||
finally for function if in instanceof new
|
||||
return switch throw try typeof var void while with
|
||||
]
|
||||
PREDEFINED_CONSTANTS = %w[
|
||||
false null true undefined
|
||||
]
|
||||
|
||||
MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
|
||||
|
||||
KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
|
||||
case delete in instanceof new return throw typeof with
|
||||
]
|
||||
|
||||
# Reserved for future use.
|
||||
RESERVED_WORDS = %w[
|
||||
abstract boolean byte char class debugger double enum export extends
|
||||
final float goto implements import int interface long native package
|
||||
private protected public short static super synchronized throws transient
|
||||
volatile
|
||||
]
|
||||
|
||||
IDENT_KIND = WordList.new(:ident).
|
||||
add(RESERVED_WORDS, :reserved).
|
||||
add(PREDEFINED_CONSTANTS, :pre_constant).
|
||||
add(MAGIC_VARIABLES, :local_variable).
|
||||
add(KEYWORDS, :keyword)
|
||||
|
||||
ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||
REGEXP_ESCAPE = / [bBdDsSwW] /x
|
||||
STRING_CONTENT_PATTERN = {
|
||||
"'" => /[^\\']+/,
|
||||
'"' => /[^\\"]+/,
|
||||
'/' => /[^\\\/]+/,
|
||||
}
|
||||
KEY_CHECK_PATTERN = {
|
||||
"'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
|
||||
'"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
|
||||
}
|
||||
|
||||
def scan_tokens tokens, options
|
||||
|
||||
state = :initial
|
||||
string_delimiter = nil
|
||||
value_expected = true
|
||||
key_expected = false
|
||||
function_expected = false
|
||||
|
||||
until eos?
|
||||
|
||||
kind = nil
|
||||
match = nil
|
||||
|
||||
case state
|
||||
|
||||
when :initial
|
||||
|
||||
if match = scan(/ \s+ | \\\n /x)
|
||||
value_expected = true if !value_expected && match.index(?\n)
|
||||
tokens << [match, :space]
|
||||
next
|
||||
|
||||
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
||||
value_expected = true
|
||||
kind = :comment
|
||||
|
||||
elsif check(/\.?\d/)
|
||||
key_expected = value_expected = false
|
||||
if scan(/0[xX][0-9A-Fa-f]+/)
|
||||
kind = :hex
|
||||
elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
|
||||
kind = :oct
|
||||
elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
||||
kind = :float
|
||||
elsif scan(/\d+/)
|
||||
kind = :integer
|
||||
end
|
||||
|
||||
elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
|
||||
# FIXME: scan over nested tags
|
||||
xml_scanner.tokenize match
|
||||
value_expected = false
|
||||
next
|
||||
|
||||
elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
|
||||
value_expected = true
|
||||
last_operator = match[-1]
|
||||
key_expected = (last_operator == ?{) || (last_operator == ?,)
|
||||
function_expected = false
|
||||
kind = :operator
|
||||
|
||||
elsif scan(/ [)\]}]+ /x)
|
||||
function_expected = key_expected = value_expected = false
|
||||
kind = :operator
|
||||
|
||||
elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
|
||||
kind = IDENT_KIND[match]
|
||||
value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
|
||||
# TODO: labels
|
||||
if kind == :ident
|
||||
if match.index(?$) # $ allowed inside an identifier
|
||||
kind = :predefined
|
||||
elsif function_expected
|
||||
kind = :function
|
||||
elsif check(/\s*[=:]\s*function\b/)
|
||||
kind = :function
|
||||
elsif key_expected && check(/\s*:/)
|
||||
kind = :key
|
||||
end
|
||||
end
|
||||
function_expected = (kind == :keyword) && (match == 'function')
|
||||
key_expected = false
|
||||
|
||||
elsif match = scan(/["']/)
|
||||
if key_expected && check(KEY_CHECK_PATTERN[match])
|
||||
state = :key
|
||||
else
|
||||
state = :string
|
||||
end
|
||||
tokens << [:open, state]
|
||||
string_delimiter = match
|
||||
kind = :delimiter
|
||||
|
||||
elsif value_expected && (match = scan(/\/(?=\S)/))
|
||||
tokens << [:open, :regexp]
|
||||
state = :regexp
|
||||
string_delimiter = '/'
|
||||
kind = :delimiter
|
||||
|
||||
elsif scan(/ \/ /x)
|
||||
value_expected = true
|
||||
key_expected = false
|
||||
kind = :operator
|
||||
|
||||
else
|
||||
getch
|
||||
kind = :error
|
||||
|
||||
end
|
||||
|
||||
when :string, :regexp, :key
|
||||
if scan(STRING_CONTENT_PATTERN[string_delimiter])
|
||||
kind = :content
|
||||
elsif match = scan(/["'\/]/)
|
||||
tokens << [match, :delimiter]
|
||||
if state == :regexp
|
||||
modifiers = scan(/[gim]+/)
|
||||
tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
|
||||
end
|
||||
tokens << [:close, state]
|
||||
string_delimiter = nil
|
||||
key_expected = value_expected = false
|
||||
state = :initial
|
||||
next
|
||||
elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
|
||||
if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
|
||||
kind = :content
|
||||
else
|
||||
kind = :char
|
||||
end
|
||||
elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||
kind = :char
|
||||
elsif scan(/\\./m)
|
||||
kind = :content
|
||||
elsif scan(/ \\ | $ /x)
|
||||
tokens << [:close, state]
|
||||
kind = :error
|
||||
key_expected = value_expected = false
|
||||
state = :initial
|
||||
else
|
||||
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||
end
|
||||
|
||||
else
|
||||
raise_inspect 'Unknown state', tokens
|
||||
|
||||
end
|
||||
|
||||
match ||= matched
|
||||
if $CODERAY_DEBUG and not kind
|
||||
raise_inspect 'Error token %p in line %d' %
|
||||
[[match, kind], line], tokens
|
||||
end
|
||||
raise_inspect 'Empty token', tokens unless match
|
||||
|
||||
tokens << [match, kind]
|
||||
|
||||
end
|
||||
|
||||
if [:string, :regexp].include? state
|
||||
tokens << [:close, state]
|
||||
end
|
||||
|
||||
tokens
|
||||
end
|
||||
|
||||
protected
|
||||
|
||||
def reset_instance
|
||||
super
|
||||
@xml_scanner.reset if defined? @xml_scanner
|
||||
end
|
||||
|
||||
def xml_scanner
|
||||
@xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
end
|
||||
end
|
|
@ -47,8 +47,8 @@ module Scanners
|
|||
'/' => /[^\\\/]+/,
|
||||
}
|
||||
KEY_CHECK_PATTERN = {
|
||||
"'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
|
||||
'"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
|
||||
"'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx,
|
||||
'"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx,
|
||||
}
|
||||
|
||||
def scan_tokens tokens, options
|
||||
|
@ -215,7 +215,7 @@ module Scanners
|
|||
end
|
||||
|
||||
def xml_scanner
|
||||
@xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
||||
@xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
|
||||
end
|
||||
|
||||
end
|
|
@ -13,9 +13,6 @@ module Scanners
|
|||
:error, :integer, :operator, :value,
|
||||
]
|
||||
|
||||
CONSTANTS = %w( true false null )
|
||||
IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
|
||||
|
||||
ESCAPE = / [bfnrt\\"\/] /x
|
||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
|
||||
|
||||
|
@ -23,7 +20,6 @@ module Scanners
|
|||
|
||||
state = :initial
|
||||
stack = []
|
||||
string_delimiter = nil
|
||||
key_expected = false
|
||||
|
||||
until eos?
|
||||
|
@ -47,7 +43,7 @@ module Scanners
|
|||
when '}', ']' then stack.pop # no error recovery, but works for valid JSON
|
||||
end
|
||||
elsif match = scan(/ true | false | null /x)
|
||||
kind = IDENT_KIND[match]
|
||||
kind = :value
|
||||
elsif match = scan(/-?(?:0|[1-9]\d*)/)
|
||||
kind = :integer
|
||||
if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
|
||||
|
@ -76,7 +72,7 @@ module Scanners
|
|||
elsif scan(/\\./m)
|
||||
kind = :content
|
||||
elsif scan(/ \\ | $ /x)
|
||||
tokens << [:close, :delimiter]
|
||||
tokens << [:close, state]
|
||||
kind = :error
|
||||
state = :initial
|
||||
else
|
|
@ -225,6 +225,12 @@ module Scanners
|
|||
end
|
||||
|
||||
def scan_tokens tokens, options
|
||||
if string.respond_to?(:encoding)
|
||||
unless string.encoding == Encoding::ASCII_8BIT
|
||||
self.string = string.encode Encoding::ASCII_8BIT,
|
||||
:invalid => :replace, :undef => :replace, :replace => '?'
|
||||
end
|
||||
end
|
||||
|
||||
if check(RE::PHP_START) || # starts with <?
|
||||
(match?(/\s*<\S/) && exist?(RE::PHP_START)) || # starts with tag and contains <?
|
|
@ -49,12 +49,16 @@ module Scanners
|
|||
@html_scanner.tokenize match
|
||||
|
||||
elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
|
||||
start_tag = match[/\A<%[-=]?/]
|
||||
start_tag = match[/\A<%[-=#]?/]
|
||||
end_tag = match[/-?%?>?\z/]
|
||||
tokens << [:open, :inline]
|
||||
tokens << [start_tag, :inline_delimiter]
|
||||
code = match[start_tag.size .. -1 - end_tag.size]
|
||||
@ruby_scanner.tokenize code
|
||||
if start_tag == '<%#'
|
||||
tokens << [code, :comment]
|
||||
else
|
||||
@ruby_scanner.tokenize code
|
||||
end
|
||||
tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
|
||||
tokens << [:close, :inline]
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
# encoding: utf-8
|
||||
module CodeRay
|
||||
module Scanners
|
||||
|
||||
|
@ -28,6 +29,16 @@ module Scanners
|
|||
|
||||
private
|
||||
def scan_tokens tokens, options
|
||||
if string.respond_to?(:encoding)
|
||||
unless string.encoding == Encoding::UTF_8
|
||||
self.string = string.encode Encoding::UTF_8,
|
||||
:invalid => :replace, :undef => :replace, :replace => '?'
|
||||
end
|
||||
unicode = false
|
||||
else
|
||||
unicode = exist?(/[^\x00-\x7f]/)
|
||||
end
|
||||
|
||||
last_token_dot = false
|
||||
value_expected = true
|
||||
heredocs = nil
|
||||
|
@ -35,7 +46,7 @@ module Scanners
|
|||
state = :initial
|
||||
depth = nil
|
||||
inline_block_stack = []
|
||||
unicode = string.respond_to?(:encoding) && string.encoding.name == 'UTF-8'
|
||||
|
||||
|
||||
patterns = Patterns # avoid constant lookup
|
||||
|
||||
|
@ -170,19 +181,28 @@ module Scanners
|
|||
if last_token_dot
|
||||
kind = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
|
||||
else
|
||||
kind = patterns::IDENT_KIND[match]
|
||||
if kind == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
|
||||
kind = :constant
|
||||
elsif kind == :reserved
|
||||
state = patterns::DEF_NEW_STATE[match]
|
||||
value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
|
||||
if value_expected != :expect_colon && scan(/:(?= )/)
|
||||
tokens << [match, :key]
|
||||
match = ':'
|
||||
kind = :operator
|
||||
else
|
||||
kind = patterns::IDENT_KIND[match]
|
||||
if kind == :ident
|
||||
if match[/\A[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
|
||||
kind = :constant
|
||||
end
|
||||
elsif kind == :reserved
|
||||
state = patterns::DEF_NEW_STATE[match]
|
||||
value_expected = :set if patterns::KEYWORDS_EXPECTING_VALUE[match]
|
||||
end
|
||||
end
|
||||
end
|
||||
value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
|
||||
|
||||
elsif last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}|\(/o)
|
||||
kind = :ident
|
||||
value_expected = :set if check(/#{patterns::VALUE_FOLLOWS}/o)
|
||||
value_expected = :set if check(unicode ? /#{patterns::VALUE_FOLLOWS}/uo :
|
||||
/#{patterns::VALUE_FOLLOWS}/o)
|
||||
|
||||
# OPERATORS #
|
||||
elsif not last_token_dot and match = scan(/ \.\.\.? | (?:\.|::)() | [,\(\)\[\]\{\}] | ==?=? /x)
|
||||
|
@ -212,7 +232,8 @@ module Scanners
|
|||
kind = :delimiter
|
||||
state = patterns::StringState.new :string, match == '"', match # important for streaming
|
||||
|
||||
elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
|
||||
elsif match = scan(unicode ? /#{patterns::INSTANCE_VARIABLE}/uo :
|
||||
/#{patterns::INSTANCE_VARIABLE}/o)
|
||||
kind = :instance_variable
|
||||
|
||||
elsif value_expected and match = scan(/\//)
|
||||
|
@ -225,7 +246,8 @@ module Scanners
|
|||
elsif match = value_expected ? scan(/[-+]?#{patterns::NUMERIC}/o) : scan(/#{patterns::NUMERIC}/o)
|
||||
kind = self[1] ? :float : :integer
|
||||
|
||||
elsif match = scan(/#{patterns::SYMBOL}/o)
|
||||
elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
|
||||
/#{patterns::SYMBOL}/o)
|
||||
case delim = match[1]
|
||||
when ?', ?"
|
||||
tokens << [:open, :symbol]
|
||||
|
@ -237,11 +259,12 @@ module Scanners
|
|||
kind = :symbol
|
||||
end
|
||||
|
||||
elsif match = scan(/ [-+!~^]=? | [*|&]{1,2}=? | >>? /x)
|
||||
elsif match = scan(/ -[>=]? | [+!~^]=? | [*|&]{1,2}=? | >>? /x)
|
||||
value_expected = :set
|
||||
kind = :operator
|
||||
|
||||
elsif value_expected and match = scan(/#{patterns::HEREDOC_OPEN}/o)
|
||||
elsif value_expected and match = scan(unicode ? /#{patterns::HEREDOC_OPEN}/uo :
|
||||
/#{patterns::HEREDOC_OPEN}/o)
|
||||
indented = self[1] == '-'
|
||||
quote = self[3]
|
||||
delim = self[quote ? 4 : 2]
|
||||
|
@ -261,7 +284,8 @@ module Scanners
|
|||
state = patterns::StringState.new kind, interpreted, self[2]
|
||||
kind = :delimiter
|
||||
|
||||
elsif value_expected and match = scan(/#{patterns::CHARACTER}/o)
|
||||
elsif value_expected and match = scan(unicode ? /#{patterns::CHARACTER}/uo :
|
||||
/#{patterns::CHARACTER}/o)
|
||||
kind = :integer
|
||||
|
||||
elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
|
||||
|
@ -277,14 +301,16 @@ module Scanners
|
|||
state = patterns::StringState.new :shell, true, match
|
||||
end
|
||||
|
||||
elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
|
||||
elsif match = scan(unicode ? /#{patterns::GLOBAL_VARIABLE}/uo :
|
||||
/#{patterns::GLOBAL_VARIABLE}/o)
|
||||
kind = :global_variable
|
||||
|
||||
elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
|
||||
elsif match = scan(unicode ? /#{patterns::CLASS_VARIABLE}/uo :
|
||||
/#{patterns::CLASS_VARIABLE}/o)
|
||||
kind = :class_variable
|
||||
|
||||
else
|
||||
if !unicode
|
||||
if !unicode && !string.respond_to?(:encoding)
|
||||
# check for unicode
|
||||
debug, $DEBUG = $DEBUG, false
|
||||
begin
|
||||
|
@ -300,7 +326,7 @@ module Scanners
|
|||
next if unicode
|
||||
end
|
||||
kind = :error
|
||||
match = getch
|
||||
match = scan(unicode ? /./mu : /./m)
|
||||
|
||||
end
|
||||
|
||||
|
@ -322,7 +348,8 @@ module Scanners
|
|||
kind = :operator
|
||||
else
|
||||
state = :initial
|
||||
if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
|
||||
if match = scan(unicode ? /(?:#{patterns::IDENT}::)*#{patterns::IDENT}/uo :
|
||||
/(?:#{patterns::IDENT}::)*#{patterns::IDENT}/o)
|
||||
kind = :class
|
||||
else
|
||||
next
|
||||
|
@ -331,9 +358,11 @@ module Scanners
|
|||
|
||||
elsif state == :undef_expected
|
||||
state = :undef_comma_expected
|
||||
if match = scan(/#{patterns::METHOD_NAME_EX}/o)
|
||||
if match = scan(unicode ? /#{patterns::METHOD_NAME_EX}/uo :
|
||||
/#{patterns::METHOD_NAME_EX}/o)
|
||||
kind = :method
|
||||
elsif match = scan(/#{patterns::SYMBOL}/o)
|
||||
elsif match = scan(unicode ? /#{patterns::SYMBOL}/uo :
|
||||
/#{patterns::SYMBOL}/o)
|
||||
case delim = match[1]
|
||||
when ?', ?"
|
||||
tokens << [:open, :symbol]
|
||||
|
@ -375,7 +404,9 @@ module Scanners
|
|||
# }}}
|
||||
|
||||
unless kind == :error
|
||||
value_expected = value_expected == :set
|
||||
if value_expected = value_expected == :set
|
||||
value_expected = :expect_colon if match == '?' || match == 'when'
|
||||
end
|
||||
last_token_dot = last_token_dot == :set
|
||||
end
|
||||
|
|
@ -16,7 +16,7 @@ module Scanners
|
|||
DEF_KEYWORDS = %w[ def ]
|
||||
UNDEF_KEYWORDS = %w[ undef ]
|
||||
ALIAS_KEYWORDS = %w[ alias ]
|
||||
MODULE_KEYWORDS = %w[class module]
|
||||
MODULE_KEYWORDS = %w[ class module ]
|
||||
DEF_NEW_STATE = WordList.new(:initial).
|
||||
add(DEF_KEYWORDS, :def_expected).
|
||||
add(UNDEF_KEYWORDS, :undef_expected).
|
||||
|
@ -25,14 +25,26 @@ module Scanners
|
|||
|
||||
PREDEFINED_CONSTANTS = %w[
|
||||
nil true false self
|
||||
DATA ARGV ARGF __FILE__ __LINE__
|
||||
DATA ARGV ARGF
|
||||
__FILE__ __LINE__ __ENCODING__
|
||||
]
|
||||
|
||||
IDENT_KIND = WordList.new(:ident).
|
||||
add(RESERVED_WORDS, :reserved).
|
||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
||||
|
||||
IDENT = 'ä'[/[[:alpha:]]/] == 'ä' ? /[[:alpha:]_][[:alnum:]_]*/ : /[^\W\d]\w*/
|
||||
if /\w/u === '∑'
|
||||
# MRI 1.8.6, 1.8.7
|
||||
IDENT = /[^\W\d]\w*/
|
||||
else
|
||||
if //.respond_to? :encoding
|
||||
# MRI 1.9.1, 1.9.2
|
||||
IDENT = Regexp.new '[\p{L}\p{M}\p{Pc}\p{Sm}&&[^\x00-\x40\x5b-\x5e\x60\x7b-\x7f]][\p{L}\p{M}\p{N}\p{Pc}\p{Sm}&&[^\x00-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]]*'
|
||||
else
|
||||
# JRuby, Rubinius
|
||||
IDENT = /[^\x00-\x40\x5b-\x5e\x60\x7b-\x7f][^\x00-\x2f\x3a-\x40\x5b-\x5e\x60\x7b-\x7f]*/
|
||||
end
|
||||
end
|
||||
|
||||
METHOD_NAME = / #{IDENT} [?!]? /ox
|
||||
METHOD_NAME_OPERATOR = /
|
||||
|
@ -109,10 +121,12 @@ module Scanners
|
|||
|
||||
# NOTE: This is not completely correct, but
|
||||
# nobody needs heredoc delimiters ending with \n.
|
||||
# Also, delimiters starting with numbers are allowed.
|
||||
# but they are more often than not a false positive.
|
||||
HEREDOC_OPEN = /
|
||||
<< (-)? # $1 = float
|
||||
(?:
|
||||
( [A-Za-z_0-9]+ ) # $2 = delim
|
||||
( #{IDENT} ) # $2 = delim
|
||||
|
|
||||
( ["'`\/] ) # $3 = quote, type
|
||||
( [^\n]*? ) \3 # $4 = delim
|
|
@ -56,7 +56,7 @@ module CodeRay module Scanners
|
|||
if scan(/ \s+ | \\\n /x)
|
||||
kind = :space
|
||||
|
||||
elsif scan(/^(?:--\s?|#).*/)
|
||||
elsif scan(/(?:--\s?|#).*/)
|
||||
kind = :comment
|
||||
|
||||
elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
|
|
@ -33,6 +33,7 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top }
|
|||
}
|
||||
.CodeRay .line_numbers tt { font-weight: bold }
|
||||
.CodeRay .line_numbers .highlighted { color: red }
|
||||
.CodeRay .line { display: block; float: left; width: 100%; }
|
||||
.CodeRay .no { padding: 0px 4px }
|
||||
.CodeRay .code { width: 100% }
|
||||
|
|
@ -32,6 +32,8 @@ table.CodeRay td { padding: 2px 4px; vertical-align: top; }
|
|||
text-align: right;
|
||||
}
|
||||
.CodeRay .line_numbers tt { font-weight: bold; }
|
||||
.CodeRay .line_numbers .highlighted { color: red }
|
||||
.CodeRay .line { display: block; float: left; width: 100%; }
|
||||
.CodeRay .no { padding: 0px 4px; }
|
||||
.CodeRay .code { width: 100%; }
|
||||
|
|
@ -7,29 +7,30 @@ module CodeRay
|
|||
#
|
||||
# A token is not a special object, just a two-element Array
|
||||
# consisting of
|
||||
# * the _token_ _text_ (the original source of the token in a String) or
|
||||
# a _token_ _action_ (:open, :close, :begin_line, :end_line)
|
||||
# * the _token_ _kind_ (a Symbol representing the type of the token)
|
||||
# * the _token_ _text_ (the original source of the token in a String)
|
||||
#
|
||||
# A token looks like this:
|
||||
#
|
||||
# [:comment, '# It looks like this']
|
||||
# [:float, '3.1415926']
|
||||
# [:error, '$^']
|
||||
# ['# It looks like this', :comment]
|
||||
# ['3.1415926', :float]
|
||||
# ['$^', :error]
|
||||
#
|
||||
# Some scanners also yield some kind of sub-tokens, represented by special
|
||||
# token texts, namely :open and :close .
|
||||
# Some scanners also yield sub-tokens, represented by special
|
||||
# token actions, namely :open and :close.
|
||||
#
|
||||
# The Ruby scanner, for example, splits "a string" into:
|
||||
#
|
||||
# [
|
||||
# [:open, :string],
|
||||
# [:delimiter, '"'],
|
||||
# [:content, 'a string'],
|
||||
# [:delimiter, '"'],
|
||||
# ['"', :delimiter],
|
||||
# ['a string', :content],
|
||||
# ['"', :delimiter],
|
||||
# [:close, :string]
|
||||
# ]
|
||||
#
|
||||
# Tokens is also the interface between Scanners and Encoders:
|
||||
# Tokens is the interface between Scanners and Encoders:
|
||||
# The input is split and saved into a Tokens object. The Encoder
|
||||
# then builds the output from this object.
|
||||
#
|
||||
|
@ -43,6 +44,9 @@ module CodeRay
|
|||
# Tokens gives you the power to handle pre-scanned code very easily:
|
||||
# You can convert it to a webpage, a YAML file, or dump it into a gzip'ed string
|
||||
# that you put in your DB.
|
||||
#
|
||||
# It also allows you to generate tokens directly (without using a scanner),
|
||||
# to load them from a file, and still use any Encoder that CodeRay provides.
|
||||
#
|
||||
# Tokens' subclass TokenStream allows streaming to save memory.
|
||||
class Tokens < Array
|
||||
|
@ -239,9 +243,7 @@ module CodeRay
|
|||
size
|
||||
end
|
||||
|
||||
# The total size of the tokens.
|
||||
# Should be equal to the input size before
|
||||
# scanning.
|
||||
# Return all text tokens joined into a single string.
|
||||
def text
|
||||
map { |t, k| t if t.is_a? ::String }.join
|
||||
end
|
||||
|
@ -301,11 +303,11 @@ module CodeRay
|
|||
#
|
||||
# require 'coderay'
|
||||
#
|
||||
# token_stream = CodeRay::TokenStream.new do |kind, text|
|
||||
# token_stream = CodeRay::TokenStream.new do |text, kind|
|
||||
# puts 'kind: %s, text size: %d.' % [kind, text.size]
|
||||
# end
|
||||
#
|
||||
# token_stream << [:regexp, '/\d+/']
|
||||
# token_stream << ['/\d+/', :regexp]
|
||||
# #-> kind: rexpexp, text size: 5.
|
||||
#
|
||||
def initialize &block
|
|
@ -0,0 +1,122 @@
|
|||
require 'test/unit'
|
||||
require 'coderay'
|
||||
|
||||
class BasicTest < Test::Unit::TestCase
|
||||
|
||||
def test_version
|
||||
assert_nothing_raised do
|
||||
assert_match(/\A\d\.\d\.\d\z/, CodeRay::VERSION)
|
||||
end
|
||||
end
|
||||
|
||||
RUBY_TEST_CODE = 'puts "Hello, World!"'
|
||||
|
||||
RUBY_TEST_TOKENS = [
|
||||
['puts', :ident],
|
||||
[' ', :space],
|
||||
[:open, :string],
|
||||
['"', :delimiter],
|
||||
['Hello, World!', :content],
|
||||
['"', :delimiter],
|
||||
[:close, :string]
|
||||
]
|
||||
def test_simple_scan
|
||||
assert_nothing_raised do
|
||||
assert_equal RUBY_TEST_TOKENS, CodeRay.scan(RUBY_TEST_CODE, :ruby).to_ary
|
||||
end
|
||||
end
|
||||
|
||||
RUBY_TEST_HTML = 'puts <span class="s"><span class="dl">"</span>' +
|
||||
'<span class="k">Hello, World!</span><span class="dl">"</span></span>'
|
||||
def test_simple_highlight
|
||||
assert_nothing_raised do
|
||||
assert_equal RUBY_TEST_HTML, CodeRay.scan(RUBY_TEST_CODE, :ruby).html
|
||||
end
|
||||
end
|
||||
|
||||
def test_duo
|
||||
assert_equal(RUBY_TEST_CODE,
|
||||
CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE))
|
||||
assert_equal(RUBY_TEST_CODE,
|
||||
CodeRay::Duo[:plain => :plain].highlight(RUBY_TEST_CODE))
|
||||
end
|
||||
|
||||
def test_duo_stream
|
||||
assert_equal(RUBY_TEST_CODE,
|
||||
CodeRay::Duo[:plain, :plain].highlight(RUBY_TEST_CODE, :stream => true))
|
||||
end
|
||||
|
||||
def test_comment_filter
|
||||
assert_equal <<-EXPECTED, CodeRay.scan(<<-INPUT, :ruby).comment_filter.text
|
||||
#!/usr/bin/env ruby
|
||||
|
||||
code
|
||||
|
||||
more code
|
||||
EXPECTED
|
||||
#!/usr/bin/env ruby
|
||||
=begin
|
||||
A multi-line comment.
|
||||
=end
|
||||
code
|
||||
# A single-line comment.
|
||||
more code # and another comment, in-line.
|
||||
INPUT
|
||||
end
|
||||
|
||||
def test_lines_of_code
|
||||
assert_equal 2, CodeRay.scan(<<-INPUT, :ruby).lines_of_code
|
||||
#!/usr/bin/env ruby
|
||||
=begin
|
||||
A multi-line comment.
|
||||
=end
|
||||
code
|
||||
# A single-line comment.
|
||||
more code # and another comment, in-line.
|
||||
INPUT
|
||||
rHTML = <<-RHTML
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
|
||||
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
|
||||
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head>
|
||||
<meta http-equiv="content-type" content="text/html;charset=UTF-8" />
|
||||
<title><%= controller.controller_name.titleize %>: <%= controller.action_name %></title>
|
||||
<%= stylesheet_link_tag 'scaffold' %>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<p style="color: green"><%= flash[:notice] %></p>
|
||||
|
||||
<div id="main">
|
||||
<%= yield %>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
RHTML
|
||||
assert_equal 0, CodeRay.scan(rHTML, :html).lines_of_code
|
||||
assert_equal 0, CodeRay.scan(rHTML, :php).lines_of_code
|
||||
assert_equal 0, CodeRay.scan(rHTML, :yaml).lines_of_code
|
||||
assert_equal 4, CodeRay.scan(rHTML, :rhtml).lines_of_code
|
||||
end
|
||||
|
||||
def test_rubygems_not_loaded
|
||||
assert_equal nil, defined? Gem
|
||||
end if ENV['check_rubygems'] && RUBY_VERSION < '1.9'
|
||||
|
||||
def test_list_of_encoders
|
||||
assert_kind_of(Array, CodeRay::Encoders.list)
|
||||
assert CodeRay::Encoders.list.include?('count')
|
||||
end
|
||||
|
||||
def test_list_of_scanners
|
||||
assert_kind_of(Array, CodeRay::Scanners.list)
|
||||
assert CodeRay::Scanners.list.include?('plaintext')
|
||||
end
|
||||
|
||||
def test_scan_a_frozen_string
|
||||
CodeRay.scan RUBY_VERSION, :ruby
|
||||
end
|
||||
|
||||
end
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,77 @@
|
|||
require 'test/unit'
|
||||
$:.unshift 'lib'
|
||||
require 'coderay'
|
||||
|
||||
begin
|
||||
require 'rubygems' unless defined? Gem
|
||||
gem 'RedCloth', '>= 4.0.3' rescue nil
|
||||
require 'redcloth'
|
||||
rescue LoadError
|
||||
warn 'RedCloth not found - skipping for_redcloth tests.'
|
||||
end
|
||||
|
||||
class BasicTest < Test::Unit::TestCase
|
||||
|
||||
def test_for_redcloth
|
||||
require 'coderay/for_redcloth'
|
||||
assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">puts <span style=\"background-color:#fff0f0;color:#D20\"><span style=\"color:#710\">"</span><span style=\"\">Hello, World!</span><span style=\"color:#710\">"</span></span></span></p>",
|
||||
RedCloth.new('@[ruby]puts "Hello, World!"@').to_html
|
||||
assert_equal <<-BLOCKCODE.chomp,
|
||||
<div lang="ruby" class="CodeRay">
|
||||
<div class="code"><pre>puts <span style="background-color:#fff0f0;color:#D20"><span style="color:#710">"</span><span style="">Hello, World!</span><span style="color:#710">"</span></span></pre></div>
|
||||
</div>
|
||||
BLOCKCODE
|
||||
RedCloth.new('bc[ruby]. puts "Hello, World!"').to_html
|
||||
end
|
||||
|
||||
def test_for_redcloth_no_lang
|
||||
require 'coderay/for_redcloth'
|
||||
assert_equal "<p><code>puts \"Hello, World!\"</code></p>",
|
||||
RedCloth.new('@puts "Hello, World!"@').to_html
|
||||
assert_equal <<-BLOCKCODE.chomp,
|
||||
<pre><code>puts \"Hello, World!\"</code></pre>
|
||||
BLOCKCODE
|
||||
RedCloth.new('bc. puts "Hello, World!"').to_html
|
||||
end
|
||||
|
||||
def test_for_redcloth_style
|
||||
require 'coderay/for_redcloth'
|
||||
assert_equal <<-BLOCKCODE.chomp,
|
||||
<pre style=\"color: red;\"><code style=\"color: red;\">puts \"Hello, World!\"</code></pre>
|
||||
BLOCKCODE
|
||||
RedCloth.new('bc{color: red}. puts "Hello, World!"').to_html
|
||||
end
|
||||
|
||||
def test_for_redcloth_escapes
|
||||
require 'coderay/for_redcloth'
|
||||
assert_equal '<p><span lang="ruby" class="CodeRay">></span></p>',
|
||||
RedCloth.new('@[ruby]>@').to_html
|
||||
assert_equal <<-BLOCKCODE.chomp,
|
||||
<div lang="ruby" class="CodeRay">
|
||||
<div class="code"><pre>&</pre></div>
|
||||
</div>
|
||||
BLOCKCODE
|
||||
RedCloth.new('bc[ruby]. &').to_html
|
||||
end
|
||||
|
||||
def test_for_redcloth_escapes2
|
||||
require 'coderay/for_redcloth'
|
||||
assert_equal "<p><span lang=\"c\" class=\"CodeRay\"><span style=\"color:#579\">#include</span> <span style=\"color:#B44;font-weight:bold\"><test.h></span></span></p>",
|
||||
RedCloth.new('@[c]#include <test.h>@').to_html
|
||||
end
|
||||
|
||||
# See http://jgarber.lighthouseapp.com/projects/13054/tickets/124-code-markup-does-not-allow-brackets.
|
||||
def test_for_redcloth_false_positive
|
||||
require 'coderay/for_redcloth'
|
||||
assert_equal '<p><code>[project]_dff.skjd</code></p>',
|
||||
RedCloth.new('@[project]_dff.skjd@').to_html
|
||||
# false positive, but expected behavior / known issue
|
||||
assert_equal "<p><span lang=\"ruby\" class=\"CodeRay\">_dff.skjd</span></p>",
|
||||
RedCloth.new('@[ruby]_dff.skjd@').to_html
|
||||
assert_equal <<-BLOCKCODE.chomp,
|
||||
<pre><code>[project]_dff.skjd</code></pre>
|
||||
BLOCKCODE
|
||||
RedCloth.new('bc. [project]_dff.skjd').to_html
|
||||
end
|
||||
|
||||
end if defined? RedCloth
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,11 @@
|
|||
require 'test/unit'
|
||||
require 'coderay'
|
||||
|
||||
class PluginScannerTest < Test::Unit::TestCase
|
||||
|
||||
def test_load
|
||||
require File.join(File.dirname(__FILE__), 'vhdl')
|
||||
assert_equal 'VHDL', CodeRay.scanner(:vhdl).class.name
|
||||
end
|
||||
|
||||
end
|
|
@ -0,0 +1,317 @@
|
|||
!RBIX
|
||||
0
|
||||
x
|
||||
M
|
||||
1
|
||||
n
|
||||
n
|
||||
x
|
||||
10
|
||||
__script__
|
||||
i
|
||||
53
|
||||
5
|
||||
7
|
||||
0
|
||||
64
|
||||
47
|
||||
49
|
||||
1
|
||||
1
|
||||
15
|
||||
5
|
||||
7
|
||||
2
|
||||
64
|
||||
47
|
||||
49
|
||||
1
|
||||
1
|
||||
15
|
||||
99
|
||||
7
|
||||
3
|
||||
45
|
||||
4
|
||||
5
|
||||
43
|
||||
6
|
||||
43
|
||||
7
|
||||
65
|
||||
49
|
||||
8
|
||||
3
|
||||
13
|
||||
99
|
||||
12
|
||||
7
|
||||
9
|
||||
12
|
||||
7
|
||||
10
|
||||
12
|
||||
65
|
||||
12
|
||||
49
|
||||
11
|
||||
4
|
||||
15
|
||||
49
|
||||
9
|
||||
0
|
||||
15
|
||||
2
|
||||
11
|
||||
I
|
||||
6
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
n
|
||||
p
|
||||
12
|
||||
s
|
||||
9
|
||||
test/unit
|
||||
x
|
||||
7
|
||||
require
|
||||
s
|
||||
7
|
||||
coderay
|
||||
x
|
||||
17
|
||||
PluginScannerTest
|
||||
x
|
||||
4
|
||||
Test
|
||||
n
|
||||
x
|
||||
4
|
||||
Unit
|
||||
x
|
||||
8
|
||||
TestCase
|
||||
x
|
||||
10
|
||||
open_class
|
||||
x
|
||||
14
|
||||
__class_init__
|
||||
M
|
||||
1
|
||||
n
|
||||
n
|
||||
x
|
||||
17
|
||||
PluginScannerTest
|
||||
i
|
||||
16
|
||||
5
|
||||
66
|
||||
99
|
||||
7
|
||||
0
|
||||
7
|
||||
1
|
||||
65
|
||||
67
|
||||
49
|
||||
2
|
||||
0
|
||||
49
|
||||
3
|
||||
4
|
||||
11
|
||||
I
|
||||
5
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
n
|
||||
p
|
||||
4
|
||||
x
|
||||
9
|
||||
test_load
|
||||
M
|
||||
1
|
||||
n
|
||||
n
|
||||
x
|
||||
9
|
||||
test_load
|
||||
i
|
||||
48
|
||||
5
|
||||
45
|
||||
0
|
||||
1
|
||||
45
|
||||
0
|
||||
2
|
||||
65
|
||||
49
|
||||
3
|
||||
0
|
||||
49
|
||||
4
|
||||
1
|
||||
7
|
||||
5
|
||||
64
|
||||
49
|
||||
6
|
||||
2
|
||||
47
|
||||
49
|
||||
7
|
||||
1
|
||||
15
|
||||
5
|
||||
7
|
||||
8
|
||||
64
|
||||
45
|
||||
9
|
||||
10
|
||||
7
|
||||
11
|
||||
49
|
||||
12
|
||||
1
|
||||
49
|
||||
13
|
||||
0
|
||||
49
|
||||
14
|
||||
0
|
||||
47
|
||||
49
|
||||
15
|
||||
2
|
||||
11
|
||||
I
|
||||
4
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
n
|
||||
p
|
||||
16
|
||||
x
|
||||
4
|
||||
File
|
||||
n
|
||||
n
|
||||
x
|
||||
11
|
||||
active_path
|
||||
x
|
||||
7
|
||||
dirname
|
||||
s
|
||||
4
|
||||
vhdl
|
||||
x
|
||||
4
|
||||
join
|
||||
x
|
||||
7
|
||||
require
|
||||
s
|
||||
4
|
||||
VHDL
|
||||
x
|
||||
7
|
||||
CodeRay
|
||||
n
|
||||
x
|
||||
4
|
||||
vhdl
|
||||
x
|
||||
7
|
||||
scanner
|
||||
x
|
||||
5
|
||||
class
|
||||
x
|
||||
4
|
||||
name
|
||||
x
|
||||
12
|
||||
assert_equal
|
||||
p
|
||||
7
|
||||
I
|
||||
0
|
||||
I
|
||||
6
|
||||
I
|
||||
0
|
||||
I
|
||||
7
|
||||
I
|
||||
19
|
||||
I
|
||||
8
|
||||
I
|
||||
30
|
||||
x
|
||||
69
|
||||
/Users/murphy/ruby/coderay-0.9/test/functional/load_plugin_scanner.rb
|
||||
p
|
||||
0
|
||||
x
|
||||
17
|
||||
method_visibility
|
||||
x
|
||||
15
|
||||
add_defn_method
|
||||
p
|
||||
3
|
||||
I
|
||||
2
|
||||
I
|
||||
6
|
||||
I
|
||||
10
|
||||
x
|
||||
69
|
||||
/Users/murphy/ruby/coderay-0.9/test/functional/load_plugin_scanner.rb
|
||||
p
|
||||
0
|
||||
x
|
||||
13
|
||||
attach_method
|
||||
p
|
||||
7
|
||||
I
|
||||
0
|
||||
I
|
||||
1
|
||||
I
|
||||
9
|
||||
I
|
||||
2
|
||||
I
|
||||
12
|
||||
I
|
||||
4
|
||||
I
|
||||
35
|
||||
x
|
||||
69
|
||||
/Users/murphy/ruby/coderay-0.9/test/functional/load_plugin_scanner.rb
|
||||
p
|
||||
0
|
|
@ -0,0 +1,12 @@
|
|||
require 'test/unit'
|
||||
|
||||
MYDIR = File.dirname(__FILE__)
|
||||
|
||||
$:.unshift 'lib'
|
||||
require 'coderay'
|
||||
puts "Running basic CodeRay #{CodeRay::VERSION} tests..."
|
||||
|
||||
suite = %w(basic load_plugin_scanner word_list)
|
||||
for test_case in suite
|
||||
load File.join(MYDIR, test_case + '.rb')
|
||||
end
|
|
@ -0,0 +1,322 @@
|
|||
!RBIX
|
||||
0
|
||||
x
|
||||
M
|
||||
1
|
||||
n
|
||||
n
|
||||
x
|
||||
10
|
||||
__script__
|
||||
i
|
||||
95
|
||||
5
|
||||
7
|
||||
0
|
||||
64
|
||||
47
|
||||
49
|
||||
1
|
||||
1
|
||||
15
|
||||
65
|
||||
7
|
||||
2
|
||||
45
|
||||
3
|
||||
4
|
||||
65
|
||||
49
|
||||
5
|
||||
0
|
||||
49
|
||||
6
|
||||
1
|
||||
49
|
||||
7
|
||||
2
|
||||
15
|
||||
99
|
||||
43
|
||||
8
|
||||
7
|
||||
9
|
||||
49
|
||||
10
|
||||
1
|
||||
7
|
||||
11
|
||||
64
|
||||
49
|
||||
12
|
||||
1
|
||||
15
|
||||
5
|
||||
7
|
||||
13
|
||||
64
|
||||
47
|
||||
49
|
||||
1
|
||||
1
|
||||
15
|
||||
5
|
||||
7
|
||||
14
|
||||
45
|
||||
15
|
||||
16
|
||||
43
|
||||
17
|
||||
47
|
||||
49
|
||||
18
|
||||
0
|
||||
7
|
||||
19
|
||||
63
|
||||
3
|
||||
47
|
||||
49
|
||||
20
|
||||
1
|
||||
15
|
||||
7
|
||||
21
|
||||
64
|
||||
7
|
||||
22
|
||||
64
|
||||
7
|
||||
23
|
||||
64
|
||||
35
|
||||
3
|
||||
19
|
||||
0
|
||||
15
|
||||
20
|
||||
0
|
||||
56
|
||||
24
|
||||
50
|
||||
25
|
||||
0
|
||||
15
|
||||
2
|
||||
11
|
||||
I
|
||||
6
|
||||
I
|
||||
2
|
||||
I
|
||||
0
|
||||
I
|
||||
0
|
||||
n
|
||||
p
|
||||
26
|
||||
s
|
||||
9
|
||||
test/unit
|
||||
x
|
||||
7
|
||||
require
|
||||
x
|
||||
5
|
||||
MYDIR
|
||||
x
|
||||
4
|
||||
File
|
||||
n
|
||||
x
|
||||
11
|
||||
active_path
|
||||
x
|
||||
7
|
||||
dirname
|
||||
x
|
||||
9
|
||||
const_set
|
||||
x
|
||||
7
|
||||
Globals
|
||||
x
|
||||
2
|
||||
$:
|
||||
x
|
||||
2
|
||||
[]
|
||||
s
|
||||
3
|
||||
lib
|
||||
x
|
||||
2
|
||||
<<
|
||||
s
|
||||
7
|
||||
coderay
|
||||
s
|
||||
22
|
||||
Running basic CodeRay
|
||||
x
|
||||
7
|
||||
CodeRay
|
||||
n
|
||||
x
|
||||
7
|
||||
VERSION
|
||||
x
|
||||
4
|
||||
to_s
|
||||
s
|
||||
9
|
||||
tests...
|
||||
x
|
||||
4
|
||||
puts
|
||||
s
|
||||
5
|
||||
basic
|
||||
s
|
||||
19
|
||||
load_plugin_scanner
|
||||
s
|
||||
9
|
||||
word_list
|
||||
M
|
||||
1
|
||||
p
|
||||
2
|
||||
x
|
||||
9
|
||||
for_block
|
||||
t
|
||||
n
|
||||
x
|
||||
9
|
||||
__block__
|
||||
i
|
||||
28
|
||||
57
|
||||
22
|
||||
1
|
||||
1
|
||||
15
|
||||
5
|
||||
45
|
||||
0
|
||||
1
|
||||
45
|
||||
2
|
||||
3
|
||||
21
|
||||
1
|
||||
1
|
||||
7
|
||||
4
|
||||
64
|
||||
81
|
||||
5
|
||||
49
|
||||
6
|
||||
2
|
||||
47
|
||||
49
|
||||
7
|
||||
1
|
||||
11
|
||||
I
|
||||
6
|
||||
I
|
||||
0
|
||||
I
|
||||
1
|
||||
I
|
||||
1
|
||||
n
|
||||
p
|
||||
8
|
||||
x
|
||||
4
|
||||
File
|
||||
n
|
||||
x
|
||||
5
|
||||
MYDIR
|
||||
n
|
||||
s
|
||||
3
|
||||
.rb
|
||||
x
|
||||
1
|
||||
+
|
||||
x
|
||||
4
|
||||
join
|
||||
x
|
||||
4
|
||||
load
|
||||
p
|
||||
5
|
||||
I
|
||||
0
|
||||
I
|
||||
a
|
||||
I
|
||||
5
|
||||
I
|
||||
b
|
||||
I
|
||||
1c
|
||||
x
|
||||
55
|
||||
/Users/murphy/ruby/coderay-0.9/test/functional/suite.rb
|
||||
p
|
||||
0
|
||||
x
|
||||
4
|
||||
each
|
||||
p
|
||||
15
|
||||
I
|
||||
0
|
||||
I
|
||||
1
|
||||
I
|
||||
9
|
||||
I
|
||||
3
|
||||
I
|
||||
1a
|
||||
I
|
||||
5
|
||||
I
|
||||
29
|
||||
I
|
||||
6
|
||||
I
|
||||
32
|
||||
I
|
||||
7
|
||||
I
|
||||
47
|
||||
I
|
||||
9
|
||||
I
|
||||
55
|
||||
I
|
||||
a
|
||||
I
|
||||
5f
|
||||
x
|
||||
55
|
||||
/Users/murphy/ruby/coderay-0.9/test/functional/suite.rb
|
||||
p
|
||||
2
|
||||
x
|
||||
5
|
||||
suite
|
||||
x
|
||||
9
|
||||
test_case
|
|
@ -0,0 +1,126 @@
|
|||
class VHDL < CodeRay::Scanners::Scanner
|
||||
|
||||
register_for :vhdl
|
||||
|
||||
RESERVED_WORDS = [
|
||||
'access','after','alias','all','assert','architecture','begin',
|
||||
'block','body','buffer','bus','case','component','configuration','constant',
|
||||
'disconnect','downto','else','elsif','end','entity','exit','file','for',
|
||||
'function','generate','generic','group','guarded','if','impure','in',
|
||||
'inertial','inout','is','label','library','linkage','literal','loop',
|
||||
'map','new','next','null','of','on','open','others','out','package',
|
||||
'port','postponed','procedure','process','pure','range','record','register',
|
||||
'reject','report','return','select','severity','signal','shared','subtype',
|
||||
'then','to','transport','type','unaffected','units','until','use','variable',
|
||||
'wait','when','while','with','note','warning','error','failure','and',
|
||||
'or','xor','not','nor',
|
||||
'array'
|
||||
]
|
||||
|
||||
PREDEFINED_TYPES = [
|
||||
'bit','bit_vector','character','boolean','integer','real','time','string',
|
||||
'severity_level','positive','natural','signed','unsigned','line','text',
|
||||
'std_logic','std_logic_vector','std_ulogic','std_ulogic_vector','qsim_state',
|
||||
'qsim_state_vector','qsim_12state','qsim_12state_vector','qsim_strength',
|
||||
'mux_bit','mux_vector','reg_bit','reg_vector','wor_bit','wor_vector'
|
||||
]
|
||||
|
||||
PREDEFINED_CONSTANTS = [
|
||||
|
||||
]
|
||||
|
||||
IDENT_KIND = CodeRay::CaseIgnoringWordList.new(:ident).
|
||||
add(RESERVED_WORDS, :reserved).
|
||||
add(PREDEFINED_TYPES, :pre_type).
|
||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
||||
|
||||
ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
||||
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
||||
|
||||
def scan_tokens tokens, options
|
||||
|
||||
state = :initial
|
||||
|
||||
until eos?
|
||||
|
||||
kind = nil
|
||||
match = nil
|
||||
|
||||
case state
|
||||
|
||||
when :initial
|
||||
|
||||
if scan(/ \s+ | \\\n /x)
|
||||
kind = :space
|
||||
|
||||
elsif scan(/-- .*/x)
|
||||
kind = :comment
|
||||
|
||||
elsif scan(/ [-+*\/=<>?:;,!&^|()\[\]{}~%]+ | \.(?!\d) /x)
|
||||
kind = :operator
|
||||
|
||||
elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
|
||||
kind = IDENT_KIND[match.downcase]
|
||||
|
||||
elsif match = scan(/[a-z]?"/i)
|
||||
tokens << [:open, :string]
|
||||
state = :string
|
||||
kind = :delimiter
|
||||
|
||||
elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
|
||||
kind = :char
|
||||
|
||||
elsif scan(/(?:\d+)(?![.eEfF])/)
|
||||
kind = :integer
|
||||
|
||||
elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
||||
kind = :float
|
||||
|
||||
else
|
||||
getch
|
||||
kind = :error
|
||||
|
||||
end
|
||||
|
||||
when :string
|
||||
if scan(/[^\\\n"]+/)
|
||||
kind = :content
|
||||
elsif scan(/"/)
|
||||
tokens << ['"', :delimiter]
|
||||
tokens << [:close, :string]
|
||||
state = :initial
|
||||
next
|
||||
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
||||
kind = :char
|
||||
elsif scan(/ \\ | $ /x)
|
||||
tokens << [:close, :string]
|
||||
kind = :error
|
||||
state = :initial
|
||||
else
|
||||
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
||||
end
|
||||
|
||||
else
|
||||
raise_inspect 'Unknown state', tokens
|
||||
|
||||
end
|
||||
|
||||
match ||= matched
|
||||
if $DEBUG and not kind
|
||||
raise_inspect 'Error token %p in line %d' %
|
||||
[[match, kind], line], tokens
|
||||
end
|
||||
raise_inspect 'Empty token', tokens unless match
|
||||
|
||||
tokens << [match, kind]
|
||||
|
||||
end
|
||||
|
||||
if state == :string
|
||||
tokens << [:close, :string]
|
||||
end
|
||||
|
||||
tokens
|
||||
end
|
||||
|
||||
end
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,79 @@
|
|||
require 'test/unit'
|
||||
require 'coderay'
|
||||
|
||||
class WordListTest < Test::Unit::TestCase
|
||||
|
||||
include CodeRay
|
||||
|
||||
# define word arrays
|
||||
RESERVED_WORDS = %w[
|
||||
asm break case continue default do else
|
||||
...
|
||||
]
|
||||
|
||||
PREDEFINED_TYPES = %w[
|
||||
int long short char void
|
||||
...
|
||||
]
|
||||
|
||||
PREDEFINED_CONSTANTS = %w[
|
||||
EOF NULL ...
|
||||
]
|
||||
|
||||
# make a WordList
|
||||
IDENT_KIND = WordList.new(:ident).
|
||||
add(RESERVED_WORDS, :reserved).
|
||||
add(PREDEFINED_TYPES, :pre_type).
|
||||
add(PREDEFINED_CONSTANTS, :pre_constant)
|
||||
|
||||
def test_word_list_example
|
||||
assert_equal :pre_type, IDENT_KIND['void']
|
||||
# assert_equal :pre_constant, IDENT_KIND['...'] # not specified
|
||||
end
|
||||
|
||||
def test_word_list
|
||||
list = WordList.new(:ident).add(['foobar'], :reserved)
|
||||
assert_equal :reserved, list['foobar']
|
||||
assert_equal :ident, list['FooBar']
|
||||
end
|
||||
|
||||
def test_word_list_cached
|
||||
list = WordList.new(:ident, true).add(['foobar'], :reserved)
|
||||
assert_equal :reserved, list['foobar']
|
||||
assert_equal :ident, list['FooBar']
|
||||
end
|
||||
|
||||
def test_case_ignoring_word_list
|
||||
list = CaseIgnoringWordList.new(:ident).add(['foobar'], :reserved)
|
||||
assert_equal :ident, list['foo']
|
||||
assert_equal :reserved, list['foobar']
|
||||
assert_equal :reserved, list['FooBar']
|
||||
|
||||
list = CaseIgnoringWordList.new(:ident).add(['FooBar'], :reserved)
|
||||
assert_equal :ident, list['foo']
|
||||
assert_equal :reserved, list['foobar']
|
||||
assert_equal :reserved, list['FooBar']
|
||||
end
|
||||
|
||||
def test_case_ignoring_word_list_cached
|
||||
list = CaseIgnoringWordList.new(:ident, true).add(['foobar'], :reserved)
|
||||
assert_equal :ident, list['foo']
|
||||
assert_equal :reserved, list['foobar']
|
||||
assert_equal :reserved, list['FooBar']
|
||||
|
||||
list = CaseIgnoringWordList.new(:ident, true).add(['FooBar'], :reserved)
|
||||
assert_equal :ident, list['foo']
|
||||
assert_equal :reserved, list['foobar']
|
||||
assert_equal :reserved, list['FooBar']
|
||||
end
|
||||
|
||||
def test_dup
|
||||
list = WordList.new(:ident).add(['foobar'], :reserved)
|
||||
assert_equal :reserved, list['foobar']
|
||||
list2 = list.dup
|
||||
list2.add(%w[foobar], :keyword)
|
||||
assert_equal :keyword, list2['foobar']
|
||||
assert_equal :reserved, list['foobar']
|
||||
end
|
||||
|
||||
end
|
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue