138 lines
3.2 KiB
Ruby
138 lines
3.2 KiB
Ruby
module CodeRay
|
|
|
|
# = WordList
|
|
#
|
|
# <b>A Hash subclass designed for mapping word lists to token types.</b>
|
|
#
|
|
# Copyright (c) 2006 by murphy (Kornelius Kalnbach) <murphy rubychan de>
|
|
#
|
|
# License:: LGPL / ask the author
|
|
# Version:: 1.1 (2006-Oct-19)
|
|
#
|
|
# A WordList is a Hash with some additional features.
|
|
# It is intended to be used for keyword recognition.
|
|
#
|
|
# WordList is highly optimized to be used in Scanners,
|
|
# typically to decide whether a given ident is a special token.
|
|
#
|
|
# For case insensitive words use CaseIgnoringWordList.
|
|
#
|
|
# Example:
|
|
#
|
|
# # define word arrays
|
|
# RESERVED_WORDS = %w[
|
|
# asm break case continue default do else
|
|
# ...
|
|
# ]
|
|
#
|
|
# PREDEFINED_TYPES = %w[
|
|
# int long short char void
|
|
# ...
|
|
# ]
|
|
#
|
|
# PREDEFINED_CONSTANTS = %w[
|
|
# EOF NULL ...
|
|
# ]
|
|
#
|
|
# # make a WordList
|
|
# IDENT_KIND = WordList.new(:ident).
|
|
# add(RESERVED_WORDS, :reserved).
|
|
# add(PREDEFINED_TYPES, :pre_type).
|
|
# add(PREDEFINED_CONSTANTS, :pre_constant)
|
|
#
|
|
# ...
|
|
#
|
|
# def scan_tokens tokens, options
|
|
# ...
|
|
#
|
|
# elsif scan(/[A-Za-z_][A-Za-z_0-9]*/)
|
|
# # use it
|
|
# kind = IDENT_KIND[match]
|
|
# ...
|
|
class WordList < Hash
|
|
|
|
# Creates a new WordList with +default+ as default value.
|
|
#
|
|
# You can activate +caching+ to store the results for every [] request.
|
|
#
|
|
# With caching, methods like +include?+ or +delete+ may no longer behave
|
|
# as you expect. Therefore, it is recommended to use the [] method only.
|
|
def initialize default = false, caching = false, &block
|
|
if block
|
|
raise ArgumentError, 'Can\'t combine block with caching.' if caching
|
|
super(&block)
|
|
else
|
|
if caching
|
|
super() do |h, k|
|
|
h[k] = h.fetch k, default
|
|
end
|
|
else
|
|
super default
|
|
end
|
|
end
|
|
end
|
|
|
|
# Add words to the list and associate them with +kind+.
|
|
#
|
|
# Returns +self+, so you can concat add calls.
|
|
def add words, kind = true
|
|
words.each do |word|
|
|
self[word] = kind
|
|
end
|
|
self
|
|
end
|
|
|
|
end
|
|
|
|
|
|
# A CaseIgnoringWordList is like a WordList, only that
|
|
# keys are compared case-insensitively.
|
|
#
|
|
# Ignoring the text case is realized by sending the +downcase+ message to
|
|
# all keys.
|
|
#
|
|
# Caching usually makes a CaseIgnoringWordList faster, but it has to be
|
|
# activated explicitely.
|
|
class CaseIgnoringWordList < WordList
|
|
|
|
# Creates a new case-insensitive WordList with +default+ as default value.
|
|
#
|
|
# You can activate caching to store the results for every [] request.
|
|
# This speeds up subsequent lookups for the same word, but also
|
|
# uses memory.
|
|
def initialize default = false, caching = false
|
|
if caching
|
|
super(default, false) do |h, k|
|
|
h[k] = h.fetch k.downcase, default
|
|
end
|
|
else
|
|
super(default, false)
|
|
extend Uncached
|
|
end
|
|
end
|
|
|
|
module Uncached # :nodoc:
|
|
def [] key
|
|
super(key.downcase)
|
|
end
|
|
end
|
|
|
|
# Add +words+ to the list and associate them with +kind+.
|
|
def add words, kind = true
|
|
words.each do |word|
|
|
self[word.downcase] = kind
|
|
end
|
|
self
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
__END__
|
|
# check memory consumption
|
|
END {
|
|
ObjectSpace.each_object(CodeRay::CaseIgnoringWordList) do |wl|
|
|
p wl.inject(0) { |memo, key, value| memo + key.size + 24 }
|
|
end
|
|
} |