diff --git a/.travis.yml b/.travis.yml index f8156982..b99c95e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +env: + global: + - "JRUBY_OPTS=-Xcext.enabled=true" rvm: - 1.8.7 - ree @@ -5,21 +8,19 @@ rvm: - 2.0 - 2.1 - 2.2 - - 2.3.0 + - 2.3 + - 2.4 - ruby-head - - jruby-18mode - - jruby-19mode - - jruby-head - - rbx-18mode - - rbx-19mode + - jruby branches: only: - master +before_script: +- if (ruby -e "exit RUBY_VERSION.to_f >= 2.3"); then export RUBYOPT="--enable-frozen-string-literal"; fi; echo $RUBYOPT matrix: allow_failures: - rvm: ruby-head - - rvm: jruby-head - - rvm: rbx-18mode - - rvm: rbx-19mode + - rvm: jruby + - rvm: rbx script: "rake test" # test:scanners" sudo: false diff --git a/Changes.textile b/Changes.textile index 10f1d6eb..37de4ace 100644 --- a/Changes.textile +++ b/Changes.textile @@ -2,6 +2,13 @@ h1=. CodeRay Version History p=. _This files lists all changes in the CodeRay library since the 0.9.8 release._ +h2. Changes in 1.1.2 + +* Ruby future: Add support for frozen string literals. [#211, thanks to Pat Allan] +* C++ scanner: Add C++11 keywords. [#195, thanks to Johnny Willemsen] +* Haml scanner: Allow @-@ in tags. +* Java scanner: Allow Unicode characters in identifiers. [#212, thanks to t-gergely] + h2. Changes in 1.1.1 * SQL scanner: Allow @$@ signs in SQL identifiers [#164, thanks to jasir and Ben Basson] diff --git a/FOLDERS b/FOLDERS index f29255ae..1709d08a 100644 --- a/FOLDERS +++ b/FOLDERS @@ -1,48 +1,48 @@ -= CodeRay - folder structure - -== bench - Benchmarking system - -All benchmarking stuff goes here. - -Test inputs are stored in files named example.. -Test outputs go to bench/test.. - -Run bench/bench.rb to get a usage description. - -Run rake bench to perform an example benchmark. - - -== bin - Scripts - -Executional files for CodeRay. - -coderay:: The CodeRay executable. - -== demo - Demos and functional tests - -Demonstrational scripts to show of CodeRay's features. - -Run them as functional tests with rake test:demos. - - -== etc - Lots of stuff - -Some addidtional files for CodeRay, mainly graphics and Vim scripts. - - -== lib - CodeRay library code - -This is the base directory for the CodeRay library. - - -== rake_helpers - Rake helper libraries - -Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. - - -== test - Tests - -In the subfolder scanners/ are the scanners tests. -Each language has its own subfolder and sub-suite. - -Run with rake test. += CodeRay - folder structure + +== bench - Benchmarking system + +All benchmarking stuff goes here. + +Test inputs are stored in files named example.. +Test outputs go to bench/test.. + +Run bench/bench.rb to get a usage description. + +Run rake bench to perform an example benchmark. + + +== bin - Scripts + +Executional files for CodeRay. + +coderay:: The CodeRay executable. + +== demo - Demos and functional tests + +Demonstrational scripts to show of CodeRay's features. + +Run them as functional tests with rake test:demos. + + +== etc - Lots of stuff + +Some additional files for CodeRay, mainly graphics and Vim scripts. + + +== lib - CodeRay library code + +This is the base directory for the CodeRay library. + + +== rake_helpers - Rake helper libraries + +Some files to enhance Rake, including the Autumnal Rdoc template and some scripts. + + +== test - Tests + +In the subfolder scanners/ are the scanners tests. +Each language has its own subfolder and sub-suite. + +Run with rake test. diff --git a/Gemfile b/Gemfile index 6d3a176f..530c0e80 100644 --- a/Gemfile +++ b/Gemfile @@ -6,13 +6,13 @@ gemspec # Add dependencies to develop your gem here. # Include everything needed to run rake, tests, features, etc. group :development do - gem "bundler" - gem "rake" - gem "RedCloth", RUBY_PLATFORM == 'java' ? ">= 4.2.7" : ">= 4.0.3" - gem "term-ansicolor" - gem 'tins', '~> 1.6.0' - gem "shoulda-context" - gem "test-unit" - gem "json" if RUBY_VERSION < '1.9' - gem "rdoc" + gem 'bundler' + gem 'rake', RUBY_VERSION < '1.9' ? '~> 10.5' : '>= 10.5' + gem 'RedCloth', RUBY_PLATFORM == 'java' ? '= 4.2.9' : '>= 4.0.3' + gem 'term-ansicolor', RUBY_VERSION < '2.0' ? '~> 1.3.2' : '>= 1.3.2' + gem 'tins', RUBY_VERSION < '2.0' ? '~> 1.6.0' : '>= 1.6.0' + gem 'shoulda-context', RUBY_VERSION < '1.9' ? '= 1.2.1' : '>= 1.2.1' + gem 'test-unit', RUBY_VERSION < '1.9' ? '~> 2.0' : '>= 3.0' + gem 'json', '>= 1.8' if RUBY_VERSION < '1.9' + gem 'rdoc', RUBY_VERSION < '1.9' ? '~> 4.2.2' : '>= 4.2.2' end diff --git a/MIT-LICENSE.txt b/MIT-LICENSE similarity index 100% rename from MIT-LICENSE.txt rename to MIT-LICENSE diff --git a/bench/bench.rb b/bench/bench.rb index 92f9d07f..a47721e3 100644 --- a/bench/bench.rb +++ b/bench/bench.rb @@ -15,7 +15,7 @@ format = ARGV.fetch(1, 'html').downcase encoder = CodeRay.encoder(format) -size = ARGV.fetch(2, 1000).to_i * 1000 +size = ARGV.fetch(2, 3000).to_i * 1000 unless size.zero? data += data until data.size >= size data = data[0, size] @@ -23,14 +23,18 @@ size = data.size puts "encoding %d kB of #{lang} code to #{format}..." % [(size / 1000.0).round] -n = ARGV.fetch(3, 5).to_s[/\d+/].to_i +n = ARGV.fetch(3, 10).to_s[/\d+/].to_i require 'profile' if ARGV.include? '-p' +times = [] n.times do |i| time = Benchmark.realtime { encoder.encode(data, lang) } puts "run %d: %5.2f s, %4.0f kB/s" % [i + 1, time, size / time / 1000.0] + times << time end -STDIN.gets if ARGV.include? '-w' +times_sum = times.inject(0) { |time, sum| sum + time } +puts 'Average time: %5.2f s, %4.0f kB/s' % [times_sum / times.size, (size * n) / times_sum / 1000.0] +puts 'Best time: %5.2f s, %4.0f kB/s' % [times.min, size / times.min / 1000.0] __END__ Usage: diff --git a/coderay.gemspec b/coderay.gemspec index 328b94c1..50c195b5 100644 --- a/coderay.gemspec +++ b/coderay.gemspec @@ -24,8 +24,7 @@ Gem::Specification.new do |s| readme_file = 'README_INDEX.rdoc' - s.files = `git ls-files -- lib/* test/functional/* Rakefile #{readme_file} MIT-LICENSE`.split("\n") - s.test_files = `git ls-files -- test/functional/*`.split("\n") + s.files = `git ls-files -- lib/* #{readme_file} MIT-LICENSE`.split("\n") s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } s.require_paths = ['lib'] diff --git a/lib/coderay/encoders/encoder.rb b/lib/coderay/encoders/encoder.rb index fa5695d6..2baeedb6 100644 --- a/lib/coderay/encoders/encoder.rb +++ b/lib/coderay/encoders/encoder.rb @@ -146,7 +146,7 @@ def setup options end def get_output options - options[:out] || '' + options[:out] || ''.dup end # Append data.to_s to the output. Returns the argument. diff --git a/lib/coderay/encoders/html.rb b/lib/coderay/encoders/html.rb index 942b9c89..1b33e921 100644 --- a/lib/coderay/encoders/html.rb +++ b/lib/coderay/encoders/html.rb @@ -176,7 +176,7 @@ def setup options if options[:wrap] || options[:line_numbers] @real_out = @out - @out = '' + @out = ''.dup end @break_lines = (options[:break_lines] == true) @@ -314,7 +314,7 @@ def check_group_nesting name, kind end def break_lines text, style - reopen = '' + reopen = ''.dup @opened.each_with_index do |kind, index| reopen << (@span_for_kinds[index > 0 ? [kind, *@opened[0...index]] : kind] || '') end diff --git a/lib/coderay/scanners/cpp.rb b/lib/coderay/scanners/cpp.rb index e61f56f4..40aeb426 100644 --- a/lib/coderay/scanners/cpp.rb +++ b/lib/coderay/scanners/cpp.rb @@ -2,14 +2,14 @@ module CodeRay module Scanners # Scanner for C++. - # + # # Aliases: +cplusplus+, c++ class CPlusPlus < Scanner register_for :cpp file_extension 'cpp' title 'C++' - + #-- http://www.cppreference.com/wiki/keywords/start KEYWORDS = [ 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break', @@ -17,28 +17,30 @@ class CPlusPlus < Scanner 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else', 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new', 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return', - 'sizeof', 'static_cast', 'struct', 'switch', 'template', - 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', + 'sizeof', 'static_assert', 'static_cast', 'struct', 'switch', + 'template', 'throw', 'try', 'typedef', 'typeid', 'typename', 'union', 'while', 'xor', 'xor_eq', ] # :nodoc: - + PREDEFINED_TYPES = [ - 'bool', 'char', 'double', 'float', 'int', 'long', - 'short', 'signed', 'unsigned', 'wchar_t', 'string', + 'bool', 'char', 'char16_t', 'char32_t', 'double', 'float', + 'int', 'long', 'short', 'signed', 'unsigned', + 'wchar_t', 'string', ] # :nodoc: PREDEFINED_CONSTANTS = [ 'false', 'true', - 'EOF', 'NULL', + 'EOF', 'NULL', 'nullptr' ] # :nodoc: PREDEFINED_VARIABLES = [ 'this', ] # :nodoc: DIRECTIVES = [ - 'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator', - 'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void', - 'volatile', + 'alignas', 'alignof', 'auto', 'const', 'constexpr', 'decltype', 'explicit', + 'extern', 'final', 'friend', 'inline', 'mutable', 'noexcept', 'operator', + 'override', 'private', 'protected', 'public', 'register', 'static', + 'thread_local', 'using', 'virtual', 'void', 'volatile', ] # :nodoc: - + IDENT_KIND = WordList.new(:ident). add(KEYWORDS, :keyword). add(PREDEFINED_TYPES, :predefined_type). @@ -48,9 +50,9 @@ class CPlusPlus < Scanner ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x # :nodoc: UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x # :nodoc: - + protected - + def scan_tokens encoder, options state = :initial @@ -107,7 +109,7 @@ def scan_tokens encoder, options elsif match = scan(/\$/) encoder.text_token match, :ident - + elsif match = scan(/L?"/) encoder.begin_group :string if match[0] == ?L @@ -180,7 +182,7 @@ def scan_tokens encoder, options state = :initial end - + when :class_name_expected if match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x) encoder.text_token match, :class @@ -194,7 +196,7 @@ def scan_tokens encoder, options state = :initial end - + else raise_inspect 'Unknown state', encoder diff --git a/lib/coderay/scanners/haml.rb b/lib/coderay/scanners/haml.rb index 5433790a..d516ba9e 100644 --- a/lib/coderay/scanners/haml.rb +++ b/lib/coderay/scanners/haml.rb @@ -75,7 +75,7 @@ def scan_tokens encoder, options tag = false - if match = scan(/%[\w:]+\/?/) + if match = scan(/%[-\w:]+\/?/) encoder.text_token match, :tag # if match = scan(/( +)(.+)/) # encoder.text_token self[1], :space diff --git a/lib/coderay/scanners/java.rb b/lib/coderay/scanners/java.rb index 962154eb..982a796f 100644 --- a/lib/coderay/scanners/java.rb +++ b/lib/coderay/scanners/java.rb @@ -44,7 +44,7 @@ class Java < Scanner '"' => /[^\\"]+/, '/' => /[^\\\/]+/, } # :nodoc: - IDENT = /[a-zA-Z_][A-Za-z_0-9]*/ # :nodoc: + IDENT = RUBY_VERSION < '1.9' ? /[a-zA-Z_][A-Za-z_0-9]*/ : Regexp.new('[[[:alpha:]]_][[[:alnum:]]_]*') # :nodoc: protected diff --git a/lib/coderay/token_kinds.rb b/lib/coderay/token_kinds.rb old mode 100755 new mode 100644 diff --git a/lib/coderay/version.rb b/lib/coderay/version.rb index 7ea3f70c..f5e7a39d 100644 --- a/lib/coderay/version.rb +++ b/lib/coderay/version.rb @@ -1,3 +1,3 @@ module CodeRay - VERSION = '1.1.1' + VERSION = '1.1.2' end diff --git a/rake_tasks/benchmark.rake b/rake_tasks/benchmark.rake index 2e38b577..8edeffb0 100644 --- a/rake_tasks/benchmark.rake +++ b/rake_tasks/benchmark.rake @@ -1,6 +1,6 @@ desc 'Do a benchmark' task :benchmark do - ruby 'bench/bench.rb ruby html 3000' + ruby 'bench/bench.rb ruby html' end task :bench => :benchmark diff --git a/rake_tasks/test.rake b/rake_tasks/test.rake index b15b9993..1a23a5bc 100644 --- a/rake_tasks/test.rake +++ b/rake_tasks/test.rake @@ -2,7 +2,7 @@ namespace :test do desc 'run functional tests' task :functional do ruby './test/functional/suite.rb' - ruby './test/functional/for_redcloth.rb' + ruby './test/functional/for_redcloth.rb' unless (''.chop! rescue true) end desc 'run unit tests' diff --git a/test/functional/basic.rb b/test/functional/basic.rb old mode 100755 new mode 100644 diff --git a/test/functional/examples.rb b/test/functional/examples.rb old mode 100755 new mode 100644 diff --git a/test/functional/suite.rb b/test/functional/suite.rb old mode 100755 new mode 100644