From 22d6b9043538e8c5eecda606a6efb3d6f38ee407 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Tue, 18 Dec 2012 17:04:49 -0800 Subject: [PATCH 1/3] first pass at liquid scanner --- lib/coderay/helpers/file_type.rb | 1 + lib/coderay/scanners/amps.rb | 22 +++++++++++++ lib/coderay/scanners/liquid.rb | 53 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 lib/coderay/scanners/amps.rb create mode 100644 lib/coderay/scanners/liquid.rb diff --git a/lib/coderay/helpers/file_type.rb b/lib/coderay/helpers/file_type.rb index 637001b8..59bc73fc 100644 --- a/lib/coderay/helpers/file_type.rb +++ b/lib/coderay/helpers/file_type.rb @@ -96,6 +96,7 @@ def shebang filename 'java' => :java, 'js' => :java_script, 'json' => :json, + 'amps' => :amps, 'mab' => :ruby, 'pas' => :delphi, 'patch' => :diff, diff --git a/lib/coderay/scanners/amps.rb b/lib/coderay/scanners/amps.rb new file mode 100644 index 00000000..3a802ee3 --- /dev/null +++ b/lib/coderay/scanners/amps.rb @@ -0,0 +1,22 @@ +module CodeRay +module Scanners + +load :html + +class Amps < Scanner + + register_for :amps + title 'Amps Template' + + KINDS_NOT_LOC = HTML::KINDS_NOT_LOC + + AMPS_BLOCK = / + ({[{|%]) + (.*?) + ([%|}]}) + /xm + + AMPS_COMMENT_BLOCK = / + {%\s# + (.*?) + diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb new file mode 100644 index 00000000..27ca9761 --- /dev/null +++ b/lib/coderay/scanners/liquid.rb @@ -0,0 +1,53 @@ +module CodeRay +module Scanners + + load :html + + class Liquid < Scanner + + register_for :liquid + title 'Liquid Template' + + KINDS_NOT_LOC = HTML::KINDS_NOT_LOC + + LIQUID_BLOCK = / + ({[{|%]) + (.*?) + ([%|}]}) + / + + START_OF_LIQUID = /{{|{%/ + + protected + + def setup + @html_scanner = CodeRay.scanner :html, tokens: @tokens, keep_tokens: true, keep_state: true + @liquid_attribute_scanner = CodeRay.scanner :html, tokens: @tokens, keep_tokens: true, keep_stat: true + end + + def scan_tokens + until eos? + if (match = scan_until(/(?=#{START_OF_LIQUID})/o) || scan_reset) and not match.empty? + @html_scanner.tokenize match, tokens: encoder + elsif match = scan(/#{LIQUID_BLOCK}/o) + start_tag = self[1] + code = self[2] + end_tag = self[3] + + encoder.begin_group :inline + encoder.text_token start_tag, :inline_delimiter + + unless code.empty? + @liquid_attribute_scanner.tokenize code, tokens: encoder, state: :attribute + end + + encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? + encoder.end_group :inline + else + raise_inspect 'else-case reached!', encoder + end + end + end + end +end +end From 1a2c0b87370a12742570e7fafcb0c93fcbb2517c Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Tue, 18 Dec 2012 17:05:23 -0800 Subject: [PATCH 2/3] remove premature amps scanner --- lib/coderay/scanners/amps.rb | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 lib/coderay/scanners/amps.rb diff --git a/lib/coderay/scanners/amps.rb b/lib/coderay/scanners/amps.rb deleted file mode 100644 index 3a802ee3..00000000 --- a/lib/coderay/scanners/amps.rb +++ /dev/null @@ -1,22 +0,0 @@ -module CodeRay -module Scanners - -load :html - -class Amps < Scanner - - register_for :amps - title 'Amps Template' - - KINDS_NOT_LOC = HTML::KINDS_NOT_LOC - - AMPS_BLOCK = / - ({[{|%]) - (.*?) - ([%|}]}) - /xm - - AMPS_COMMENT_BLOCK = / - {%\s# - (.*?) - From dadbe8434652c0d7a191dc43cca785f2ab914052 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Wed, 19 Dec 2012 12:59:45 -0800 Subject: [PATCH 3/3] fails with actual liquid, not sure why --- lib/coderay/scanners/liquid.rb | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/coderay/scanners/liquid.rb b/lib/coderay/scanners/liquid.rb index 27ca9761..a0255712 100644 --- a/lib/coderay/scanners/liquid.rb +++ b/lib/coderay/scanners/liquid.rb @@ -12,7 +12,9 @@ class Liquid < Scanner LIQUID_BLOCK = / ({[{|%]) + \s (.*?) + \s ([%|}]}) / @@ -25,26 +27,26 @@ def setup @liquid_attribute_scanner = CodeRay.scanner :html, tokens: @tokens, keep_tokens: true, keep_stat: true end - def scan_tokens + def scan_tokens(tokens, options) until eos? - if (match = scan_until(/(?=#{START_OF_LIQUID})/o) || scan_reset) and not match.empty? - @html_scanner.tokenize match, tokens: encoder + if (match = scan_until(/(?=#{START_OF_LIQUID})/o) || scan_rest) and not match.empty? + @html_scanner.tokenize match, tokens: tokens elsif match = scan(/#{LIQUID_BLOCK}/o) start_tag = self[1] code = self[2] end_tag = self[3] - encoder.begin_group :inline - encoder.text_token start_tag, :inline_delimiter + tokens.begin_group :inline + tokens.text_token start_tag, :inline_delimiter unless code.empty? - @liquid_attribute_scanner.tokenize code, tokens: encoder, state: :attribute + @liquid_attribute_scanner.tokenize code, tokens: tokens, state: :attribute end - encoder.text_token end_tag, :inline_delimiter unless end_tag.empty? - encoder.end_group :inline + tokens.text_token end_tag, :inline_delimiter unless end_tag.empty? + tokens.end_group :inline else - raise_inspect 'else-case reached!', encoder + raise_inspect 'else-case reached!', tokens end end end