From 5c2bf6b3659f4b5ffc73ce16f03769aac175a07e Mon Sep 17 00:00:00 2001
From: schneems <richard.schneeman+foo@gmail.com>
Date: Tue, 7 Mar 2023 17:49:11 -0600
Subject: [PATCH 1/6] Handle alias already exists when debugging

When `tmp/alias` already exists, I'm now getting phantom folders in the directory pointing at older aliases which is distracting/confusing. By checking and removing that alias before symlinking we can prevent this strange behavior (possibly caused by newer Mac OS?).
---
 lib/syntax_suggest/api.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/syntax_suggest/api.rb b/lib/syntax_suggest/api.rb
index 5b725e1..ad8f641 100644
--- a/lib/syntax_suggest/api.rb
+++ b/lib/syntax_suggest/api.rb
@@ -91,7 +91,9 @@ def self.record_dir(dir)
     dir = Pathname(dir)
     dir.join(time).tap { |path|
       path.mkpath
-      FileUtils.ln_sf(time, dir.join("last"))
+      alias_dir = dir.join("last")
+      FileUtils.rm_rf(alias_dir) if alias_dir.exist?
+      FileUtils.ln_sf(time, alias_dir)
     }
   end
 

From dc18f9c84af9d35eb99f14b4eeb0e7471ef8128b Mon Sep 17 00:00:00 2001
From: schneems <richard.schneeman+foo@gmail.com>
Date: Wed, 8 Mar 2023 08:51:00 -0600
Subject: [PATCH 2/6] Preserve whitespace in front of comments

When removing comments I previously replaced them with a newline. This loses some context and may affect the order of the indent search which in turn affects the final result. By preserving whitespace in front of the comment, we preserve the "natural" indentation order of the line while also allowing the parser/lexer to see and join naturally consecutive (method chain) lines.

close #177
---
 lib/syntax_suggest/clean_document.rb    |  7 ++--
 lib/syntax_suggest/code_line.rb         |  6 ++--
 spec/integration/syntax_suggest_spec.rb | 27 +++++++++++++++
 spec/unit/clean_document_spec.rb        | 44 +++++++++++++------------
 spec/unit/code_line_spec.rb             |  1 +
 spec/unit/code_search_spec.rb           |  1 +
 6 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb
index b572189..08a465d 100644
--- a/lib/syntax_suggest/clean_document.rb
+++ b/lib/syntax_suggest/clean_document.rb
@@ -110,7 +110,7 @@ def to_s
       @document.join
     end
 
-    # Remove comments and whitespace only lines
+    # Remove comments
     #
     # replace with empty newlines
     #
@@ -156,8 +156,9 @@ def to_s
     #
     def clean_sweep(source:)
       source.lines.map do |line|
-        if line.match?(/^\s*(#[^{].*)?$/) # https://rubular.com/r/LLE10D8HKMkJvs
-          $/
+        if line.match?(/^\s*#([^{].*)?$/) # https://rubular.com/r/LLE10D8HKMkJvs
+          whitespace = /^(?<whitespace>\s*)#([^{].*)?$/.match(line).named_captures["whitespace"] || ""
+          whitespace + $/
         else
           line
         end
diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb
index dc738ab..d771a2c 100644
--- a/lib/syntax_suggest/code_line.rb
+++ b/lib/syntax_suggest/code_line.rb
@@ -48,11 +48,9 @@ def initialize(line:, index:, lex:)
       strip_line = line.dup
       strip_line.lstrip!
 
-      if strip_line.empty?
-        @empty = true
-        @indent = 0
+      if (@empty = strip_line.empty?)
+        @indent = line.length - 1 # Newline removed from strip_line is not "whitespace"
       else
-        @empty = false
         @indent = line.length - strip_line.length
       end
 
diff --git a/spec/integration/syntax_suggest_spec.rb b/spec/integration/syntax_suggest_spec.rb
index bb50faf..21c02ca 100644
--- a/spec/integration/syntax_suggest_spec.rb
+++ b/spec/integration/syntax_suggest_spec.rb
@@ -207,5 +207,32 @@ def bark
         > 4  end
       EOM
     end
+
+    it "comment inside of a method" do
+      source = <<~'EOM'
+        class Dog
+          def bark
+            # todo
+          end
+
+          def sit
+            print "sit"
+          end
+        end
+        end # extra end
+      EOM
+
+      io = StringIO.new
+      SyntaxSuggest.call(
+        io: io,
+        source: source
+      )
+      out = io.string
+      expect(out).to include(<<~EOM)
+        >  1  class Dog
+        >  9  end
+        > 10  end # extra end
+      EOM
+    end
   end
 end
diff --git a/spec/unit/clean_document_spec.rb b/spec/unit/clean_document_spec.rb
index 4fb79ef..bb26e2f 100644
--- a/spec/unit/clean_document_spec.rb
+++ b/spec/unit/clean_document_spec.rb
@@ -72,6 +72,25 @@ module SyntaxSuggest
       EOM
     end
 
+
+    it "joins multi-line chained methods when separated by comments" do
+      source = <<~EOM
+        User.
+          # comment
+          where(name: 'schneems').
+          # another comment
+          first
+      EOM
+
+      doc = CleanDocument.new(source: source).join_consecutive!
+      code_lines = doc.lines
+
+      expect(code_lines[0].to_s.count($/)).to eq(5)
+      code_lines[1..-1].each do |line|
+        expect(line.to_s.strip.length).to eq(0)
+      end
+    end
+
     it "helper method: take_while_including" do
       source = <<~EOM
         User
@@ -92,27 +111,10 @@ module SyntaxSuggest
           # yolo
       EOM
 
-      out = CleanDocument.new(source: source).lines.join
-      expect(out.to_s).to eq(<<~EOM)
-
-        puts "what"
-
-      EOM
-    end
-
-    it "whitespace: removes whitespace" do
-      source = "  \n" + <<~EOM
-        puts "what"
-      EOM
-
-      out = CleanDocument.new(source: source).lines.join
-      expect(out.to_s).to eq(<<~EOM)
-
-        puts "what"
-      EOM
-
-      expect(source.lines.first.to_s).to_not eq("\n")
-      expect(out.lines.first.to_s).to eq("\n")
+      lines = CleanDocument.new(source: source).lines
+      expect(lines[0].to_s).to eq($/)
+      expect(lines[1].to_s).to eq('puts "what"' + $/)
+      expect(lines[2].to_s).to eq('  ' + $/)
     end
 
     it "trailing slash: does not join trailing do" do
diff --git a/spec/unit/code_line_spec.rb b/spec/unit/code_line_spec.rb
index cc4fa48..d5b568f 100644
--- a/spec/unit/code_line_spec.rb
+++ b/spec/unit/code_line_spec.rb
@@ -48,6 +48,7 @@ def square(x) = x * x
       # Indicates line 1 can join 2, 2 can join 3, but 3 won't join it's next line
       expect(code_lines.map(&:ignore_newline_not_beg?)).to eq([true, true, false, false])
     end
+
     it "trailing if" do
       code_lines = CodeLine.from_source(<<~'EOM')
         puts "lol" if foo
diff --git a/spec/unit/code_search_spec.rb b/spec/unit/code_search_spec.rb
index f836ba3..9a8115c 100644
--- a/spec/unit/code_search_spec.rb
+++ b/spec/unit/code_search_spec.rb
@@ -338,6 +338,7 @@ def dog
             end
           EOM
           search.call
+          puts "done"
 
           expect(search.invalid_blocks.join).to eq(<<~'EOM')
             Foo.call do

From 13739c694622cbf36d9d02d16517a2fba30f6e3a Mon Sep 17 00:00:00 2001
From: schneems <richard.schneeman+foo@gmail.com>
Date: Wed, 8 Mar 2023 17:44:58 -0600
Subject: [PATCH 3/6] Fix sibling bug to #177

While #177 is reported as being caused by a comment, the underlying behavior is a problem due to the newline that we generated (from a comment). The prior commit fixed that problem by preserving whitespace before the comment. That guarantees that a block will form there from the frontier before it will be expanded there via a "neighbors" method. Since empty lines are valid ruby code, it will be hidden and be safe.

## Problem setup

This failure mode is not fixed by the prior commit, because the indentation is 0. To provide good results, we must make the algorithm less greedy. One heuristic/signal to follow is developer added newlines. If a developer puts a newline between code, it's more likely they're unrelated. For example:

```
port = rand(1000...9999)
stub_request(:any, "localhost:#{port}")

query = Cutlass::FunctionQuery.new(
  port: port
).call

expect(WebMock).to have_requested(:post, "localhost:#{port}").
  with(body: "{}")
```

This code is split into three chunks by the developer. Each are likely (but not guaranteed) to be intended to stand on their own (in terms of syntax). This behavior is good for scanning neighbors (same indent or higher) within a method, but bad for parsing neighbors across methods.

## Problem

Code is expanded to capture all neighbors, and then it decreases indent level which allows it to capture surrounding scope (think moving from within the method to also capturing the `def/end` definition. Once the indentation level has been increased, we go back to scanning neighbors, but now neighbors also contain keywords.

For example:

```
  1 def bark
  2
  3 end
  4
  5 def sit
  6 end
```

In this case if lines 4, 5, and 6 are in a block when it tries to expand neighbors it will expand up. If it stops after line 2 or 3 it may cause problems since there's a valid kw/end pair, but the block will be checked without it.

TLDR; It's good to stop scanning code after hitting a newline when you're in a method...it causes a problem scanning code between methods when everything inside of one of the methods is an empty line.

In this case it grabs the end on line 3 and since the problem was an extra end, the program now compiles correctly. It incorrectly assumes that the block it captured was causing the problem.

## Extra bit of context

One other technical detail is that after we've decided to stop scanning code for a new neighbor block expansion, we look around the block and grab any empty newlines. Basically adding empty newlines before of after a code block do not affect the parsing of that block.

## The fix

Since we know that this problem only happens when there's a newline inside of a method and we know this particular failure mode is due to having an invalid block (capturing an extra end, but not it's keyword) we have all the metadata we need to detect this scenario and correct it.

We know that the next line above our block must be code or empty (since we grabbed extra newlines). Same for code below it. We can count all the keywords and ends in the block. If they are balanced, it's likely (but not guaranteed) we formed the block correctly. If they're imbalanced, look above or below (depending on the nature of the imbalance), check to see if adding that line would balance the count.

This concept of balance and "leaning" comes from work in https://github.com/ruby/syntax_suggest/pull/152 and has proven useful, but not been formally introduced into the main branch.

## Outcome

Adding this extra check introduced no regressions and fixed the test case. It might be possible there's a mirror or similar problem that we're not handling. That will come out in time. It might also be possible that this causes a worse case in some code not under test. That too would come out in time.

One other possible concern to adding logic in this area (which is a hot codepath), is performance. This extra count check will be performed for every block. In general the two most helpful performance strategies I've found are reducing total number of blocks (therefore reducing overall N internal iterations) and making better matches (the parser to determine if a close block is valid or not is a major bottleneck. If we can split valid code into valid blocks, then it's only evaluated by the parser once, where as invalid code must be continuously re-checked by the parser until it becomes valid, or is determined to be the cause of the core problem.

This extra logic should very rarely result in a change, but when it does it should tend to produce slightly larger blocks (by one line) and more accurate blocks.

Informally it seems to have no impact on performance:

``
This branch:
DEBUG_DISPLAY=1 bundle exec rspec spec/ --format=failures  3.01s user 1.62s system 113% cpu 4.076 total
```

```
On main:
DEBUG_DISPLAY=1 bundle exec rspec spec/ --format=failures  3.02s user 1.64s system 113% cpu 4.098 total
```
---
 lib/syntax_suggest/around_block_scan.rb    | 51 +++++++++++-
 lib/syntax_suggest/block_expand.rb         | 97 ++++++++++++++++++++--
 lib/syntax_suggest/capture_code_context.rb |  1 -
 spec/integration/syntax_suggest_spec.rb    | 25 ++++++
 spec/unit/around_block_scan_spec.rb        |  4 +-
 spec/unit/block_expand_spec.rb             | 30 +++++++
 spec/unit/code_search_spec.rb              |  1 -
 7 files changed, 197 insertions(+), 12 deletions(-)

diff --git a/lib/syntax_suggest/around_block_scan.rb b/lib/syntax_suggest/around_block_scan.rb
index 2a57d1b..4793c3b 100644
--- a/lib/syntax_suggest/around_block_scan.rb
+++ b/lib/syntax_suggest/around_block_scan.rb
@@ -61,7 +61,6 @@ def stop_after_kw
 
     def scan_while
       stop_next = false
-
       kw_count = 0
       end_count = 0
       index = before_lines.reverse_each.take_while do |line|
@@ -166,7 +165,55 @@ def on_falling_indent
       end
     end
 
-    def scan_neighbors
+    # Scanning is intentionally conservative because
+    # we have no way of rolling back an agressive block (at this time)
+    #
+    # If a block was stopped for some trivial reason, (like an empty line)
+    # but the next line would have caused it to be balanced then we
+    # can check that condition and grab just one more line either up or
+    # down.
+    #
+    # For example, below if we're scanning up, line 2 might cause
+    # the scanning to stop. This is because empty lines might
+    # denote logical breaks where the user intended to chunk code
+    # which is a good place to stop and check validity. Unfortunately
+    # it also means we might have a "dangling" keyword or end.
+    #
+    #   1 def bark
+    #   2
+    #   3 end
+    #
+    # If lines 2 and 3 are in the block, then when this method is
+    # run it would see it is unbalanced, but that acquiring line 1
+    # would make it balanced, so that's what it does.
+    def lookahead_balance_one_line
+      kw_count = 0
+      end_count = 0
+      lines.each do |line|
+        kw_count += 1 if line.is_kw?
+        end_count += 1 if line.is_end?
+      end
+
+      return self if kw_count == end_count # nothing to balance
+
+      # More ends than keywords, check if we can balance expanding up
+      if (end_count - kw_count) == 1 && next_up
+        return self unless next_up.is_kw?
+        return self unless next_up.indent >= @orig_indent
+
+        @before_index = next_up.index
+
+      # More keywords than ends, check if we can balance by expanding down
+      elsif (kw_count - end_count) == 1 && next_down
+        return self unless next_down.is_end?
+        return self unless next_down.indent >= @orig_indent
+
+        @after_index = next_down.index
+      end
+      self
+    end
+
+    def scan_neighbors_not_empty
       scan_while { |line| line.not_empty? && line.indent >= @orig_indent }
     end
 
diff --git a/lib/syntax_suggest/block_expand.rb b/lib/syntax_suggest/block_expand.rb
index 396b2c3..8142e74 100644
--- a/lib/syntax_suggest/block_expand.rb
+++ b/lib/syntax_suggest/block_expand.rb
@@ -35,14 +35,31 @@ def initialize(code_lines:)
       @code_lines = code_lines
     end
 
+    # Main interface. Expand current indentation, before
+    # expanding to a lower indentation
     def call(block)
       if (next_block = expand_neighbors(block))
-        return next_block
+        next_block
+      else
+        expand_indent(block)
       end
-
-      expand_indent(block)
     end
 
+    # Expands code to the next lowest indentation
+    #
+    # For example:
+    #
+    #   1 def dog
+    #   2   print "dog"
+    #   3 end
+    #
+    # If a block starts on line 2 then it has captured all it's "neighbors" (code at
+    # the same indentation or higher). To continue expanding, this block must capture
+    # lines one and three which are at a different indentation level.
+    #
+    # This method allows fully expanded blocks to decrease their indentation level (so
+    # they can expand to capture more code up and down). It does this conservatively
+    # as there's no undo (currently).
     def expand_indent(block)
       AroundBlockScan.new(code_lines: @code_lines, block: block)
         .skip(:hidden?)
@@ -51,14 +68,82 @@ def expand_indent(block)
         .code_block
     end
 
+    # A neighbor is code that is at or above the current indent line.
+    #
+    # First we build a block with all neighbors. If we can't go further
+    # then we decrease the indentation threshold and expand via indentation
+    # i.e. `expand_indent`
+    #
+    # Handles two general cases.
+    #
+    # ## Case #1: Check code inside of methods/classes/etc.
+    #
+    # It's important to note, that not everything in a given indentation level can be parsed
+    # as valid code even if it's part of valid code. For example:
+    #
+    #   1 hash = {
+    #   2   name: "richard",
+    #   3   dog: "cinco",
+    #   4 }
+    #
+    # In this case lines 2 and 3 will be neighbors, but they're invalid until `expand_indent`
+    # is called on them.
+    #
+    # When we are adding code within a method or class (at the same indentation level),
+    # use the empty lines to denote the programmer intended logical chunks.
+    # Stop and check each one. For example:
+    #
+    #   1 def dog
+    #   2   print "dog"
+    #   3
+    #   4   hash = {
+    #   5 end
+    #
+    # If we did not stop parsing at empty newlines then the block might mistakenly grab all
+    # the contents (lines 2, 3, and 4) and report them as being problems, instead of only
+    # line 4.
+    #
+    # ## Case #2: Expand/grab other logical blocks
+    #
+    # Once the search algorithm has converted all lines into blocks at a given indentation
+    # it will then `expand_indent`. Once the blocks that generates are expanded as neighbors
+    # we then begin seeing neighbors being other logical blocks i.e. a block's neighbors
+    # may be another method or class (something with keywords/ends).
+    #
+    # For example:
+    #
+    #   1 def bark
+    #   2
+    #   3 end
+    #   4
+    #   5 def sit
+    #   6 end
+    #
+    # In this case if lines 4, 5, and 6 are in a block when it tries to expand neighbors
+    # it will expand up. If it stops after line 2 or 3 it may cause problems since there's a
+    # valid kw/end pair, but the block will be checked without it.
+    #
+    # We try to resolve this edge case with `lookahead_balance_one_line` below.
     def expand_neighbors(block)
-      expanded_lines = AroundBlockScan.new(code_lines: @code_lines, block: block)
+      neighbors = AroundBlockScan.new(code_lines: @code_lines, block: block)
         .skip(:hidden?)
         .stop_after_kw
-        .scan_neighbors
-        .scan_while { |line| line.empty? } # Slurp up empties
+        .scan_neighbors_not_empty
+
+      # Slurp up empties
+      with_empties = neighbors
+        .scan_while { |line| line.empty? }
+
+      # If next line is kw and it will balance us, take it
+      expanded_lines = with_empties
+        .lookahead_balance_one_line
         .lines
 
+      # Don't allocate a block if it won't be used
+      #
+      # If nothing was taken, return nil to indicate that status
+      # used in `def call` to determine if
+      # we need to expand up/out (`expand_indent`)
       if block.lines == expanded_lines
         nil
       else
diff --git a/lib/syntax_suggest/capture_code_context.rb b/lib/syntax_suggest/capture_code_context.rb
index 7d6a550..547072e 100644
--- a/lib/syntax_suggest/capture_code_context.rb
+++ b/lib/syntax_suggest/capture_code_context.rb
@@ -76,7 +76,6 @@ def call
     #     end
     #   end
     #
-    #
     def capture_falling_indent(block)
       AroundBlockScan.new(
         block: block,
diff --git a/spec/integration/syntax_suggest_spec.rb b/spec/integration/syntax_suggest_spec.rb
index 21c02ca..e961737 100644
--- a/spec/integration/syntax_suggest_spec.rb
+++ b/spec/integration/syntax_suggest_spec.rb
@@ -234,5 +234,30 @@ def sit
         > 10  end # extra end
       EOM
     end
+
+    it "space inside of a method" do
+      source = <<~'EOM'
+        class Dog       #  1
+          def bark      #  2
+
+          end           #  4
+
+          def sit       #  6
+            print "sit" #  7
+          end           #  8
+        end             #  9
+        end # extra end
+      EOM
+
+      io = StringIO.new
+      SyntaxSuggest.call(
+        io: io,
+        source: source
+      )
+      out = io.string
+      expect(out).to include(<<~EOM)
+        > 10  end # extra end
+      EOM
+    end
   end
 end
diff --git a/spec/unit/around_block_scan_spec.rb b/spec/unit/around_block_scan_spec.rb
index 6053c39..be1c3a4 100644
--- a/spec/unit/around_block_scan_spec.rb
+++ b/spec/unit/around_block_scan_spec.rb
@@ -13,7 +13,7 @@ module SyntaxSuggest
       code_lines = CodeLine.from_source(source)
       block = CodeBlock.new(lines: code_lines[1])
       expand = AroundBlockScan.new(code_lines: code_lines, block: block)
-        .scan_neighbors
+        .scan_neighbors_not_empty
 
       expect(expand.code_block.to_s).to eq(source)
       expand.scan_while { |line| false }
@@ -151,7 +151,7 @@ def foo
       expand = AroundBlockScan.new(code_lines: code_lines, block: block)
       expand.skip(:empty?)
       expand.skip(:hidden?)
-      expand.scan_neighbors
+      expand.scan_neighbors_not_empty
 
       expect(expand.code_block.to_s).to eq(<<~EOM.indent(4))
 
diff --git a/spec/unit/block_expand_spec.rb b/spec/unit/block_expand_spec.rb
index ba0b045..4f93210 100644
--- a/spec/unit/block_expand_spec.rb
+++ b/spec/unit/block_expand_spec.rb
@@ -4,6 +4,36 @@
 
 module SyntaxSuggest
   RSpec.describe BlockExpand do
+    it "empty line in methods" do
+      source_string = <<~EOM
+        class Dog        # index 0
+          def bark       # index 1
+
+          end            # index 3
+
+          def sit        # index 5
+            print "sit"  # index 6
+          end            # index 7
+        end              # index 8
+        end # extra end
+      EOM
+
+      code_lines = code_line_array(source_string)
+
+      sit = code_lines[4..7]
+      sit.each(&:mark_invisible)
+
+      block = CodeBlock.new(lines: sit)
+      expansion = BlockExpand.new(code_lines: code_lines)
+      block = expansion.expand_neighbors(block)
+
+      expect(block.to_s).to eq(<<~EOM.indent(2))
+          def bark       # index 1
+
+          end            # index 3
+      EOM
+    end
+
     it "captures multiple empty and hidden lines" do
       source_string = <<~EOM
         def foo
diff --git a/spec/unit/code_search_spec.rb b/spec/unit/code_search_spec.rb
index 9a8115c..f836ba3 100644
--- a/spec/unit/code_search_spec.rb
+++ b/spec/unit/code_search_spec.rb
@@ -338,7 +338,6 @@ def dog
             end
           EOM
           search.call
-          puts "done"
 
           expect(search.invalid_blocks.join).to eq(<<~'EOM')
             Foo.call do

From cecd12292cc9a62e955c4893f15e928be83c6090 Mon Sep 17 00:00:00 2001
From: schneems <richard.schneeman+foo@gmail.com>
Date: Wed, 8 Mar 2023 18:30:33 -0600
Subject: [PATCH 4/6] Add comments and refactor AroundBlockScan methods

---
 lib/syntax_suggest/around_block_scan.rb       | 136 ++++++++++++++++--
 lib/syntax_suggest/block_expand.rb            |   4 +-
 .../parse_blocks_from_indent_line.rb          |   4 +-
 spec/integration/syntax_suggest_spec.rb       |  52 -------
 spec/unit/around_block_scan_spec.rb           |   4 +-
 5 files changed, 127 insertions(+), 73 deletions(-)

diff --git a/lib/syntax_suggest/around_block_scan.rb b/lib/syntax_suggest/around_block_scan.rb
index 4793c3b..656246c 100644
--- a/lib/syntax_suggest/around_block_scan.rb
+++ b/lib/syntax_suggest/around_block_scan.rb
@@ -38,35 +38,64 @@ def initialize(code_lines:, block:)
       @before_array = []
       @stop_after_kw = false
 
-      @skip_hidden = false
-      @skip_empty = false
+      @force_add_hidden = false
+      @force_add_empty = false
     end
 
-    def skip(name)
-      case name
-      when :hidden?
-        @skip_hidden = true
-      when :empty?
-        @skip_empty = true
-      else
-        raise "Unsupported skip #{name}"
-      end
+    # When using this flag, `scan_while` will
+    # bypass the block it's given and always add a
+    # line that responds truthy to `CodeLine#hidden?`
+    #
+    # Lines are hidden when they've been evaluated by
+    # the parser as part of a block and found to contain
+    # valid code.
+    def force_add_hidden
+      @force_add_hidden = true
+      self
+    end
+
+    # When using this flag, `scan_while` will
+    # bypass the block it's given and always add a
+    # line that responds truthy to `CodeLine#empty?`
+    #
+    # Empty lines contain no code, only whitespace such
+    # as leading spaces a newline.
+    def force_add_empty
+      @force_add_empty = true
       self
     end
 
+    # Tells `scan_while` to look for mismatched keyword/end-s
+    #
+    # When scanning up, if we see more keywords then end-s it will
+    # stop. This might happen when scanning outside of a method body.
+    # the first scan line up would be a keyword and this setting would
+    # trigger a stop.
+    #
+    # When scanning down, stop if there are more end-s than keywords.
     def stop_after_kw
       @stop_after_kw = true
       self
     end
 
+    # Main work method
+    #
+    # The scan_while method takes a block that yields lines above and
+    # below the block. If the yield returns true, the @before_index
+    # or @after_index are modified to include the matched line.
+    #
+    # In addition to yielding individual lines, the internals of this
+    # object give a mini DSL to handle common situations such as
+    # stopping if we've found a keyword/end mis-match in one direction
+    # or the other.
     def scan_while
       stop_next = false
       kw_count = 0
       end_count = 0
       index = before_lines.reverse_each.take_while do |line|
         next false if stop_next
-        next true if @skip_hidden && line.hidden?
-        next true if @skip_empty && line.empty?
+        next true if @force_add_hidden && line.hidden?
+        next true if @force_add_empty && line.empty?
 
         kw_count += 1 if line.is_kw?
         end_count += 1 if line.is_end?
@@ -86,8 +115,8 @@ def scan_while
       end_count = 0
       index = after_lines.take_while do |line|
         next false if stop_next
-        next true if @skip_hidden && line.hidden?
-        next true if @skip_empty && line.empty?
+        next true if @force_add_hidden && line.hidden?
+        next true if @force_add_empty && line.empty?
 
         kw_count += 1 if line.is_kw?
         end_count += 1 if line.is_end?
@@ -104,6 +133,33 @@ def scan_while
       self
     end
 
+    # Shows surrounding kw/end pairs
+    #
+    # The purpose of showing these extra pairs is due to cases
+    # of ambiguity when only one visible line is matched.
+    #
+    # For example:
+    #
+    #     1  class Dog
+    #     2    def bark
+    #     4    def eat
+    #     5    end
+    #     6  end
+    #
+    # In this case either line 2 could be missing an `end` or
+    # line 4 was an extra line added by mistake (it happens).
+    #
+    # When we detect the above problem it shows the issue
+    # as only being on line 2
+    #
+    #     2    def bark
+    #
+    # Showing "neighbor" keyword pairs gives extra context:
+    #
+    #     2    def bark
+    #     4    def eat
+    #     5    end
+    #
     def capture_neighbor_context
       lines = []
       kw_count = 0
@@ -145,6 +201,20 @@ def capture_neighbor_context
       lines
     end
 
+    # Shows the context around code provided by "falling" indentation
+    #
+    # Converts:
+    #
+    #       it "foo" do
+    #
+    # into:
+    #
+    #   class OH
+    #     def hello
+    #       it "foo" do
+    #     end
+    #   end
+    #
     def on_falling_indent
       last_indent = @orig_indent
       before_lines.reverse_each do |line|
@@ -213,18 +283,31 @@ def lookahead_balance_one_line
       self
     end
 
+    # Finds code lines at the same or greater indentation and adds them
+    # to the block
     def scan_neighbors_not_empty
       scan_while { |line| line.not_empty? && line.indent >= @orig_indent }
     end
 
+    # Returns the next line to be scanned above the current block.
+    # Returns `nil` if at the top of the document already
     def next_up
       @code_lines[before_index.pred]
     end
 
+    # Returns the next line to be scanned below the current block.
+    # Returns `nil` if at the bottom of the document already
     def next_down
       @code_lines[after_index.next]
     end
 
+    # Scan blocks based on indentation of next line above/below block
+    #
+    # Determines indentaion of the next line above/below the current block.
+    #
+    # Normally this is called when a block has expanded to capture all "neighbors"
+    # at the same (or greater) indentation and needs to expand out. For example
+    # the `def/end` lines surrounding a method.
     def scan_adjacent_indent
       before_after_indent = []
       before_after_indent << (next_up&.indent || 0)
@@ -236,6 +319,16 @@ def scan_adjacent_indent
       self
     end
 
+    # TODO: Doc or delete
+    #
+    # I don't remember why this is needed, but it's called in code_context.
+    # It's related to the implementation of `capture_neighbor_context` somehow
+    # and that display improvement is only triggered when there's one visible line
+    #
+    # I think the primary purpose is to not include the current line in the
+    # logic evaluation of `capture_neighbor_context`. If that's true, then
+    # we should fix that method to handle this logic instead of only using
+    # it in one place and together.
     def start_at_next_line
       before_index
       after_index
@@ -244,26 +337,39 @@ def start_at_next_line
       self
     end
 
+    # Return the currently matched lines as a `CodeBlock`
+    #
+    # When a `CodeBlock` is created it will gather metadata about
+    # itself, so this is not a free conversion. Avoid allocating
+    # more CodeBlock's than needed
     def code_block
       CodeBlock.new(lines: lines)
     end
 
+    # Returns the lines matched by the current scan as an
+    # array of CodeLines
     def lines
       @code_lines[before_index..after_index]
     end
 
+    # Gives the index of the first line currently scanned
     def before_index
       @before_index ||= @orig_before_index
     end
 
+    # Gives the index of the last line currently scanned
     def after_index
       @after_index ||= @orig_after_index
     end
 
+    # Returns an array of all the CodeLines that exist before
+    # the currently scanned block
     private def before_lines
       @code_lines[0...before_index] || []
     end
 
+    # Returns an array of all the CodeLines that exist after
+    # the currently scanned block
     private def after_lines
       @code_lines[after_index.next..-1] || []
     end
diff --git a/lib/syntax_suggest/block_expand.rb b/lib/syntax_suggest/block_expand.rb
index 8142e74..8431d15 100644
--- a/lib/syntax_suggest/block_expand.rb
+++ b/lib/syntax_suggest/block_expand.rb
@@ -62,7 +62,7 @@ def call(block)
     # as there's no undo (currently).
     def expand_indent(block)
       AroundBlockScan.new(code_lines: @code_lines, block: block)
-        .skip(:hidden?)
+        .force_add_hidden
         .stop_after_kw
         .scan_adjacent_indent
         .code_block
@@ -126,7 +126,7 @@ def expand_indent(block)
     # We try to resolve this edge case with `lookahead_balance_one_line` below.
     def expand_neighbors(block)
       neighbors = AroundBlockScan.new(code_lines: @code_lines, block: block)
-        .skip(:hidden?)
+        .force_add_hidden
         .stop_after_kw
         .scan_neighbors_not_empty
 
diff --git a/lib/syntax_suggest/parse_blocks_from_indent_line.rb b/lib/syntax_suggest/parse_blocks_from_indent_line.rb
index d107173..241ed6a 100644
--- a/lib/syntax_suggest/parse_blocks_from_indent_line.rb
+++ b/lib/syntax_suggest/parse_blocks_from_indent_line.rb
@@ -36,8 +36,8 @@ def initialize(code_lines:)
     # Builds blocks from bottom up
     def each_neighbor_block(target_line)
       scan = AroundBlockScan.new(code_lines: code_lines, block: CodeBlock.new(lines: target_line))
-        .skip(:empty?)
-        .skip(:hidden?)
+        .force_add_empty
+        .force_add_hidden
         .scan_while { |line| line.indent >= target_line.indent }
 
       neighbors = scan.code_block.lines
diff --git a/spec/integration/syntax_suggest_spec.rb b/spec/integration/syntax_suggest_spec.rb
index e961737..bb50faf 100644
--- a/spec/integration/syntax_suggest_spec.rb
+++ b/spec/integration/syntax_suggest_spec.rb
@@ -207,57 +207,5 @@ def bark
         > 4  end
       EOM
     end
-
-    it "comment inside of a method" do
-      source = <<~'EOM'
-        class Dog
-          def bark
-            # todo
-          end
-
-          def sit
-            print "sit"
-          end
-        end
-        end # extra end
-      EOM
-
-      io = StringIO.new
-      SyntaxSuggest.call(
-        io: io,
-        source: source
-      )
-      out = io.string
-      expect(out).to include(<<~EOM)
-        >  1  class Dog
-        >  9  end
-        > 10  end # extra end
-      EOM
-    end
-
-    it "space inside of a method" do
-      source = <<~'EOM'
-        class Dog       #  1
-          def bark      #  2
-
-          end           #  4
-
-          def sit       #  6
-            print "sit" #  7
-          end           #  8
-        end             #  9
-        end # extra end
-      EOM
-
-      io = StringIO.new
-      SyntaxSuggest.call(
-        io: io,
-        source: source
-      )
-      out = io.string
-      expect(out).to include(<<~EOM)
-        > 10  end # extra end
-      EOM
-    end
   end
 end
diff --git a/spec/unit/around_block_scan_spec.rb b/spec/unit/around_block_scan_spec.rb
index be1c3a4..88d973e 100644
--- a/spec/unit/around_block_scan_spec.rb
+++ b/spec/unit/around_block_scan_spec.rb
@@ -149,8 +149,8 @@ def foo
 
       block = CodeBlock.new(lines: code_lines[3])
       expand = AroundBlockScan.new(code_lines: code_lines, block: block)
-      expand.skip(:empty?)
-      expand.skip(:hidden?)
+      expand.force_add_empty
+      expand.force_add_hidden
       expand.scan_neighbors_not_empty
 
       expect(expand.code_block.to_s).to eq(<<~EOM.indent(4))

From f94fd1fdb995141f4911655a58a2c133d0b06c09 Mon Sep 17 00:00:00 2001
From: schneems <richard.schneeman+foo@gmail.com>
Date: Thu, 9 Mar 2023 09:18:48 -0600
Subject: [PATCH 5/6] Changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f8f9f29..0354972 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
 ## HEAD (unreleased)
 
+- Output improvement: Handle methods with only newlines or comments in them (https://github.com/ruby/syntax_suggest/pull/179)
 - No longer shows the detail of monkey patch as the document (https://github.com/ruby/syntax_suggest/pull/174)
 - Drop CI for Ruby 3.2.0-rc1, now that 3.2.0 is available (https://github.com/ruby/syntax_suggest/pull/172)
 

From eacf9e687f002e43b835e52b73998dcd4c358d0e Mon Sep 17 00:00:00 2001
From: schneems <richard.schneeman+foo@gmail.com>
Date: Thu, 9 Mar 2023 14:13:46 -0600
Subject: [PATCH 6/6] Rollback comment indentation behavior

Originally I fixed #177 by making the process of comment removal indentation aware. The next commit is the more general fix and means we don't need to carry that additional logic/overhead.

Also: Update syntax via linter
---
 lib/syntax_suggest/clean_document.rb | 7 ++++---
 lib/syntax_suggest/code_line.rb      | 6 +++---
 spec/unit/block_expand_spec.rb       | 4 ++--
 spec/unit/clean_document_spec.rb     | 3 +--
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb
index 08a465d..2c26061 100644
--- a/lib/syntax_suggest/clean_document.rb
+++ b/lib/syntax_suggest/clean_document.rb
@@ -155,10 +155,11 @@ def to_s
     #    ).to eq(2)
     #
     def clean_sweep(source:)
+      # Match comments, but not HEREDOC strings with #{variable} interpolation
+      # https://rubular.com/r/HPwtW9OYxKUHXQ
       source.lines.map do |line|
-        if line.match?(/^\s*#([^{].*)?$/) # https://rubular.com/r/LLE10D8HKMkJvs
-          whitespace = /^(?<whitespace>\s*)#([^{].*)?$/.match(line).named_captures["whitespace"] || ""
-          whitespace + $/
+        if line.match?(/^\s*#([^{].*|)$/)
+          $/
         else
           line
         end
diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb
index d771a2c..a20f34a 100644
--- a/lib/syntax_suggest/code_line.rb
+++ b/lib/syntax_suggest/code_line.rb
@@ -48,10 +48,10 @@ def initialize(line:, index:, lex:)
       strip_line = line.dup
       strip_line.lstrip!
 
-      if (@empty = strip_line.empty?)
-        @indent = line.length - 1 # Newline removed from strip_line is not "whitespace"
+      @indent = if (@empty = strip_line.empty?)
+        line.length - 1 # Newline removed from strip_line is not "whitespace"
       else
-        @indent = line.length - strip_line.length
+        line.length - strip_line.length
       end
 
       set_kw_end
diff --git a/spec/unit/block_expand_spec.rb b/spec/unit/block_expand_spec.rb
index 4f93210..5cff736 100644
--- a/spec/unit/block_expand_spec.rb
+++ b/spec/unit/block_expand_spec.rb
@@ -28,9 +28,9 @@ def sit        # index 5
       block = expansion.expand_neighbors(block)
 
       expect(block.to_s).to eq(<<~EOM.indent(2))
-          def bark       # index 1
+        def bark       # index 1
 
-          end            # index 3
+        end            # index 3
       EOM
     end
 
diff --git a/spec/unit/clean_document_spec.rb b/spec/unit/clean_document_spec.rb
index bb26e2f..25a62e4 100644
--- a/spec/unit/clean_document_spec.rb
+++ b/spec/unit/clean_document_spec.rb
@@ -72,7 +72,6 @@ module SyntaxSuggest
       EOM
     end
 
-
     it "joins multi-line chained methods when separated by comments" do
       source = <<~EOM
         User.
@@ -114,7 +113,7 @@ module SyntaxSuggest
       lines = CleanDocument.new(source: source).lines
       expect(lines[0].to_s).to eq($/)
       expect(lines[1].to_s).to eq('puts "what"' + $/)
-      expect(lines[2].to_s).to eq('  ' + $/)
+      expect(lines[2].to_s).to eq($/)
     end
 
     it "trailing slash: does not join trailing do" do