Patch indentation with -indent -rewrite

Ensure indentation is correct when removing braces.

If the first indentation of the region is greater than the indentation
of the enclosing region, we use it to indent the whole region.
Otherwise we use the incremented indentation of the enclosing region.

```scala
  def foo = {
        x // we replicate indentation of x downward in region
    y
  }
```

```scala
  def foo = {
x // indentation of x is incorrect, we increment enclosing indentation
  y
  }
```

A bigger indentation than the required one is permitted except
just after a closing brace.

```scala
def bar = {
  x
    .toString // permitted indentation
  def foo = {
  }
    bar //  must be unindented, to not fall into the body of foo
}
```

Author: adpi2

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -60,16 +60,19 @@ object Scanners {
     /** the base of a number */
     var base: Int = 0
 
-    def copyFrom(td: TokenData): Unit = {
+    def copyFrom(td: TokenData): this.type = {
       this.token = td.token
       this.offset = td.offset
       this.lastOffset = td.lastOffset
       this.lineOffset = td.lineOffset
       this.name = td.name
       this.strVal = td.strVal
       this.base = td.base
+      this
     }
 
+    def saveCopy: TokenData = newTokenData.copyFrom(this)
+
     def isNewLine = token == NEWLINE || token == NEWLINES
     def isStatSep = isNewLine || token == SEMI
     def isIdent = token == IDENTIFIER || token == BACKQUOTED_IDENT
@@ -86,12 +89,14 @@ object Scanners {
 
     def isOperator =
       token == BACKQUOTED_IDENT
-      || token == IDENTIFIER && isOperatorPart(name(name.length - 1))
+      || token == IDENTIFIER && isOperatorPart(name.last)
 
     def isArrow =
       token == ARROW || token == CTXARROW
   }
 
+  def newTokenData: TokenData = new TokenData {}
+
   abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData {
     val buf: Array[Char] = source.content
     def nextToken(): Unit
@@ -264,11 +269,10 @@ object Scanners {
       if (idx >= 0 && idx <= lastKeywordStart) handleMigration(kwArray(idx))
       else IDENTIFIER
 
-    def newTokenData: TokenData = new TokenData {}
-
     /** We need one token lookahead and one token history
      */
     val next = newTokenData
+    val last = newTokenData
     private val prev = newTokenData
 
     /** The current region. This is initially an Indented region with zero indentation width. */
@@ -385,6 +389,7 @@ object Scanners {
     /** Produce next token, filling TokenData fields of Scanner.
      */
     def nextToken(): Unit =
+      last.copyFrom(this)
       val lastToken = token
       val lastName = name
       adjustSepRegions(lastToken)
@@ -433,7 +438,7 @@ object Scanners {
         // in backticks and is a binary operator. Hence, `x` is not classified as a
         // leading infix operator.
         def assumeStartsExpr(lexeme: TokenData) =
-          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONeol)
+          (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONfollow)
           && (!lexeme.isOperator || nme.raw.isUnary(lexeme.name))
         val lookahead = LookaheadScanner()
         lookahead.allowLeadingInfixOperators = false
@@ -483,7 +488,7 @@ object Scanners {
             if (nextChar == ch)
               recur(idx - 1, ch, n + 1, k)
             else {
-              val k1: IndentWidth => IndentWidth = if (n == 0) k else Conc(_, Run(ch, n))
+              val k1: IndentWidth => IndentWidth = if (n == 0) k else iw => k(Conc(iw, Run(ch, n)))
               recur(idx - 1, nextChar, 1, k1)
             }
           else recur(idx - 1, ' ', 0, identity)
@@ -523,7 +528,7 @@ object Scanners {
      *
      *      The following tokens can start an indentation region:
      *
-     *         :  =  =>  <-  if  then  else  while  do  try  catch  
+     *         :  =  =>  <-  if  then  else  while  do  try  catch
      *         finally  for  yield  match  throw  return  with
      *
      *      Inserting an INDENT starts a new indentation region with the indentation of the current
@@ -638,7 +643,8 @@ object Scanners {
             currentRegion.knownWidth = nextWidth
         else if (lastWidth != nextWidth)
           val lw = lastWidth
-          errorButContinue(spaceTabMismatchMsg(lw, nextWidth))
+          val msg = spaceTabMismatchMsg(lw, nextWidth)
+          if rewriteToIndent then report.warning(msg) else errorButContinue(msg)
       if token != OUTDENT then
         handleNewIndentWidth(currentRegion, _.otherIndentWidths += nextWidth)
       if next.token == EMPTY then
@@ -1266,6 +1272,7 @@ object Scanners {
               putChar(ch) ; nextRawChar()
               loopRest()
             else
+              next.lineOffset = if next.lastOffset < lineStartOffset then lineStartOffset else -1
               finishNamedToken(IDENTIFIER, target = next)
           end loopRest
           setStrVal()
@@ -1312,10 +1319,10 @@ object Scanners {
       }
     end getStringPart
 
-    private def fetchStringPart(multiLine: Boolean) = {
+    private def fetchStringPart(multiLine: Boolean) =
       offset = charOffset - 1
+      lineOffset = if lastOffset < lineStartOffset then lineStartOffset else -1
       getStringPart(multiLine)
-    }
 
     private def isTripleQuote(): Boolean =
       if (ch == '"') {
@@ -1657,21 +1664,31 @@ object Scanners {
     case Run(ch: Char, n: Int)
     case Conc(l: IndentWidth, r: Run)
 
-    def <= (that: IndentWidth): Boolean = this match {
-      case Run(ch1, n1) =>
-        that match {
-          case Run(ch2, n2) => n1 <= n2 && (ch1 == ch2 || n1 == 0)
-          case Conc(l, r) => this <= l
-        }
-      case Conc(l1, r1) =>
-        that match {
-          case Conc(l2, r2) => l1 == l2 && r1 <= r2
-          case _ => false
-        }
-    }
+    def <= (that: IndentWidth): Boolean = (this, that) match
+      case (Run(ch1, n1), Run(ch2, n2)) => n1 <= n2 && (ch1 == ch2 || n1 == 0)
+      case (Conc(l1, r1), Conc(l2, r2)) => (l1 == l2 && r1 <= r2) || this <= l2
+      case (_, Conc(l2, _)) => this <= l2
+      case _ => false
 
     def < (that: IndentWidth): Boolean = this <= that && !(that <= this)
 
+    def >= (that: IndentWidth): Boolean = that <= this
+
+    def >(that: IndentWidth): Boolean = that < this
+
+    def size: Int = this match
+      case Run(_, n) => n
+      case Conc(l, r) => l.size + r.n
+
+    /** Add one level of indentation (one tab or two spaces depending on the last char) */
+    def increment: IndentWidth =
+      def incRun(ch: Char, n: Int): Run = ch match
+        case ' ' => IndentWidth.Run(' ', n + 2)
+        case ch => IndentWidth.Run(ch, n + 1)
+      this match
+        case Run(ch, n) => incRun(ch, n)
+        case Conc(l, Run(ch, n)) => Conc(l, incRun(ch, n))
+
     /** Does `this` differ from `that` by not more than a single space? */
     def isClose(that: IndentWidth): Boolean = this match
       case Run(ch1, n1) =>

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -324,8 +324,7 @@ object MarkupParsers {
     /** Some try/catch/finally logic used by xLiteral and xLiteralPattern.  */
     inline private def xLiteralCommon(f: () => Tree, ifTruncated: String => Unit): Tree = {
       assert(parser.in.token == Tokens.XMLSTART)
-      val saved = parser.in.newTokenData
-      saved.copyFrom(parser.in)
+      val saved = parser.in.saveCopy
       var output: Tree = null.asInstanceOf[Tree]
       try output = f()
       catch {
@@ -404,7 +403,7 @@ object MarkupParsers {
     def escapeToScala[A](op: => A, kind: String): A = {
       xEmbeddedBlock = false
       val res = saving(parser.in.currentRegion, parser.in.currentRegion = _) {
-        val lbrace = parser.in.newTokenData
+        val lbrace = Scanners.newTokenData
         lbrace.token = LBRACE
         lbrace.offset = parser.in.charOffset - 1
         lbrace.lastOffset = parser.in.lastOffset

compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala

@@ -25,9 +25,13 @@ object Rewrites {
     def addPatch(span: Span, replacement: String): Unit =
       pbuf += Patch(span, replacement)
 
+    def patchOver(span: Span, replacement: String): Unit =
+      pbuf.indices.reverse.find(i => span.contains(pbuf(i).span)).foreach(pbuf.remove)
+      pbuf += Patch(span, replacement)
+
     def apply(cs: Array[Char]): Array[Char] = {
       val delta = pbuf.map(_.delta).sum
-      val patches = pbuf.toList.sortBy(_.span.start)
+      val patches = pbuf.toList.sortBy(p => (p.span.start, p.span.end))
       if (patches.nonEmpty)
         patches.reduceLeft {(p1, p2) =>
           assert(p1.span.end <= p2.span.start, s"overlapping patches in $source: $p1 and $p2")
@@ -71,6 +75,14 @@ object Rewrites {
          .addPatch(span, replacement)
     )
 
+  /** Record a patch that replaces the first patch that it contains */
+  def patchOver(source: SourceFile, span: Span, replacement: String)(using Context): Unit =
+    if ctx.reporter != Reporter.NoReporter // NoReporter is used for syntax highlighting
+    then ctx.settings.rewrite.value.foreach(_.patched
+         .getOrElseUpdate(source, new Patches(source))
+         .patchOver(span, replacement)
+    )
+
   /** Patch position in `ctx.compilationUnit.source`. */
   def patch(span: Span, replacement: String)(using Context): Unit =
     patch(ctx.compilationUnit.source, span, replacement)

compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala

@@ -82,9 +82,14 @@ class CompilationTests {
       compileFile("tests/rewrites/rewrites3x.scala", defaultOptions.and("-rewrite", "-source", "future-migration")),
       compileFile("tests/rewrites/filtering-fors.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")),
       compileFile("tests/rewrites/refutable-pattern-bindings.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")),
-      compileFile("tests/rewrites/i8982.scala", defaultOptions.and("-indent", "-rewrite")),
-      compileFile("tests/rewrites/i9632.scala", defaultOptions.and("-indent", "-rewrite")),
-      compileFile("tests/rewrites/i11895.scala", defaultOptions.and("-indent", "-rewrite")),
+      compileFile("tests/rewrites/i8982.scala", indentRewrite),
+      compileFile("tests/rewrites/i9632.scala", indentRewrite),
+      compileFile("tests/rewrites/i11895.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-rewrite.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-comments.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-mix-tab-space.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-3-spaces.scala", indentRewrite),
+      compileFile("tests/rewrites/indent-mix-brace.scala", indentRewrite),
       compileFile("tests/rewrites/i12340.scala", unindentOptions.and("-rewrite")),
       compileFile("tests/rewrites/i17187.scala", unindentOptions.and("-rewrite")),
       compileFile("tests/rewrites/i17399.scala", unindentOptions.and("-rewrite")),

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -65,6 +65,7 @@ object TestConfiguration {
 
   val commonOptions = Array("-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions
   val defaultOptions = TestFlags(basicClasspath, commonOptions)
+  val indentRewrite = defaultOptions.and("-rewrite")
   val unindentOptions = TestFlags(basicClasspath, Array("-no-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions)
   val withCompilerOptions =
     defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath)

compiler/test/dotty/tools/vulpix/TestConfiguration.scala

@@ -152,6 +152,11 @@ object Test23:
   val _ = 1 `+`:  // ok
     x
 
+  // leading infix op
+  val _ = 1
+  `+` :
+    x
+
   val r = 1 to:
     100
 

tests/pos/indent-colons.scala

@@ -0,0 +1,21 @@
+// Rewrite to indent, keeping 3 spaces as indentation
+
+def m1 =
+   def m2 =
+      "" +
+         "" +
+         ""
+   m2
+
+def m4 =
+   def m5 =
+      def m6 =
+         val x = ""
+         x
+            .apply(0)
+            .toString
+      m6
+         .toString
+   m5 +
+      m5
+         .toString

tests/rewrites/indent-3-spaces.check

@@ -0,0 +1,26 @@
+// Rewrite to indent, keeping 3 spaces as indentation
+
+def m1 = {
+   def m2 = {
+      "" +
+         "" +
+         ""
+   }
+   m2
+}
+
+def m4 = {
+   def m5 = {
+      def m6 = {
+         val x = ""
+         x
+            .apply(0)
+            .toString
+      }
+      m6
+         .toString
+   }
+   m5 +
+      m5
+         .toString
+}

tests/rewrites/indent-3-spaces.scala

@@ -0,0 +1,25 @@
+// Rewriting to indent should preserve comments
+class A: /* 1 */ /* 2 */
+  def m1(b: Boolean) = /* 3 */ /* 4 */
+    val x = if (b)
+    /* 5 */
+      "true"
+    /* 6 */
+    else
+    /* 7 */
+      "false"
+    /* 8 */
+    /* 9 */ x.toBoolean
+  /* 10 */ /* 11 */
+  /* 12 */def m2 = // 12
+    m1:// 14
+    /* 15 */// 16
+      true
+    /* 17 */// 18
+// because of the missing indent before {
+// the scanner inserts a new line between || and {
+// cannot rewrite to indentation without messing the comments up
+    true ||// 19
+    /* 20 */{
+      false
+    }// 21