10000 Merge branch 'precedence' into dev · jtjlehi/scala-parser-combinators@585c297 · GitHub
[go: up one dir, main page]

Skip to content

Commit 585c297

Browse files
committed
Merge branch 'precedence' into dev
2 parents 11e08c6 + 7c397a2 commit 585c297

22 files changed

+206
-47
lines changed

build.sbt

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
ThisBuild / licenses += (("Apache-2.0", url("https://www.apache.org/licenses/LICENSE-2.0")))
22
ThisBuild / startYear := Some(2004)
33

4-
// I thought we could declare these in `ThisBuild` scope but no :-/
54
val commonSettings = Seq(
65
versionScheme := Some("early-semver"),
76
// next version will bump minor (because we dropped Scala 2.11 and upgraded
87
// Scala.js and Scala Native); we could go back to BinaryAndSourceCompatible
98
// once that's done
109
versionPolicyIntention := Compatibility.BinaryCompatible,
10+
crossScalaVersions := Seq("2.13.10", "2.12.17", "3.2.2"),
11+
scalaVersion := crossScalaVersions.value.head,
1112
)
1213

1314
lazy val root = project.in(file("."))
@@ -25,9 +26,6 @@ lazy val parserCombinators = crossProject(JVMPlatform, JSPlatform, NativePlatfor
2526
name := "scala-parser-combinators",
2627
scalaModuleAutomaticModuleName := Some("scala.util.parsing"),
2728

28-
crossScalaVersions := Seq("2.13.10", "2.12.17", "3.2.2"),
29-
scalaVersion := crossScalaVersions.value.head,
30-
3129
libraryDependencies += "junit" % "junit" % "4.13.2" % Test,
3230
libraryDependencies += "com.github.sbt" % "junit-interface" % "0.13.3" % Test,
3331

@@ -38,7 +36,7 @@ lazy val parserCombinators = crossProject(JVMPlatform, JSPlatform, NativePlatfor
3836

3937
// go nearly warning-free, but only on 2.13, it's too hard across all versions
4038
Compile / scalacOptions ++= (CrossVersion.partialVersion(scalaVersion.value) match {
41-
case Some((2, 13)) => Seq("-Werror",
39+
case Some((2, 13)) => Seq("-Werror", "-Wunused",
4240
// ideally we'd do something about this. `^?` is the responsible method
4341
"-Wconf:site=scala.util.parsing.combinator.Parsers.*&cat=lint-multiarg-infix:i",
4442
// not sure what resolving this would look like? didn't think about it too hard

shared/src/main/scala/scala/util/parsing/combinator/PackratParsers.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -208,14 +208,14 @@ to update each parser involved in the recursion.
208208

209209
private def lrAnswer[T](p: Parser[T], in: PackratReader[Elem], growable: LR): ParseResult[T] = growable match {
210210
//growable will always be having a head, we can't enter lrAnswer otherwise
211-
case LR(seed ,rule, Some(head)) =>
211+
case LR(seed, _, Some(head)) =>
212212
if(head.getHead != p) /*not head rule, so not growing*/ seed.asInstanceOf[ParseResult[T]]
213213
else {
214214
in.updateCacheAndGet(p, MemoEntry(Right(seed.asInstanceOf[ParseResult[T]])))
215215
seed match {
216216
case f@Failure(_,_) => f
217217
case e@Error(_,_) => e
218-
case s@Success(_,_) => /*growing*/ grow(p, in, head)
218+
case Success(_,_) => /*growing*/ grow(p, in, head)
219219
}
220220
}
221221
case _=> throw new Exception("lrAnswer with no head !!")
@@ -256,7 +256,7 @@ to update each parser involved in the recursion.
256256
/*simple result*/
257257
inMem.updateCacheAndGet(p,MemoEntry(Right(tempRes)))
258258
tempRes
259-
case s@Some(_) =>
259+
case Some(_) =>
260260
/*non simple result*/
261261
base.seed = tempRes
262262
//the base variable has passed equality tests with the cache
@@ -303,7 +303,7 @@ to update each parser involved in the recursion.
303303
case _ => throw new Exception("impossible match")
304304
}
305305
}
306-
case f =>
306+
case _ =>
307307
rest.recursionHeads -= rest.pos
308308
/*rest.updateCacheAndGet(p, MemoEntry(Right(f)));*/oldRes
309309
}

shared/src/main/scala/scala/util/parsing/combinator/Parsers.scala

Lines changed: 111 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,15 @@ import scala.language.implicitConversions
2020

2121
// TODO: better error handling (labelling like parsec's <?>)
2222

23+
/** An enumeration of operator associativity values: `Left`, `Right`, and
24+
* `Non`.
25+
*/
26+
object Associativity extends Enumeration {
27+
type Associativity = Value
28+
29+
val Left, Right, Non = Value
30+
}
31+
2332
/** `Parsers` is a component that ''provides'' generic parser combinators.
2433
*
2534
* There are two abstract members that must be defined in order to
@@ -223,7 +232,7 @@ trait Parsers {
223232
case s @ Success(result, rest) =>
224233
val failure = selectLastFailure(Some(this), s.lastFailure)
225234
Success(result, rest, failure)
226-
case ns: NoSuccess => if (alt.next.pos < next.pos) this else alt
235+
case _: NoSuccess => if (alt.next.pos < next.pos) this else alt
227236
}
228237
}
229238
}
@@ -316,7 +325,7 @@ trait Parsers {
316325
* @return a `Parser` that -- on success -- returns the result of `q`.
317326
*/
318327
def ~> [U](q: => Parser[U]): Parser[U] = { lazy val p = q // lazy argument
319-
(for(a <- this; b <- p) yield b).named("~>")
328+
(for(_ <- this; b <- p) yield b).named("~>")
320329
}
321330

322331
/** A parser combinator for sequential composition which keeps only the left result.
@@ -330,7 +339,7 @@ trait Parsers {
330339
* @return a `Parser` that -- on success -- returns the result of `p`.
331340
*/
332341
def <~ [U](q: => Parser[U]): Parser[T] = { lazy val p = q // lazy argument
333-
(for(a <- this; b <- p) yield a).named("<~")
342+
(for(a <- this; _ <- p) yield a).named("<~")
334343
}
335344

336345
/**
@@ -372,7 +381,7 @@ trait Parsers {
372381
* The resulting parser fails if either `p` or `q` fails, this failure is fatal.
373382
*/
374383
def ~>! [U](q: => Parser[U]): Parser[U] = { lazy val p = q // lazy argument
375-
OnceParser { (for(a <- this; b <- commit(p)) yield b).named("~>!") }
384+
OnceParser { (for(_ <- this; b <- commit(p)) yield b).named("~>!") }
376385
}
377386

378387
/** A parser combinator for non-back-tracking sequential composition which only keeps the left result.
@@ -385,7 +394,7 @@ trait Parsers {
385394
* The resulting parser fails if either `p` or `q` fails, this failure is fatal.
386395
*/
387396
def <~! [U](q: => Parser[U]): Parser[T] = { lazy val p = q // lazy argument
388-
OnceParser { (for(a <- this; b <- commit(p)) yield a).named("<~!") }
397+
OnceParser { (for(a <- this; _ <- commit(p)) yield a).named("<~!") }
389398
}
390399

391400

@@ -448,7 +457,7 @@ trait Parsers {
448457
*/
449458
def ^^^ [U](v: => U): Parser[U] = new Parser[U] {
450459
lazy val v0 = v // lazy argument
451-
def apply(in: Input) = Parser.this(in) map (x => v0)
460+
def apply(in: Input) = Parser.this(in) map (_ => v0)
452461
}.named(toString+"^^^")
453462

454463
/** A parser combinator for partial function application.
@@ -601,7 +610,7 @@ trait Parsers {
601610
p(in) match{
602611
case s @ Success(_, _) => s
603612
case e @ Error(_, _) => e
604-
case f @ Failure(msg, next) => Error(msg, next)
613+
case Failure(msg, next) => Error(msg, next)
605614
}
606615
}
607616

@@ -613,7 +622,7 @@ trait Parsers {
613622
* @param p A predicate that determines which elements match.
614623
* @return
615624
*/
616-
def elem(kind: String, p: Elem => Boolean) = acceptIf(p)(inEl => kind+" expected")
625+
def elem(kind: String, p: Elem => Boolean) = acceptIf(p)(_ => kind + " expected")
617626

618627
/** A parser that matches only the given element `e`.
619628
*
@@ -995,7 +1004,7 @@ trait Parsers {
9951004
*/
9961005
def phrase[T](p: Parser[T]) = new Parser[T] {
9971006
def apply(in: Input) = p(in) match {
998-
case s @ Success(out, in1) =>
1007+
case s @ Success(_, in1) =>
9991008
if (in1.atEnd) s
10001009
else s.lastFailure match {
10011010
case Some(failure) => failure
@@ -1032,9 +1041,100 @@ trait Parsers {
10321041
= OnceParser{ (for(a <- this; b <- commit(p)) yield new ~(a,b)).named("~") }
10331042

10341043
override def ~> [U](p: => Parser[U]): Parser[U]
1035-
= OnceParser{ (for(a <- this; b <- commit(p)) yield b).named("~>") }
1044+
= OnceParser{ (for(_ <- this; b <- commit(p)) yield b).named("~>") }
10361045

10371046
override def <~ [U](p: => Parser[U]): Parser[T]
1038-
= OnceParser{ (for(a <- this; b <- commit(p)) yield a).named("<~") }
1047+
= OnceParser{ (for(a <- this; _ <- commit(p)) yield a).named("<~") }
1048+
}
1049+
1050+
import Associativity._
1051+
1052+
/** A parser that respects operator the precedence and associativity
1053+
* conventions specified in its constructor.
1054+
*
1055+
* @param primary a parser that matches atomic expressions (the atomicity is
1056+
* from the perspective of binary operators). May include
1057+
* unary operators or parentheses.
1058+
* @param binop a parser that matches binary operators.
1059+
* @param prec_table a list of tuples, each of which encodes a level of
1060+
* precedence. Precedence is encoded highest to lowest.
1061+
* Each precedence level contains an Associativity value
1062+
* and a list of operators.
1063+
* @param makeBinop a function that combines two operands and an operator
1064+
* into a new expression. The result must have the same type
1065+
* as the operands because intermediate results become
1066+
* operands to other operators.
1067+
*/
1068+
class PrecedenceParser[Exp,Op,E <: Exp](primary: Parser[E],
1069+
binop: Parser[Op],
1070+
prec_table: List[(Associativity, List[Op])],
1071+
makeBinop: (Exp, Op, Exp) => Exp) extends Parser[Exp] {
1072+
private def decodePrecedence: (Map[Op, Int], Map[Op, Associativity]) = {
1073+
var precedence = Map.empty[Op, Int]
1074+
var associativity = Map.empty[Op, Associativity]
1075+
var level = prec_table.length
1076+
for ((assoc, ops) <- prec_table) {
1077+
precedence = precedence ++ (for (op <- ops) yield (op, level))
1078+
associativity = associativity ++ (for (op <- ops) yield (op, assoc))
1079+
level -= 1
1080+
}
1081+
(precedence, associativity)
1082+
}
1083+
val (precedence, associativity) = decodePrecedence
1084+
private class ExpandLeftParser(lhs: Exp, minLevel: Int) extends Parser[Exp] {
1085+
def apply(input: Input): ParseResult[Exp] = {
1086+
(binop ~ primary)(input) match {
1087+
case Success(op ~ rhs, next) if precedence(op) >= minLevel => {
1088+
new ExpandRightParser(rhs, precedence(op), minLevel)(next) match {
1089+
case Success(r, nextInput) => new ExpandLeftParser(makeBinop(lhs, op, r), minLevel)(nextInput);
1090+
case ns => ns // dead code
1091+
}
1092+
}
1093+
case _ => {
1094+
Success(lhs, input);
1095+
}
1096+
}
1097+
}
1098+
}
1099+
1100+
private class ExpandRightParser(rhs: Exp, currentLevel: Int, minLevel: Int) extends Parser[Exp] {
1101+
private def nextLevel(nextBinop: Op): Option[Int] = {
1102+
if (precedence(nextBinop) > currentLevel) {
1103+
Some(minLevel + 1)
1104+
} else if (precedence(nextBinop) == currentLevel && associativity(nextBinop) == Associativity.Right) {
1105+
Some(minLevel)
1106+
} else {
1107+
None
1108+
}
1109+
}
1110+
def apply(input: Input): ParseResult[Exp] = {
1111+
def done: ParseResult[Exp] = Success(rhs, input)
1112+
binop(input) match {
1113+
case Success(nextBinop,_) => {
1114+
nextLevel(nextBinop) match {
1115+
case Some(level) => {
1116+
new ExpandLeftParser(rhs, level)(input) match {
1117+
case Success(r, next) => new ExpandRightParser(r, currentLevel, minLevel)(next)
1118+
case ns => ns // dead code
1119+
}
1120+
}
1121+
case None => done
1122+
}
1123+
}
1124+
case _ => done
1125+
}
1126+
}
1127+
}
1128+
1129+
/** Parse an expression.
1130+
*/
1131+
def apply(input: Input): ParseResult[Exp] = {
1132+
primary(input) match {
1133+
case Success(lhs, next) => {
1134+
new ExpandLeftParser(lhs,0)(next)
1135+
}
1136+
case noSuccess => noSuccess
1137+
}
1138+
}
10391139
}
10401140
}

shared/src/main/scala/scala/util/parsing/combinator/lexical/StdLexical.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ class StdLexical extends Lexical with StdTokens {
7979
// construct parser for delimiters by |'ing together the parsers for the individual delimiters,
8080
// starting with the longest one -- otherwise a deli 341A miter D will never be matched if there is
8181
// another delimiter that is a prefix of D
82-
def parseDelim(s: String): Parser[Token] = accept(s.toList) ^^ { x => Keyword(s) }
82+
def parseDelim(s: String): Parser[Token] = accept(s.toList) ^^ { _ => Keyword(s) }
8383

8484
val d = new Array[String](delimiters.size)
8585
delimiters.copyToArray(d, 0)

shared/src/test/scala/scala/util/parsing/combinator/JavaTokenParsersTest.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212

1313
package scala.util.parsing.combinator
1414

15-
import scala.language.implicitConversions
1615
import scala.util.parsing.input.CharArrayReader
1716

1817
import org.junit.Test
@@ -46,7 +45,7 @@ class JavaTokenParsersTest {
4645
def parseFailure(s: String, errorColPos: Int): Unit = {
4746
val parseResult = parseAll(ident, s)
4847
parseResult match {
49-
case Failure(msg, next) =>
48+
case Failure(_, next) =>
5049
val pos = next.pos
5150
assertEquals(1, pos.line)
5251
assertEquals(errorColPos, pos.column)
@@ -85,7 +84,7 @@ class JavaTokenParsersTest {
8584

8685
val parseResult1 = parseAll(p, "start start")
8786
parseResult1 match {
88-
case e @ Failure(message, next) =>
87+
case Failure(message, next) =>
8988
assertEquals(next.pos.line, 1)
9089
assertEquals(next.pos.column, 7)
9190
assert(message.endsWith("string matching regex '(?i)AND' expected but 's' found"))

shared/src/test/scala/scala/util/parsing/combinator/PackratParsersTest.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import org.junit.Test
1616
import org.junit.Assert.assertEquals
1717
import org.junit.Assert.assertTrue
1818

19-
import scala.language.implicitConversions
2019
import scala.util.parsing.combinator.syntactical.StandardTokenParsers
2120

2221
class PackratParsersTest {
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package scala.util.parsing.combinator
2+
3+
// import scala.language.implicitConversions
4+
5+
import java.io.StringReader
6+
import org.junit.Test
7+
import org.junit.Assert.{ assertEquals, assertTrue, fail }
8+
import scala.util.parsing.input.StreamReader
9+
10+
class PrecedenceParsersTest {
11+
12+
abstract class Op
13+
object Plus extends Op {
14+
override def toString = "+"
15+
}
16+
object Minus extends Op {
17+
override def toString = "-"
18+
}
19+
object Mult extends Op {
20+
override def toString = "*"
21+
}
22+
object Divide extends Op {
23+
override def toString = "/"
24+
}
25+
object Equals extends Op {
26+
override def toString = "="
27+
}
28+
29+
abstract class Node
30+
case class Leaf(v: Int) extends Node {
31+
override def toString = v.toString
32+
}
33+
case class Binop(lhs: Node, op: Op, rhs: Node) extends Node {
34+
override def toString = s"($lhs $op $rhs)"
35+
}
36+
37+
object ArithmeticParser extends RegexParsers {
38+
val prec = List(
39+
(Associativity.Left, List(Mult, Divide)),
40+
(Associativity.Left, List(Plus, Minus)),
41+
(Associativity.Right, List(Equals)))
42+
def integer: Parser[Leaf] = "[0-9]+".r ^^ { s: String => Leaf(s.toInt) }
43+
def binop: Parser[Op] = "+" ^^^ Plus | "-" ^^^ Minus | "*" ^^^ Mult | "/" ^^^ Divide | "=" ^^^ Equals
44+
def expression = new PrecedenceParser(integer, binop, prec, Binop.apply)
45+
}
46+
47+
def testExp(expected: Node, input: String): Unit = {
48+
ArithmeticParser.expression(StreamReader(new StringReader(input))) match {
49+
case ArithmeticParser.Success(r, next) => {
50+
assertEquals(expected, r);
51+
assertTrue(next.atEnd);
52+
}
53+
case e => {
54+
fail(s"Error parsing $input: $e");
55+
}
56+
}
57+
}
58+
59+
@Test
60+
def basicExpTests: Unit = {
61+
testExp(Leaf(4), "4")
62+
testExp(Binop(Leaf(1), Plus, Leaf(2)), "1 + 2")
63+
testExp(Binop(Leaf(2), Mult, Leaf(1)), "2 * 1")
64+
}
65+
66+
@Test
67+
def associativityTests: Unit = {
68+
testExp(Binop(Binop(Leaf(1), Minus, Leaf(2)), Plus, Leaf(3)), "1 - 2 + 3")
69+
testExp(Binop(Leaf(1), Equals, Binop(Leaf(2), Equals, Leaf(3))), "1 = 2 = 3")
70+
}
71+
72+
@Test
73+
def precedenceTests: Unit = {
74+
testExp(Binop(Binop(Leaf(0), Mult, Leaf(5)), Minus, Leaf(2)), "0 * 5 - 2")
75+
testExp(Binop(Leaf(3), Plus, Binop(Leaf(9), Divide, Leaf(11))), "3 + 9 / 11")
76+
testExp(Binop(Binop(Leaf(6), Plus, Leaf(8)), Equals, Leaf(1)), "6 + 8 = 1")
77+
testExp(Binop(Leaf(4), Equals, Binop(Leaf(5), Minus, Leaf(3))), "4 = 5 - 3")
78+
}
79+
}

0 commit comments

Comments
 (0)
0