8000 memory performance: two-level cmap lookup by lsf37 · Pull Request #697 · jflex-de/jflex · GitHub
[go: up one dir, main page]
Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

memory performance: two-level cmap lookup #697

Merged
merged 3 commits into from
Jan 3, 2020
Merged

memory performance: two-level cmap lookup #697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4f5a64c
Use a sparse two-level table for cmap lookup in generated scanner.
lsf37 Dec 30, 2019
795dd64
avoid calling getClassCode for each code point
lsf37 Dec 31, 2019
bab86ff
assert invariants in quickcheck shrinking
lsf37 Dec 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 22 additions & 23 deletions javatests/jflex/testcase/include/skeleton.nested
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@

/** the textposition at the last accepting state */
private int zzMarkedPos;

/** the current text position in the buffer */
private int zzCurrentPos;

Expand All @@ -54,12 +54,12 @@
private int yychar;

/**
* the number of characters from the last newline up to the start of the
* the number of characters from the last newline up to the start of the
* matched text
*/
private int yycolumn;

/**
/**
* zzAtBOL == true <=> the scanner is currently at the beginning of a line
*/
private boolean zzAtBOL = true;
Expand All @@ -69,8 +69,8 @@

/** denotes if the user-EOF-code has already been executed */
private boolean zzEOFDone;
/**

/**
* The number of occupied positions in zzBuffer beyond zzEndRead.
* When a lead/high surrogate has been read from the input stream
* into the final zzBuffer position, this will have a value of 1;
Expand Down Expand Up @@ -103,7 +103,7 @@

/** sets all values stored in this class */
ZzFlexStreamInfo(java.io.Reader zzReader, int zzEndRead, int zzStartRead,
int zzCurrentPos, int zzMarkedPos, char [] zzBuffer,
int zzCurrentPos, int zzMarkedPos, char [] zzBuffer,
boolean zzAtBOL, boolean zzAtEOF, boolean zzEOFDone,
int zzFinalHighSurrogate, int yyline, int yychar, int yycolumn) {
this.zzReader = zzReader;
Expand Down Expand Up @@ -131,7 +131,7 @@
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
*
* @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
Expand Down Expand Up @@ -183,7 +183,7 @@
return true;
}


/**
* Closes the input stream.
*/
Expand Down Expand Up @@ -223,11 +223,11 @@
zzFinalHighSurrogate = 0;
yyline = yychar = yycolumn = 0;
}


/**
* Closes the current input stream and continues to
* read from the one on top of the stream stack.
* read from the one on top of the stream stack.
*
* @throws java.util.EmptyStackException
* if there is no further stream to read from.
Expand Down Expand Up @@ -257,7 +257,7 @@


/**
* Returns true iff there are still streams left
* Returns true iff there are still streams left
* to read from on the stream stack.
*/
public final boolean yymoreStreams() {
Expand All @@ -269,13 +269,13 @@
* Resets the scanner to read from a new input stream.
* Does not close the old reader.
*
* All internal variables are reset, the old input stream
* All internal variables are reset, the old input stream
* <b>cannot</b> be reused (internal buffer is discarded and lost).
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
*
* Internal scan buffer is resized down to its initial length, if it has grown.
*
* @param reader the new input stream
* @param reader the new input stream
*
* @see #yypushStream(java.io.Reader)
* @see #yypopStream()
Expand Down Expand Up @@ -329,12 +329,12 @@


/**
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* It is equivalent to yytext().charAt(pos), but faster
*
* @param pos the position of the character to fetch.
* @param pos the position of the character to fetch.
* A value from 0 to yylength()-1.
*
* @return the character at position pos
Expand All @@ -355,8 +355,8 @@
/**
* Reports an error that occured while scanning.
*
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* will only be called with things that "Can't Possibly Happen".
* If this method is called, something is seriously wrong
* (e.g. a JFlex bug producing a faulty scanner etc.).
Expand All @@ -376,7 +376,7 @@
}

--- throws clause
}
}


/**
Expand Down Expand Up @@ -415,18 +415,17 @@
int zzMarkedPosL = zzMarkedPos;
int zzEndReadL = zzEndRead;
char [] zzBufferL = zzBuffer;
char [] zzCMapL = ZZ_CMAP;

--- start admin (line, char, col count)
zzAction = -1;

zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

--- start admin (lexstate etc)

zzForAction: {
while (true) {

--- next input, line, col, char count, next transition, isFinal action
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
Expand Down
45 changes: 22 additions & 23 deletions javatests/jflex/testcase/include2/skeleton.nested
View file
Open in desktop
EF5E
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@

/** the textposition at the last accepting state */
private int zzMarkedPos;

/** the current text position in the buffer */
private int zzCurrentPos;

Expand All @@ -54,12 +54,12 @@
private int yychar;

/**
* the number of characters from the last newline up to the start of the
* the number of characters from the last newline up to the start of the
* matched text
*/
private int yycolumn;

/**
/**
* zzAtBOL == true <=> the scanner is currently at the beginning of a line
*/
private boolean zzAtBOL = true;
Expand All @@ -69,8 +69,8 @@

/** denotes if the user-EOF-code has already been executed */
private boolean zzEOFDone;
/**

/**
* The number of occupied positions in zzBuffer beyond zzEndRead.
* When a lead/high surrogate has been read from the input stream
* into the final zzBuffer position, this will have a value of 1;
Expand Down Expand Up @@ -103,7 +103,7 @@

/** sets all values stored in this class */
ZzFlexStreamInfo(java.io.Reader zzReader, int zzEndRead, int zzStartRead,
int zzCurrentPos, int zzMarkedPos, char [] zzBuffer,
int zzCurrentPos, int zzMarkedPos, char [] zzBuffer,
boolean zzAtBOL, boolean zzAtEOF, boolean zzEOFDone,
int zzFinalHighSurrogate, int yyline, int yychar, int yycolumn) {
this.zzReader = zzReader;
Expand Down Expand Up @@ -131,7 +131,7 @@
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
*
* @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
Expand Down Expand Up @@ -183,7 +183,7 @@
return true;
}


/**
* Closes the input stream.
*/
Expand Down Expand Up @@ -223,11 +223,11 @@
zzFinalHighSurrogate = 0;
yyline = yychar = yycolumn = 0;
}


/**
* Closes the current input stream and continues to
* read from the one on top of the stream stack.
* read from the one on top of the stream stack.
*
* @throws java.util.EmptyStackException
* if there is no further stream to read from.
Expand Down Expand Up @@ -257,7 +257,7 @@


/**
* Returns true iff there are still streams left
* Returns true iff there are still streams left
* to read from on the stream stack.
*/
public final boolean yymoreStreams() {
Expand All @@ -269,13 +269,13 @@
* Resets the scanner to read from a new input stream.
* Does not close the old reader.
*
* All internal variables are reset, the old input stream
* All internal variables are reset, the old input stream
* <b>cannot</b> be reused (internal buffer is discarded and lost).
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
*
* Internal scan buffer is resized down to its initial length, if it has grown.
*
* @param reader the new input stream
* @param reader the new input stream
*
* @see #yypushStream(java.io.Reader)
* @see #yypopStream()
Expand Down Expand Up @@ -328,12 +328,12 @@


/**
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* It is equivalent to yytext().charAt(pos), but faster
*
* @param pos the position of the character to fetch.
* @param pos the position of the character to fetch.
* A value from 0 to yylength()-1.
*
* @return the character at position pos
Expand All @@ -354,8 +354,8 @@
/**
* Reports an error that occured while scanning.
*
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* will only be called with things that "Can't Possibly Happen".
* If this method is called, something is seriously wrong
* (e.g. a JFlex bug producing a faulty scanner etc.).
Expand All @@ -375,7 +375,7 @@
}

--- throws clause
}
}


/**
Expand Down Expand Up @@ -414,18 +414,17 @@
int zzMarkedPosL = zzMarkedPos;
int zzEndReadL = zzEndRead;
char [] zzBufferL = zzBuffer;
char [] zzCMapL = ZZ_CMAP;

--- start admin (line, char, col count)
zzAction = -1;

zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

--- start admin (lexstate etc)

zzForAction: {
while (true) {

--- next input, line, col, char count, next transition, isFinal action
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
Expand Down
45 changes: 45 additions & 0 deletions jflex/src/main/java/jflex/base/Pair.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.8.0-SNAPSHOT *
* Copyright (C) 1998-2018 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* License: BSD *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

package jflex.base;

import java.util.Objects;

/**
* Generic immutable pair.
*
* @author Gerwin Klein
* @version JFlex 1.8.0-SNAPSHOT
*/
public class Pair<A, B> {
public final A fst;
public final B snd;

public Pair(A fst, B snd) {
this.fst = fst;
this.snd = snd;
}

@Override
public int hashCode() {
return Objects.hash(fst, snd);
}

@Override
public boolean equals(Object other) {
return other instanceof Pair<?, ?>
&& Objects.equals(fst, ((Pair<?, ?>) other).fst)
&& Objects.equals(snd, ((Pair<?, ?>) other).snd);
}

@Override
public String toString() {
return "(" + fst + ", " + snd + ")";
}
}
1 change: 1 addition & 0 deletions jflex/src/main/java/jflex/core/unicode/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ java_library(
srcs = glob(["data/*.java"]) + [
"CharClasses.java",
"CharClassInterval.java",
"CMapBlock.java",
"IntCharSet.java",
"IntCharSetComparator.java",
"ILexScan.java",
Expand Down
Loading
0