Add a lazy DFA.

A lazy DFA is much faster than executing an NFA because it doesn't
repeat the work of following epsilon transitions over and and over.
Instead, it computes states during search and caches them for reuse. We
avoid exponential state blow up by bounding the cache in size. When the
DFA isn't powerful enough to fulfill the caller's request (e.g., return
sub-capture locations), it still runs to find the boundaries of the
match and then falls back to NFA execution on the matched region. The
lazy DFA can otherwise execute on every regular expression *except* for
regular expressions that contain word boundary assertions (`\b` or
`\B`). (They are tricky to implement in the lazy DFA because they are
Unicode aware and therefore require multi-byte look-behind/ahead.)
The implementation in this PR is based on the implementation in Google's
RE2 library.

Adding a lazy DFA was a substantial change and required several
modifications:

1. The compiler can now produce both Unicode based programs (still used by the
   NFA engines) and byte based programs (required by the lazy DFA, but possible
   to use in the NFA engines too). In byte based programs, UTF-8 decoding is
   built into the automaton.
2. A new `Exec` type was introduced to implement the logic for compiling
   and choosing the right engine to use on each search.
3. Prefix literal detection was rewritten to work on bytes.
4. Benchmarks were overhauled and new ones were added to more carefully
   track the impact of various optimizations.
5. A new `HACKING.md` guide has been added that gives a high-level
   design overview of this crate.

Other changes in this commit include:

1. Protection against stack overflows. All places that once required
   recursion have now either acquired a bound or have been converted to
   using a stack on the heap.
2. Update the Aho-Corasick dependency, which includes `memchr2` and
   `memchr3` optimizations.
3. Add PCRE benchmarks using the Rust `pcre` bindings.

Closes #66, #146.

Author: BurntSushi

.travis.yml

@@ -13,18 +13,12 @@ script:
   - cargo test --verbose --manifest-path=regex-syntax/Cargo.toml
   - cargo doc --verbose --manifest-path=regex-syntax/Cargo.toml
   - if [ "$TRAVIS_RUST_VERSION" = "nightly" ]; then
-      cargo test --verbose --features pattern;
-      cargo bench --verbose;
+      travis_wait cargo test --verbose --features pattern;
+      travis_wait cargo bench --verbose --bench dynamic;
+      travis_wait cargo bench --manifest-path=regex-pcre-benchmark/Cargo.toml --verbose
       travis_wait cargo test --verbose --manifest-path=regex_macros/Cargo.toml;
-      travis_wait cargo bench --verbose --manifest-path=regex_macros/Cargo.toml;
+      travis_wait cargo bench --manifest-path=regex_macros/Cargo.toml --verbose --bench native bench::;
     fi
-  # - |
-    # [ $TRAVIS_RUST_VERSION != nightly ] || (
-      # cargo test --verbose --features pattern &&
-      # cargo bench --verbose &&
-      # travis_wait cargo test --verbose --manifest-path=regex_macros/Cargo.toml &&
-      # travis_wait cargo bench --verbose --manifest-path=regex_macros/Cargo.toml
-    # )
 after_success: |
   [ $TRAVIS_BRANCH = master ] &&
   [ $TRAVIS_PULL_REQUEST = false ] &&

Cargo.toml

@@ -8,41 +8,86 @@ repository = "https://github.com/rust-lang/regex"
 documentation = "https://doc.rust-lang.org/regex"
 homepage = "https://github.com/rust-lang/regex"
 description = """
-An implementation of regular expressions for Rust.
+An implementation of regular expressions for Rust. This implementation uses
+finite automata and guarantees linear time matching on all inputs.
 """
 
+[dependencies]
+# For very fast prefix literal matching.
+aho-corasick = "0.5"
+# For skipping along search text quickly when a leading byte is known.
+memchr = "0.1"
+# For parsing regular expressions.
+regex-syntax = { path = "regex-syntax", version = "0.2" }
+# For compiling UTF-8 decoding into automata.
+utf8-ranges = "0.1"
+
+[dev-dependencies]
+# To prevent the benchmarking harness from running setup code more than once.
+# Why? Because it takes too long.
+lazy_static = "0.1"
+# For generating random text to test/benchmark with.
+rand = "0.3"
+
+[features]
+# Enable to use the unstable pattern traits defined in std.
+pattern = []
+
+# Runs unit tests defined inside the regex package.
+# Generally these tests specific pieces of the regex implementation.
 [[test]]
 path = "src/lib.rs"
 name = "regex"
 
+# Run the test suite on the default behavior of Regex::new.
+# This includes a mish mash of NFAs and DFAs, which are chosen automatically
+# based on the regex. We test both of the NFA implementations by forcing their
+# usage with the test definitions below. (We can't test the DFA implementations
+# in the same way since they can't be used for every regex tested.)
 [[test]]
-path = "regex_macros/tests/test_dynamic.rs"
+path = "tests/test_dynamic.rs"
 name = "dynamic"
 
+# Run the test suite on the NFA algorithm over Unicode codepoints.
+[[test]]
+path = "tests/test_dynamic_nfa.rs"
+name = "dynamic-nfa"
+
+# Run the test suite on the NFA algorithm over bytes.
+[[test]]
+path = "tests/test_dynamic_nfa_bytes.rs"
+name = "dynamic-nfa-bytes"
+
+# Run the test suite on the backtracking engine over Unicode codepoints.
 [[test]]
-path = "regex_macros/tests/test_dynamic_nfa.rs"
-name = "dynamic_nfa"
+path = "tests/test_dynamic_backtrack.rs"
+name = "dynamic-backtrack"
 
+# Run the test suite on the backtracking engine over bytes.
 [[test]]
-path = "regex_macros/tests/test_dynamic_backtrack.rs"
-name = "dynamic_backtrack"
+path = "tests/test_dynamic_backtrack_bytes.rs"
+name = "dynamic-backtrack-bytes"
 
+# Run the benchmarks on the default behavior of Regex::new.
+#
+# N.B. These benchmarks were originally taken from Russ Cox.
 [[bench]]
-name = "all"
-path = "regex_macros/benches/bench_dynamic.rs"
+name = "dynamic"
+path = "benches/bench_dynamic.rs"
 test = false
 bench = true
 
-[dependencies]
-aho-corasick = "0.4"
-memchr = "0.1"
-regex-syntax = { path = "regex-syntax", version = "0.2" }
-
-[dev-dependencies]
-rand = "0.3"
-
-[features]
-pattern = []
+# Run the benchmarks on the NFA algorithm. We avoid chasing other permutations.
+#
+# N.B. These can take a *loong* time to run.
+[[bench]]
+name = "dynamic-nfa"
+path = "benches/bench_dynamic_nfa.rs"
+test = false
+bench = true
 
 [profile.bench]
-lto = true
+debug = true
+
+[profile.test]
+debug = true