Expanded readme, multi-platform support

Author: mattmassicotte

.github/workflows/ci.yml

@@ -10,11 +10,16 @@ on:
 
 jobs:
   test:
-    name: Test
+    name: Test on destination ${{ matrix.destination }}
     runs-on: macOS-latest
+    strategy:
+      matrix:
+        destination: ["platform=macOS", "platform=iOS Simulator,name=iPhone 8"]
     steps:
       - uses: actions/checkout@v1
       - name: install swiftlint
         run: HOMEBREW_NO_INSTALL_CLEANUP=1 brew install swiftlint
-      - name: xcodebuild test
-        run: set -o pipefail && xcodebuild test -scheme Flexer -destination "platform=macOS" | xcpretty
+      - name: update submodules
+        run: git submodule update --init
+      - name: xcodebuild test ${{ matrix.destination }}
+        run: set -o pipefail && xcodebuild test -scheme Flexer -destination "${{ matrix.destination }}" | xcpretty

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "xcconfigs"]
+	path = xcconfigs
+	url = https://github.com/mrackwitz/xcconfigs.git

Flexer.xcodeproj/project.pbxproj

@@ -42,6 +42,7 @@
 		C908A1272454AECC00ADE4D9 /* LookAheadIteratorProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LookAheadIteratorProtocol.swift; sourceTree = "<group>"; };
 		C908A12F2457239A00ADE4D9 /* ExampleLexerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExampleLexerTests.swift; sourceTree = "<group>"; };
 		C908A13124574B2E00ADE4D9 /* TokenProtocol.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TokenProtocol.swift; sourceTree = "<group>"; };
+		C9C421B6247942CD00694622 /* FlexerTests.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = FlexerTests.xcconfig; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -102,6 +103,7 @@
 				C908A0C224532B2E00ADE4D9 /* Info.plist */,
 				C908A0CF24532C1200ADE4D9 /* BasicTextCharacterLexerTests.swift */,
 				C908A12F2457239A00ADE4D9 /* ExampleLexerTests.swift */,
+				C9C421B6247942CD00694622 /* FlexerTests.xcconfig */,
 			);
 			path = FlexerTests;
 			sourceTree = "<group>";
@@ -410,6 +412,7 @@
 		};
 		C908A0CA24532B2E00ADE4D9 /* Debug */ = {
 			isa = XCBuildConfiguration;
+			baseConfigurationReference = C9C421B6247942CD00694622 /* FlexerTests.xcconfig */;
 			buildSettings = {
 				ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES;
 				CODE_SIGN_STYLE = Automatic;
@@ -428,6 +431,7 @@
 		};
 		C908A0CB24532B2E00ADE4D9 /* Release */ = {
 			isa = XCBuildConfiguration;
+			baseConfigurationReference = C9C421B6247942CD00694622 /* FlexerTests.xcconfig */;
 			buildSettings = {
 				ALWAYS_EMBED_SWIFT_STANDARD_LIBRARIES = YES;
 				CODE_SIGN_STYLE = Automatic;

Flexer/Flexer.xcconfig

@@ -9,6 +9,8 @@
 // Configuration settings file format documentation can be found at:
 // https://help.apple.com/xcode/#/dev745c5c974
 
+#include "xcconfigs/UniversalFramework_Framework.xcconfig"
+
 PRODUCT_NAME = Flexer
 PRODUCT_BUNDLE_IDENTIFIER = com.chimehq.Flexer
 PRODUCT_MODULE_NAME = Flexer
@@ -19,4 +21,5 @@ INFOPLIST_FILE = Flexer/Info.plist
 MACH_O_TYPE = staticlib
 
 MACOSX_DEPLOYMENT_TARGET = 10.11
+IPHONEOS_DEPLOYMENT_TARGET = 10.0
 SWIFT_VERSION = 5.0

FlexerTests/BasicTextCharacterLexerTests.swift

@@ -48,47 +48,32 @@ extension BasicTextCharacterLexerTests {
     }
 }
 
-//extension LexerTests {
-//    func testSingleCharacterLowercaseRun() {
-//        let string = "a"
-//        var lexer = BasicTextLexer(reader: StringReader(string))
-//
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .lowercaseRun, range: NSRange(0..<1), in: string))
-//        XCTAssertNil(lexer.next())
-//    }
-//
-//    func testMultiCharacterLowercaseRun() {
-//        let string = "abc"
-//        var lexer = BasicTextLexer(reader: StringReader(string))
-//
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .lowercaseRun, range: NSRange(0..<3), in: string))
-//        XCTAssertNil(lexer.next())
-//    }
-//
-//    func testMultiCharacterUppercaseRun() {
-//        let string = "ABC"
-//        var lexer = BasicTextLexer(reader: StringReader(string))
-//
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .uppercaseRun, range: NSRange(0..<3), in: string))
-//        XCTAssertNil(lexer.next())
-//    }
-//
-//    func testMixedCharacterWordRun() {
-//        let string = "AbcDef"
-//        var lexer = BasicTextLexer(reader: StringReader(string))
-//
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .uppercaseRun, range: NSRange(0..<1), in: string))
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .lowercaseRun, range: NSRange(1..<3), in: string))
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .uppercaseRun, range: NSRange(3..<4), in: string))
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .lowercaseRun, range: NSRange(4..<6), in: string))
-//        XCTAssertNil(lexer.next())
-//    }
-//
-//    func testDigitRun() {
-//        let string = "1234567890"
-//        var lexer = BasicTextLexer(reader: StringReader(string))
-//
-//        XCTAssertEqual(lexer.next(), BasicTextToken(kind: .numberRun, range: NSRange(0..<10), in: string))
-//        XCTAssertNil(lexer.next())
-//    }
-//}
+extension BasicTextCharacterLexerTests {
+    func testSingleLowercaseCharacter() {
+        let string = "a"
+        var lexer = BasicTextCharacterLexer(string: string)
+
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .lowercaseLetter, range: NSRange(0..<1), in: string))
+        XCTAssertNil(lexer.next())
+    }
+
+    func testMultiCharacterUppercaseRun() {
+        let string = "ABC"
+        var lexer = BasicTextCharacterLexer(string: string)
+
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .uppercaseLetter, range: NSRange(0..<1), in: string))
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .uppercaseLetter, range: NSRange(1..<2), in: string))
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .uppercaseLetter, range: NSRange(2..<3), in: string))
+        XCTAssertNil(lexer.next())
+    }
+
+    func testDigitRun() {
+        let string = "123"
+        var lexer = BasicTextCharacterLexer(string: string)
+
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .digit, range: NSRange(0..<1), in: string))
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .digit, range: NSRange(1..<2), in: string))
+        XCTAssertEqual(lexer.next(), BasicTextCharacter(kind: .digit, range: NSRange(2..<3), in: string))
+        XCTAssertNil(lexer.next())
+    }
+}

FlexerTests/ExampleLexerTests.swift

@@ -63,10 +63,12 @@ struct ExampleTokenSequence: Sequence, IteratorProtocol, StringInitializable {
     }
 }
 
+typealias ExampleTokenLexer = LookAheadSequence<ExampleTokenSequence>
+
 class ExampleLexerTests: XCTestCase {
     func testTokens() {
         let string = "abc d_eF\t\t\tGhi123 JKL 123 **&\nz"
-        var lexer = ExampleTokenSequence(string: string).lookAhead
+        var lexer = ExampleTokenLexer(string: string)
 
         XCTAssertEqual(lexer.next(), ExampleToken(kind: .word, range: NSRange(0..<3), in: string))
         XCTAssertEqual(lexer.next(), ExampleToken(kind: .whitespace, range: NSRange(3..<4), in: string))

FlexerTests/FlexerTests.xcconfig

@@ -0,0 +1,14 @@
+//
+//  FlexerTests.xcconfig
+//  Flexer
+//
+//  Created by Matt Massicotte on 2020-05-23.
+//  Copyright © 2020 Chime Systems Inc. All rights reserved.
+//
+
+// Configuration settings file format documentation can be found at:
+// https://help.apple.com/xcode/#/dev745c5c974
+
+#include "xcconfigs/UniversalFramework_Test.xcconfig"
+
+SUPPORTED_PLATFORMS = macosx iphonesimulator iphoneos

Package.swift

@@ -0,0 +1,16 @@
+// swift-tools-version:5.0
+
+import PackageDescription
+
+let package = Package(
+    name: "Flexer",
+    platforms: [.macOS(.v10_12), .iOS(.v10)],
+    products: [
+        .library(name: "Flexer", targets: ["Flexer"]),
+    ],
+    dependencies: [],
+    targets: [
+        .target(name: "Flexer", dependencies: [], path: "Flexer/"),
+        .testTarget(name: "FlexerTests", dependencies: ["Flexer"], path: "FlexerTests/"),
+    ]
+)

README.md

@@ -3,25 +3,41 @@
 
 # Flexer
 
-Flexer is a small library for building lexers in Swift.
+Flexer is a small library for building lexers in Swift. It is compatible with all Apple platforms.
 
 - API tailored for hand-written parsing
 - Fully Swift `String`-compatible
 - Based around `Sequence` and `IteratorProtocol` procotols
 
 It turns out that Swift's `Sequence` and `Iterator` concepts work pretty well for processing tokens. They make for a familiar API that also offers a surprsing amount of power. Flexer builds on these concepts with some new protocols that are made specifically for lexing, but are generally applicable to all `Sequence` types.
 
+## Integration
+
+### Swift Package Manager
+
+```swift
+dependencies: [
+    .package(url: "https://github.com/ChimeHQ/Flexer.git")
+]
+```
+
+### Carthage
+
+```
+github "ChimeHQ/Flexer"
+```
+
 ## Look-Ahead
 
 Core to lexing is the ability to look ahead at future tokens without advancing. Flexer implements look-ahead with a protocol called `LookAheadIteratorProtocol`. The whole implemenation is inspired by the `lazy` property of `Sequence`, and works very similarly.
 
 ```swift
 let lookAheadSequence = anySequence.lookAhead
 
-
+let next = lookAheadSequence.peek()
 ```
 
-The main work of building your lexer is then defining a Sequence type of tokens. All of lexing facilities you might need can then be exposed with a simple `typealias`.
+The main work of building your lexer is then defining a Sequence type of tokens. All of the lexing facilities you might need can then be exposed with a `typealias`.
 
 ```swift
 typealias MyLexer = LookAheadSequence<MyTokenSequence>
@@ -37,6 +53,70 @@ let tabToken = lexer.nextUntil({ $0.kind == .tab })
 
 Your custom token sequence can be built by creating a struct that conforms to `Sequence`. To make this easier, Flexer includes a type that can be used as a foundation for creating more complex token streams, called `BasicTextCharacterSequence`. It is a sequence of `BasicTextCharacter` elements. It breaks up a string into commonly-needed tokens, catagorized by kind and range within the source string. This approach uses the `Token` type, which stores a kind and a range within the source string.
 
-It is usually much easier to build up more complex lexing functionality with the convenience of Swift switch pattern matching, instead of having to worry about the underlying characters and ranges themselves.
+It is usually much easier to build up more complex lexing functionality with the convenience of Swift switch pattern matching, instead of having to worry about the underlying characters and ranges themselves. You can do this by wrapping up a `BasicTextCharacterSequence` in your own custom sequence.
+
+Here's a fully-functioning example that produces four different token types. It shows off some of the scanning and look-ahead facilities that can be handy both for constructing and also using your lexer.
+
+```swift
+enum ExampleTokenKind {
+    case word
+    case number
+    case symbol
+    case whitespace
+}
+
+typealias ExampleToken = Flexer.Token<ExampleTokenKind>
+
+struct ExampleTokenSequence: Sequence, IteratorProtocol, StringInitializable {
+    public typealias Element = ExampleToken
+
+    private var lexer: BasicTextCharacterLexer
+
+    public init(string: String) {
+        self.lexer = BasicTextCharacterLexer(string: string)
+    }
+
+    public mutating func next() -> Element? {
+        guard let token = lexer.peek() else {
+            return nil
+        }
+
+        switch token.kind {
+        case .lowercaseLetter, .uppercaseLetter, .underscore:
+            guard let endingToken = lexer.nextUntil(notIn: [.lowercaseLetter, .uppercaseLetter, .underscore, .digit]) else {
+                return nil
+            }
+
+            return ExampleToken(kind: .word, range: token.startIndex..<endingToken.endIndex)
+        case .digit:
+            guard let endingToken = lexer.nextUntil({ $0.kind != .digit }) else {
+                return nil
+            }
+
+            return ExampleToken(kind: .number, range: token.startIndex..<endingToken.endIndex)
+        case .newline, .tab, .space:
+            guard let endingToken = lexer.nextUntil(notIn: [.newline, .tab, .space]) else {
+                return nil
+            }
+
+            return ExampleToken(kind: .whitespace, range: token.startIndex..<endingToken.endIndex)
+        default:
+            break
+        }
+
+        guard let endingToken = lexer.nextUntil(in: [.newline, .tab, .space, .lowercaseLetter, .uppercaseLetter, .underscore, .digit]) else {
+            return nil
+        }
+
+        return ExampleToken(kind: .symbol, range: token.startIndex..<endingToken.endIndex)
+    }
+}
+
+typealias ExampleTokenLexer = LookAheadSequence<ExampleTokenSequence>
+```
+
+### Suggestions or Feedback
+
+We'd love to hear from you! Get in touch via [twitter](https://twitter.com/chimehq), an issue, or a pull request.
 
-Flexer also includes a generic `Token` struct, that you might find handy for defining your own custom token types. `BasicTextCharacter` is implemented in terms of `Token`.
+Please note that this project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By participating in this project you agree to abide by its terms.