Merge pull request #208 from franklinsch/support-multiple-files

Support multiple input files

Sources/AST/SourceLocation.swift

@@ -5,22 +5,26 @@
 //  Created by Franklin Schrans on 1/7/18.
 //
 
+import Foundation
+
 /// A location in a source file.
 public struct SourceLocation: Equatable {
   public var line: Int
   public var column: Int
   public var length: Int
+  public var file: URL
 
-  public init(line: Int, column: Int, length: Int) {
+  public init(line: Int, column: Int, length: Int, file: URL) {
     self.line = line
     self.column = column
     self.length = length
+    self.file = file
   }
 
   public static func spanning<S1: SourceEntity, S2: SourceEntity>(_ lowerBoundEntity: S1, to upperBoundEntity: S2) -> SourceLocation {
     let lowerBound = lowerBoundEntity.sourceLocation
     let upperBound = upperBoundEntity.sourceLocation
     guard lowerBound.line == upperBound.line else { return lowerBound }
-    return SourceLocation(line: lowerBound.line, column: lowerBound.column, length: upperBound.column + upperBound.length - lowerBound.column)
+    return SourceLocation(line: lowerBound.line, column: lowerBound.column, length: upperBound.column + upperBound.length - lowerBound.column, file: lowerBound.file)
   }
 }

Sources/Parser/Tokenizer.swift

@@ -10,11 +10,15 @@ import AST
 
 /// The tokenizer, which turns the source code into a list of tokens.
 public struct Tokenizer {
+  /// The URL of the source file of the Flint program.
+  var sourceFile: URL
+
   /// The original source code of the Flint program.
   var sourceCode: String
 
-  public init(sourceCode: String) {
-    self.sourceCode = sourceCode
+  public init(sourceFile: URL) {
+    self.sourceFile = sourceFile
+    self.sourceCode = try! String(contentsOf: sourceFile)
   }
 
   /// Converts the source code into a list of tokens.
@@ -135,7 +139,7 @@ public struct Tokenizer {
         // If we're in a comment, discard the characters up to a newline.
         if CharacterSet.newlines.contains(char.unicodeScalars.first!) {
           inComment = false
-          components.append(("\n", SourceLocation(line: line, column: column, length: 1)))
+          components.append(("\n", sourceLocation(line: line, column: column, length: 1)))
 
           line += 1
           column = 1
@@ -153,14 +157,14 @@ public struct Tokenizer {
       } else {
         if !acc.isEmpty {
           // Add the component to the list and reset.
-          components.append((acc, SourceLocation(line: line, column: column - acc.count, length: acc.count)))
+          components.append((acc, sourceLocation(line: line, column: column - acc.count, length: acc.count)))
           acc = ""
         }
 
         // If the last component and the new one can be merged, merge them.
-        if let (last, sourceLocation) = components.last, canBeMerged(last, String(char)) {
-          let sourceLocation = SourceLocation(line: sourceLocation.line, column: sourceLocation.column, length: sourceLocation.length + 1)
-          components[components.endIndex.advanced(by: -1)] = ("\(last)\(char)", sourceLocation)
+        if let (last, loc) = components.last, canBeMerged(last, String(char)) {
+          let loc = sourceLocation(line: loc.line, column: loc.column, length: loc.length + 1)
+          components[components.endIndex.advanced(by: -1)] = ("\(last)\(char)", loc)
           column += 1
 
           if components.last!.0 == "//" {
@@ -173,7 +177,7 @@ public struct Tokenizer {
         }
 
         // The character is a newline.
-        components.append((String(char), SourceLocation(line: line, column: column, length: 1)))
+        components.append((String(char), sourceLocation(line: line, column: column, length: 1)))
       }
 
       column += 1
@@ -184,7 +188,7 @@ public struct Tokenizer {
       }
     }
 
-    components.append((acc, SourceLocation(line: line, column: column - acc.count, length: acc.count)))
+    components.append((acc, sourceLocation(line: line, column: column - acc.count, length: acc.count)))
 
     // Remove empty string components.
     return components.filter { !$0.0.isEmpty }
@@ -198,4 +202,9 @@ public struct Tokenizer {
     let mergeable = syntaxMap.keys.filter { $0.count == 2 }
     return mergeable.contains { $0 == component1 + component2 }
   }
+
+  /// Creates a source location for the current file.
+  func sourceLocation(line: Int, column: Int, length: Int) -> SourceLocation {
+    return SourceLocation(line: line, column: column, length: length, file: sourceFile)
+  }
 }

Sources/flintc/Compiler.swift

@@ -14,22 +14,23 @@ import IRGen
 
 /// Runs the different stages of the compiler.
 struct Compiler {
-  var inputFile: URL
+  var inputFiles: [URL]
   var outputDirectory: URL
   var emitBytecode: Bool
   var shouldVerify: Bool
+
+  func tokenizeFiles() -> [Token] {
+    return inputFiles.flatMap { Tokenizer(sourceFile: $0).tokenize() }
+  }
 
   func compile() -> CompilationOutcome {
-    let sourceCode = try! String(contentsOf: inputFile, encoding: .utf8) + retrieveStandardLibraryCode()
-
-    // Turn the source code into tokens.
-    let tokens = Tokenizer(sourceCode: sourceCode).tokenize()
+    let tokens = tokenizeFiles()
 
     // Turn the tokens into an Abstract Syntax Tree (AST).
     let (parserAST, environment, parserDiagnostics) = Parser(tokens: tokens).parse()
 
     // Create a compilation context.
-    let compilationContext = CompilationContext(sourceCode: sourceCode, fileName: inputFile.lastPathComponent)
+    let compilationContext = CompilationContext(sourceFiles: inputFiles)
 
     guard let ast = parserAST, !parserDiagnostics.contains(where: { $0.isError }) else {
       // If there are any parser errors, abort execution.
@@ -73,22 +74,22 @@ struct Compiler {
     // Compile the IULIA IR code using solc.
     SolcCompiler(inputSource: irCode, outputDirectory: outputDirectory, emitBytecode: emitBytecode).compile()
 
+    print("Produced binary in \(outputDirectory.path.bold).")
     return CompilationOutcome(irCode: irCode, astDump: ASTDumper(topLevelModule: ast).dump())
   }
 
   func exitWithFailure() -> Never {
-    print("Failed to compile \(inputFile.lastPathComponent).")
+    print("Failed to compile.")
     exit(1)
   }
-
-  func retrieveStandardLibraryCode() -> String {
-    return StandardLibrary.default.sourceCode()
-  }
 }
 
 struct CompilationContext {
-  var sourceCode: String
-  var fileName: String
+  var sourceFiles: [URL]
+
+  func sourceCode(in sourceFile: URL) -> String {
+    return try! String(contentsOf: sourceFile)
+  }
 }
 
 struct CompilationOutcome {

Sources/flintc/DiagnosticsFormatter.swift

@@ -19,9 +19,10 @@ public struct DiagnosticsFormatter {
   }
 
   func renderDiagnostic(_ diagnostic: Diagnostic, highlightColor: Color = .lightRed, style: Style = .bold) -> String {
+    let diagnosticFile = diagnostic.sourceLocation?.file
     var sourceFileText = ""
-    if let compilationContext = compilationContext {
-      sourceFileText = " in \(compilationContext.fileName.bold)"
+    if let file = diagnosticFile {
+      sourceFileText = " in \(file.path.bold)"
     }
 
     let infoTopic: String
@@ -35,10 +36,11 @@ public struct DiagnosticsFormatter {
     let infoLine = "\(infoTopic)\(sourceFileText):"
     let body: String
 
-    if let compilationContext = compilationContext {
+    if let compilationContext = compilationContext, let file = diagnosticFile {
+      let sourceCode = compilationContext.sourceCode(in: file)
       body = """
       \(diagnostic.message.indented(by: 2).bold)\(render(diagnostic.sourceLocation).bold):
-      \(renderSourcePreview(at: diagnostic.sourceLocation, sourceCode: compilationContext.sourceCode, highlightColor: highlightColor, style: style))
+      \(renderSourcePreview(at: diagnostic.sourceLocation, sourceCode: sourceCode, highlightColor: highlightColor, style: style))
       """
     } else {
       body = "  \(diagnostic.message.indented(by: 2).bold)"

Sources/flintc/DiagnosticsVerifier.swift

@@ -17,7 +17,19 @@ struct DiagnosticsVerifier {
   private let diagnosticLineRegex = try! NSRegularExpression(pattern: "//\\s*expected-(error|note|warning)\\s*@(-?\\d+)\\s+\\{\\{(.*)\\}\\}")
 
   func verify(producedDiagnostics: [Diagnostic], compilationContext: CompilationContext) -> Bool {
-    let expectations = parseExpectations(sourceCode: compilationContext.sourceCode)
+    var success = true
+
+    for file in compilationContext.sourceFiles {
+      let sourceCode = compilationContext.sourceCode(in: file)
+      let diagnostics = producedDiagnostics.filter { $0.sourceLocation?.file == file }
+      success = success && verify(producedDiagnostics: diagnostics, sourceFile: file, sourceCode: sourceCode, compilationContext: compilationContext)
+    }
+
+    return success
+  }
+
+  func verify(producedDiagnostics: [Diagnostic], sourceFile: URL, sourceCode: String, compilationContext: CompilationContext) -> Bool {
+    let expectations = parseExpectations(sourceCode: sourceCode)
     var producedDiagnostics = flatten(producedDiagnostics)
     var verifyDiagnostics = [Diagnostic]()
 
@@ -30,12 +42,12 @@ struct DiagnosticsVerifier {
       if let index = index {
         producedDiagnostics.remove(at: index)
       } else {
-        verifyDiagnostics.append(Diagnostic(severity: .error, sourceLocation: SourceLocation(line: expectation.line, column: 0, length: 0), message: "Verify: Should have produced \(expectation.severity) \"\(expectation.message)\""))
+        verifyDiagnostics.append(Diagnostic(severity: .error, sourceLocation: SourceLocation(line: expectation.line, column: 0, length: 0, file: sourceFile), message: "Verify: Should have produced \(expectation.severity) \"\(expectation.message)\""))
       }
     }
 
     for producedDiagnostic in producedDiagnostics where producedDiagnostic.severity != .note {
-      verifyDiagnostics.append(Diagnostic(severity: .error, sourceLocation: SourceLocation(line: producedDiagnostic.sourceLocation!.line, column: 0, length: 0), message: "Verify: Unexpected \(producedDiagnostic.severity) \"\(producedDiagnostic.message)\""))
+      verifyDiagnostics.append(Diagnostic(severity: .error, sourceLocation: SourceLocation(line: producedDiagnostic.sourceLocation!.line, column: 0, length: 0, file: sourceFile), message: "Verify: Unexpected \(producedDiagnostic.severity) \"\(producedDiagnostic.message)\""))
     }
 
     let output = DiagnosticsFormatter(diagnostics: verifyDiagnostics, compilationContext: compilationContext).rendered()

Sources/flintc/StandardLibrary.swift

@@ -13,11 +13,9 @@ struct StandardLibrary {
   /// Path to the stdlib directory.
   var url: URL
 
-  func sourceCode() -> String {
-    let files = try! FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil, options: [])
+  var files: [URL] {
+    return try! FileManager.default.contentsOfDirectory(at: url, includingPropertiesForKeys: nil, options: [])
       .filter { $0.pathExtension == "flint" }
-
-    return try! files.map(String.init(contentsOf:)).joined(separator: "\n\n")
   }
 
   static var `default`: StandardLibrary {

Sources/flintc/main.swift

@@ -5,26 +5,33 @@ import AST
 /// The main function for the compiler.
 func main() {
   command(
-    Argument<String>("input file", description: "The input file to compile."),
     Flag("emit-ir", flag: "i", description: "Emit the internal representation of the code."),
     Option<String>("ir-output", default: "", description: "The path at which the IR file should be created."),
     Flag("emit-bytecode", flag: "b", description: "Emit the EVM bytecode representation of the code."),
     Flag("dump-ast", flag: "a", description: "Print the abstract syntax tree of the code."),
-    Flag("verify", flag: "v", description: "Verify expected diagnostics were produced.")
-  ) { inputFile, emitIR, irOutputPath, emitBytecode, dumpAST, shouldVerify in
-    let inputFileURL = URL(fileURLWithPath: inputFile)
+    Flag("verify", flag: "v", description: "Verify expected diagnostics were produced."),
+    VariadicArgument<String>("input files", description: "The input files to compile.")
+  ) { emitIR, irOutputPath, emitBytecode, dumpAST, shouldVerify, inputFilePaths in
+    let inputFiles = inputFilePaths.map(URL.init(fileURLWithPath:))
 
-    guard FileManager.default.fileExists(atPath: inputFile) else {
-      exitWithFileNotFoundDiagnostic(file: inputFileURL)
+    for inputFile in inputFiles {
+      guard FileManager.default.fileExists(atPath: inputFile.path) else {
+        exitWithFileNotFoundDiagnostic(file: inputFile)
+      }
     }
 
-    let fileName = inputFileURL.deletingPathExtension().lastPathComponent
-    let outputDirectory = URL(fileURLWithPath: FileManager.default.currentDirectoryPath).appendingPathComponent("bin/\(fileName)")
+    let outputDirectory = URL(fileURLWithPath: FileManager.default.currentDirectoryPath).appendingPathComponent("bin")
     try! FileManager.default.createDirectory(atPath: outputDirectory.path, withIntermediateDirectories: true, attributes: nil)
-    let compilationOutcome = Compiler(inputFile: inputFileURL, outputDirectory: outputDirectory, emitBytecode: emitBytecode, shouldVerify: shouldVerify).compile()
+
+    let compilationOutcome = Compiler(
+      inputFiles: inputFiles + StandardLibrary.default.files,
+      outputDirectory: outputDirectory,
+      emitBytecode: emitBytecode,
+      shouldVerify: shouldVerify
+    ).compile()
 
     if emitIR {
-      let fileName = inputFileURL.deletingPathExtension().lastPathComponent + ".sol"
+      let fileName = "main.sol"
       let irFileURL: URL
       if irOutputPath.isEmpty {
         irFileURL = outputDirectory.appendingPathComponent(fileName)

Tests/BehaviorTests/compile_behavior_tests.sh

@@ -1,14 +1,16 @@
-#! /bin/sh
+#! /bin/bash
 
 for t in tests/*; do
   rm -rf $t/test/contracts
   mkdir $t/test/contracts
 
-  for f in $t/*.flint; do
-    [ -f "$f" ] || break
-    echo "Compile Flint file '$f'"
-    ../../.build/release/flintc $f --emit-ir --ir-output $t/test/contracts/
-  done
+  echo "Compile $t"
+  ../../.build/release/flintc $t/*.flint --emit-ir --ir-output $t/test/contracts/
+
+  #for f in $t/*.flint; do
+    #[ -f "$f" ] || break
+    #echo "Compile Flint file '$f'"
+  #done
 
   echo "pragma solidity ^0.4.2; contract Migrations {}" > $t/test/contracts/Migrations.sol
 done

Tests/BehaviorTests/tests/currency/Coin.flint

@@ -0,0 +1,25 @@
+struct Coin {
+  var value: Int
+
+  init(amount: Int) {
+    self.value = amount
+  }
+
+  init(other: inout Coin, amount: Int) {
+    if other.getValue() >= amount {
+      value += amount
+      other.value -= amount
+    }
+  }
+
+  func getValue() -> Int {
+    return value
+  }
+
+  mutating func transfer(other: inout Coin, amount: Int) {
+    if other.getValue() >= amount {
+      value += amount
+      other.value -= amount
+    }
+  }
+}

Tests/BehaviorTests/tests/currency/currency.flint

@@ -1,29 +1,3 @@
-struct Coin {
-  var value: Int
-
-  init(amount: Int) {
-    self.value = amount
-  }
-
-  init(other: inout Coin, amount: Int) {
-    if other.getValue() >= amount {
-      value += amount
-      other.value -= amount
-    }
-  }
-
-  func getValue() -> Int {
-    return value
-  }
-
-  mutating func transfer(other: inout Coin, amount: Int) {
-    if other.getValue() >= amount {
-      value += amount
-      other.value -= amount
-    }
-  }
-}
-
 contract C {
   var accounts: [Int: Coin] = [:]
 }