diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 0046f90b12..6985daf4cd 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -55,8 +55,14 @@ jobs: if: ${{ runner.os == 'Linux' }} - name: Perform tests for C target run: | - ./gradlew test --tests org.lflang.tests.runtime.CTest.* + ./gradlew test --tests org.lflang.tests.runtime.CTest.* --tests org.lflang.tests.lsp.LspTests.lspWithDependenciesTestC if: ${{ !inputs.use-cpp }} + - name: Report to CodeCov + uses: codecov/codecov-action@v2.1.0 + with: + file: org.lflang.tests/build/reports/xml/jacoco + fail_ci_if_error: false + verbose: true - name: Perform tests for CCpp target run: | ./gradlew test --tests org.lflang.tests.runtime.CCppTest.* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 773c0baad4..2c9a048535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,10 @@ jobs: with: target: 'C' + # Run language server tests. + lsp-tests: + uses: lf-lang/lingua-franca/.github/workflows/lsp-tests.yml@master + # Run the C integration tests. c-tests: uses: lf-lang/lingua-franca/.github/workflows/c-tests.yml@master diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml index 6bafdd6c87..4d456f50f3 100644 --- a/.github/workflows/cpp-tests.yml +++ b/.github/workflows/cpp-tests.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v2 - name: Run C++ tests; run: | - ./gradlew test --tests org.lflang.tests.runtime.CppTest.* + ./gradlew test --tests org.lflang.tests.runtime.CppTest.* --tests org.lflang.tests.lsp.LspTests.lspWithDependenciesTestCpp - name: Report to CodeCov uses: codecov/codecov-action@v2.1.0 with: diff --git a/.github/workflows/lsp-tests.yml b/.github/workflows/lsp-tests.yml new file mode 100644 index 0000000000..a0b5e5ea7b --- /dev/null +++ b/.github/workflows/lsp-tests.yml @@ -0,0 +1,81 @@ +name: Language server tests + +on: + workflow_call: + +jobs: + run: + strategy: + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} + steps: + # Uninstall operations are needed because the language server is able to use multiple + # different compilers for syntax checking. We test that it correctly detects which tools are + # present and responds appropriately. + - name: Uninstall packages Ubuntu + run: sudo apt-get remove clang-* + if: ${{ runner.os == 'Linux' }} + - name: Uninstall packages MacOS + run: brew uninstall gcc + if: ${{ runner.os == 'macOS' }} + - name: Uninstall packages Windows + run: | + del "C:\ProgramData\Chocolatey\bin\g++.exe" + del "C:\Strawberry\c\bin\g++.exe" + del "C:\Program Files\LLVM\bin\clang++.exe" + if: ${{ runner.os == 'Windows' }} + - name: Install JDK + uses: actions/setup-java@v1.4.3 + with: + java-version: 11 + - name: Check out lingua-franca repository + uses: actions/checkout@v2 + with: + repository: lf-lang/lingua-franca + submodules: true + ref: ${{ inputs.compiler-ref }} + - name: Setup Node.js environment + uses: actions/setup-node@v2.1.2 + - name: Install pnpm + run: npm i -g pnpm + - name: Setup Rust + uses: ATiltedTree/setup-rust@v1 + with: + rust-version: nightly + components: clippy + - name: Install Dependencies Ubuntu + run: | + sudo apt-get install libprotobuf-dev protobuf-compiler libprotobuf-c-dev protobuf-c-compiler + if: ${{ runner.os == 'Linux' }} + - name: Install Dependencies OS X + run: | + brew install protobuf + brew install protobuf-c + if: ${{ runner.os == 'macOS' }} + - name: Install dependencies Windows + uses: lukka/run-vcpkg@v4 + with: + vcpkgArguments: protobuf + vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da + vcpkgDirectory: ${{ github.workspace }}/vcpkg/ + vcpkgTriplet: x64-windows-static + if: ${{ runner.os == 'Windows' }} + - name: Run language server Python tests without PyLint + run: ./gradlew test --tests org.lflang.tests.lsp.LspTests.pythonSyntaxOnlyValidationTest + - name: Report to CodeCov + uses: codecov/codecov-action@v2.1.0 + with: + file: org.lflang.tests/build/reports/xml/jacoco + fail_ci_if_error: false + verbose: true + - name: Install pylint + run: python3 -m pip install pylint + - name: Run language server tests + run: ./gradlew test --tests org.lflang.tests.lsp.LspTests.*ValidationTest + - name: Report to CodeCov + uses: codecov/codecov-action@v2.1.0 + with: + file: org.lflang.tests/build/reports/xml/jacoco + fail_ci_if_error: false + verbose: true diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index 650e213a66..d70292bd8f 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -69,7 +69,7 @@ jobs: if: ${{ runner.os == 'Linux' }} - name: Run Python tests run: | - ./gradlew test --tests org.lflang.tests.runtime.PythonTest.* + ./gradlew test --tests org.lflang.tests.runtime.PythonTest.* --tests org.lflang.tests.lsp.LspTests.lspWithDependenciesTestPython - name: Report to CodeCov uses: codecov/codecov-action@v2.1.0 with: diff --git a/.github/workflows/rs-tests.yml b/.github/workflows/rs-tests.yml index 639fb41497..19f6437873 100644 --- a/.github/workflows/rs-tests.yml +++ b/.github/workflows/rs-tests.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/checkout@v2 - name: Run Rust tests run: | - ./gradlew test --tests org.lflang.tests.runtime.RustTest.* + ./gradlew test --tests org.lflang.tests.runtime.RustTest.* --tests org.lflang.tests.lsp.LspTests.lspWithDependenciesTestRust - name: Report to CodeCov uses: codecov/codecov-action@v2.1.0 with: diff --git a/.github/workflows/ts-tests.yml b/.github/workflows/ts-tests.yml index f0bea1dffb..997e3e77a4 100644 --- a/.github/workflows/ts-tests.yml +++ b/.github/workflows/ts-tests.yml @@ -48,7 +48,7 @@ jobs: if: ${{ inputs.runtime-ref }} - name: Perform TypeScript tests run: | - ./gradlew test --tests org.lflang.tests.runtime.TypeScriptTest.* + ./gradlew test --tests org.lflang.tests.runtime.TypeScriptTest.* --tests org.lflang.tests.lsp.LspTests.lspWithDependenciesTestTypeScript - name: Report to CodeCov uses: codecov/codecov-action@v2.1.0 with: diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 25011f531b..5a907b1337 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -26,8 +26,8 @@ @@ -52,6 +52,8 @@ diff --git a/org.lflang.diagram/src/org/lflang/diagram/lsp/LFLanguageServerExtension.java b/org.lflang.diagram/src/org/lflang/diagram/lsp/LFLanguageServerExtension.java index 43a1213e86..0a3468f295 100644 --- a/org.lflang.diagram/src/org/lflang/diagram/lsp/LFLanguageServerExtension.java +++ b/org.lflang.diagram/src/org/lflang/diagram/lsp/LFLanguageServerExtension.java @@ -29,29 +29,6 @@ */ class LFLanguageServerExtension implements ILanguageServerExtension { - /** - * Describes a build process that has a progress. - */ - private GeneratorResult buildWithProgress(LanguageClient client, String uri, boolean mustComplete) { - URI parsedUri; - try { - parsedUri = URI.createFileURI(new java.net.URI(uri).getPath()); - } catch (java.net.URISyntaxException e) { - // This error will appear as a silent failure to most users, but that is acceptable because this error - // should be impossible. The URI is not the result of user input -- the language client provides it -- - // so it should be valid. - System.err.println(e); - return GeneratorResult.NOTHING; - } - Progress progress = new Progress(client, "Build \"" + parsedUri.lastSegment() + "\"", mustComplete); - progress.begin(); - GeneratorResult result = builder.run( - parsedUri, mustComplete, progress::report, progress.getCancelIndicator() - ); - progress.end(result.getUserMessage()); - return result; - } - /** The IntegratedBuilder instance that handles all build requests for the current session. */ private static final IntegratedBuilder builder = new LFStandaloneSetup(new LFRuntimeModule()) .createInjectorAndDoEMFRegistration().getInstance(IntegratedBuilder.class); @@ -81,7 +58,13 @@ public CompletableFuture build(String uri) { "Please wait for the Lingua Franca language server to be fully initialized." ); return CompletableFuture.supplyAsync( - () -> buildWithProgress(client, uri, true).getUserMessage() + () -> { + try { + return buildWithProgress(client, uri, true).getUserMessage(); + } catch (Exception e) { + return "An internal error occurred:\n" + e; + } + } ); } @@ -115,4 +98,31 @@ public CompletableFuture buildAndRun(String uri) { return ret.toArray(new String[0]); }); } + + /** + * Describes a build process that has a progress. + */ + private GeneratorResult buildWithProgress(LanguageClient client, String uri, boolean mustComplete) { + URI parsedUri; + try { + parsedUri = URI.createFileURI(new java.net.URI(uri).getPath()); + } catch (java.net.URISyntaxException e) { + // This error will appear as a silent failure to most users, but that is acceptable because this error + // should be impossible. The URI is not the result of user input -- the language client provides it -- + // so it should be valid. + System.err.println(e); + return GeneratorResult.NOTHING; + } + Progress progress = new Progress(client, "Build \"" + parsedUri.lastSegment() + "\"", mustComplete); + progress.begin(); + GeneratorResult result = null; + try { + result = builder.run( + parsedUri, mustComplete, progress::report, progress.getCancelIndicator() + ); + } finally { + progress.end(result == null ? "An internal error occurred." : result.getUserMessage()); + } + return result; + } } diff --git a/org.lflang.targetplatform/org.lflang.targetplatform.target b/org.lflang.targetplatform/org.lflang.targetplatform.target index 3fcc8ede4c..91becaac83 100644 --- a/org.lflang.targetplatform/org.lflang.targetplatform.target +++ b/org.lflang.targetplatform/org.lflang.targetplatform.target @@ -30,6 +30,9 @@ + + + diff --git a/org.lflang.tests/src/org/lflang/tests/AbstractTest.java b/org.lflang.tests/src/org/lflang/tests/AbstractTest.java index 02b0d42196..4e707e1fe2 100644 --- a/org.lflang.tests/src/org/lflang/tests/AbstractTest.java +++ b/org.lflang.tests/src/org/lflang/tests/AbstractTest.java @@ -57,7 +57,7 @@ protected boolean supportsGenericTypes() { } /** - * Whether to enable {@link #runDockerNonfederatedTests()} and {@link #runDockerFederatedTests()}. + * Whether to enable {@link #runDockerTests()} and {@link #runDockerFederatedTests()}. */ protected boolean supportsDockerOption() { return false; diff --git a/org.lflang.tests/src/org/lflang/tests/TestBase.java b/org.lflang.tests/src/org/lflang/tests/TestBase.java index 37b26b43af..eaa0d119c9 100644 --- a/org.lflang.tests/src/org/lflang/tests/TestBase.java +++ b/org.lflang.tests/src/org/lflang/tests/TestBase.java @@ -7,7 +7,6 @@ import java.io.PrintStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; import java.nio.file.Path; import java.io.File; import java.io.FileWriter; @@ -45,12 +44,13 @@ import org.lflang.LFStandaloneSetup; import org.lflang.Target; import org.lflang.TargetConfig.Mode; +import org.lflang.generator.GeneratorResult; import org.lflang.generator.LFGenerator; +import org.lflang.generator.LFGeneratorContext; import org.lflang.generator.MainContext; import org.lflang.tests.Configurators.Configurator; import org.lflang.tests.LFTest.Result; import org.lflang.tests.TestRegistry.TestCategory; -import org.lflang.util.StringUtil; import org.lflang.util.LFCommand; import com.google.inject.Inject; @@ -361,7 +361,7 @@ private static void checkAndReportFailures(Set tests) { * transformation that may have occured in other tests. * @throws IOException if there is any file access problem */ - private IGeneratorContext configure(LFTest test, Configurator configurator, TestLevel level) throws IOException { + private LFGeneratorContext configure(LFTest test, Configurator configurator, TestLevel level) throws IOException { var context = new MainContext( Mode.STANDALONE, CancelIndicator.NullImpl, (m, p) -> {}, new Properties(), true, fileConfig -> new DefaultErrorReporter() @@ -431,14 +431,17 @@ protected void addExtraLfcArgs(Properties args) { * Invoke the code generator for the given test. * @param test The test to generate code for. */ - private void generateCode(LFTest test) { + private GeneratorResult generateCode(LFTest test) { + GeneratorResult result = GeneratorResult.NOTHING; if (test.fileConfig.resource != null) { generator.doGenerate(test.fileConfig.resource, fileAccess, test.fileConfig.context); + result = test.fileConfig.context.getResult(); if (generator.errorsOccurred()) { test.result = Result.CODE_GEN_FAIL; throw new AssertionError("Code generation unsuccessful."); } } + return result; } @@ -447,8 +450,8 @@ private void generateCode(LFTest test) { * did not execute, took too long to execute, or executed but exited with * an error code. */ - private void execute(LFTest test) { - final List pbList = getExecCommand(test); + private void execute(LFTest test, GeneratorResult generatorResult) { + final List pbList = getExecCommand(test, generatorResult); if (pbList.isEmpty()) { return; } @@ -601,11 +604,7 @@ private List getFederatedDockerExecCommand(LFTest test) { * that should be used to execute the test program. * @param test The test to get the execution command for. */ - private List getExecCommand(LFTest test) { - final var nameWithExtension = test.srcFile.getFileName().toString(); - final var nameOnly = nameWithExtension.substring(0, nameWithExtension.lastIndexOf('.')); - - var srcGenPath = test.fileConfig.getSrcGenPath(); + private List getExecCommand(LFTest test, GeneratorResult generatorResult) { var srcBasePath = test.fileConfig.srcPkgPath.resolve("src"); var relativePathName = srcBasePath.relativize(test.fileConfig.srcPath).toString(); @@ -614,76 +613,15 @@ private List getExecCommand(LFTest test) { return getNonfederatedDockerExecCommand(test); } else if (relativePathName.equalsIgnoreCase(TestCategory.DOCKER_FEDERATED.getPath())) { return getFederatedDockerExecCommand(test); - } - - var binPath = test.fileConfig.binPath; - var binaryName = nameOnly; - - switch (test.target) { - case C: - case CPP: - case Rust: - case CCPP: { - if (test.target == Target.Rust) { - // rust binaries uses snake_case - binaryName = StringUtil.camelToSnakeCase(binaryName); - } - // Adjust binary extension if running on Window - if (System.getProperty("os.name").startsWith("Windows")) { - binaryName += ".exe"; - } - - var fullPath = binPath.resolve(binaryName); - if (Files.exists(fullPath)) { - // Running the command as .\binary.exe does not work on Windows for - // some reason... Thus we simply pass the full path here, which - // should work across all platforms - return Arrays.asList(new ProcessBuilder(fullPath.toString()).directory(binPath.toFile())); - } else { - test.issues.append(fullPath).append(": No such file or directory.").append(System.lineSeparator()); - test.result = Result.NO_EXEC_FAIL; - return new ArrayList<>(); - } - } - case Python: { - var fullPath = binPath.resolve(binaryName); - if (Files.exists(fullPath)) { - // If execution script exists, run it. - return Arrays.asList(new ProcessBuilder(fullPath.toString()).directory(binPath.toFile())); - } - fullPath = srcGenPath.resolve(nameOnly + ".py"); - if (Files.exists(fullPath)) { - return Arrays.asList(new ProcessBuilder("python3", fullPath.getFileName().toString()) - .directory(srcGenPath.toFile())); - } else { - test.result = Result.NO_EXEC_FAIL; - test.issues.append("File: ").append(fullPath).append(System.lineSeparator()); - return new ArrayList<>(); - } - } - case TS: { - // Adjust binary extension if running on Window - if (System.getProperty("os.name").startsWith("Windows")) { - binaryName += ".exe"; - } - var fullPath = binPath.resolve(binaryName); - if (Files.exists(fullPath)) { - // If execution script exists, run it. - return Arrays.asList(new ProcessBuilder(fullPath.toString()).directory(binPath.toFile())); - } - // If execution script does not exist, run .js directly. - var dist = test.fileConfig.getSrcGenPath().resolve("dist"); - var file = dist.resolve(nameOnly + ".js"); - if (Files.exists(file)) { - return Arrays.asList(new ProcessBuilder("node", file.toString())); - } else { + } else { + LFCommand command = generatorResult.getCommand(); + if (command == null) { test.result = Result.NO_EXEC_FAIL; - test.issues.append("File: ").append(file).append(System.lineSeparator()); - return new ArrayList<>(); + test.issues.append("File: ").append(generatorResult.getExecutable()).append(System.lineSeparator()); } - } - default: - throw new AssertionError("unreachable"); + return command == null ? List.of() : List.of( + new ProcessBuilder(command.command()).directory(command.directory()) + ); } } @@ -708,11 +646,12 @@ private void validateAndRun(Set tests, Configurator configurator, TestLe redirectOutputs(test); var context = configure(test, configurator, level); validate(test, context); + GeneratorResult result = GeneratorResult.NOTHING; if (level.compareTo(TestLevel.CODE_GEN) >= 0) { - generateCode(test); + result = generateCode(test); } if (level == TestLevel.EXECUTION) { - execute(test); + execute(test, result); } else if (test.result == Result.UNKNOWN) { test.result = Result.TEST_PASS; } diff --git a/org.lflang.tests/src/org/lflang/tests/lsp/ErrorInserter.java b/org.lflang.tests/src/org/lflang/tests/lsp/ErrorInserter.java new file mode 100644 index 0000000000..ebb131983d --- /dev/null +++ b/org.lflang.tests/src/org/lflang/tests/lsp/ErrorInserter.java @@ -0,0 +1,337 @@ +package org.lflang.tests.lsp; + +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Random; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Stream; + +import org.jetbrains.annotations.NotNull; + +import com.google.common.collect.ImmutableList; + +/** + * Insert problems into integration tests. + * + * @author Peter Donovan + */ +class ErrorInserter { + + /** A basic error inserter builder on which more specific error inserters can be built. */ + private static final Builder BASE_ERROR_INSERTER = new Builder() + .insertCondition(s -> Stream.of(";", "}", "{").anyMatch(s::endsWith)) + .insertable(" 0 = 1;").insertable("some_undeclared_var1524263 = 9;").insertable(" ++;"); + public static final Builder C = BASE_ERROR_INSERTER + .replacer("SET(", "UNDEFINED_NAME2828376(") + .replacer("schedule(", "undefined_name15291838("); + public static final Builder CPP = BASE_ERROR_INSERTER + .replacer(".get", ".undefined_name15291838") + .replacer("std::", "undefined_name3286634::"); + public static final Builder PYTHON_SYNTAX_ONLY = new Builder() + .insertable("+++++;").insertable(" .."); + public static final Builder PYTHON = PYTHON_SYNTAX_ONLY + .replacer("print(", "undefined_name15291838("); + public static final Builder RUST = BASE_ERROR_INSERTER + .replacer("println!", "undefined_name15291838!") + .replacer("ctx.", "undefined_name3286634."); + public static final Builder TYPESCRIPT = BASE_ERROR_INSERTER + .replacer("requestErrorStop(", "not_an_attribute_of_util9764(") + .replacer("const ", "var "); + + /** An {@code AlteredTest} represents an altered version of what was a valid LF file. */ + static class AlteredTest implements Closeable { + + /** A {@code OnceTrue} is randomly true once, and then never again. */ + private static class OnceTrue { + boolean beenTrue; + Random random; + private OnceTrue(Random random) { + this.beenTrue = false; + this.random = random; + } + private boolean get() { + if (beenTrue) return false; + return beenTrue = random.nextBoolean() && random.nextBoolean(); + } + } + + /** The zero-based indices of the touched lines. */ + private final List badLines; + /** The original test on which this is based. */ + private final Path path; + /** The content of this test. */ + private final LinkedList lines; + /** Whether the error inserter is permitted to insert a line before the current line. */ + private final Predicate> insertCondition; + + /** + * Initialize a possibly altered copy of {@code originalTest}. + * @param originalTest A path to an LF file that serves as a test. + * @param insertCondition Whether the error inserter is permitted to insert a line after a given line. + * @throws IOException if the content of {@code originalTest} cannot be read. + */ + private AlteredTest(Path originalTest, Predicate insertCondition) throws IOException { + this.badLines = new ArrayList<>(); + this.path = originalTest; + this.lines = new LinkedList<>(); // Constant-time insertion during iteration is desired. + this.lines.addAll(Files.readAllLines(originalTest)); + this.insertCondition = it -> { + boolean ret = true; + it.previous(); + if (it.hasPrevious()) { + ret = insertCondition.test(it.previous()); + } + it.next(); + it.next(); + return ret; + }; + } + + /** Return the location where the content of {@code this} lives. */ + public Path getPath() { + return path; + } + + /** + * Write the altered version of the test to the file system. + * @throws IOException If an I/O error occurred. + */ + public void write() throws IOException { + if (!path.toFile().renameTo(swapFile(path).toFile())) { + throw new IOException("Failed to create a swap file."); + } + try (PrintWriter writer = new PrintWriter(path.toFile())) { + lines.forEach(writer::println); + } + } + + /** + * Restore the file associated with this test to its original state. + */ + @Override + public void close() throws IOException { + if (!swapFile(path).toFile().exists()) throw new IllegalStateException("Swap file does not exist."); + if (!path.toFile().delete()) { + throw new IOException("Failed to delete the file associated with the original test."); + } + if (!swapFile(path).toFile().renameTo(path.toFile())) { + throw new IOException("Failed to restore the altered LF file to its original state."); + } + } + + /** Return the lines where this differs from the test from which it was derived. */ + public ImmutableList getBadLines() { + return ImmutableList.copyOf(badLines); + } + + /** + * Attempt to replace a line of this test with a different line of target language code. + * @param replacer A function that replaces lines of code with possibly different lines. + */ + public void replace(Function replacer, Random random) { + OnceTrue onceTrue = new OnceTrue(random); + alter((it, current) -> { + if (!onceTrue.get()) return false; + String newLine = replacer.apply(current); + it.remove(); + it.add(newLine); + return !newLine.equals(current); + }); + } + + /** + * Attempt to insert a new line of target language code into this test. + * @param line The line to be inserted. + */ + public void insert(String line, Random random) { + OnceTrue onceTrue = new OnceTrue(random); + alter((it, current) -> { + if (insertCondition.test(it) && onceTrue.get()) { + it.remove(); + it.add(line); + it.add(current); + return true; + } + return false; + }); + } + + /** + * Alter the content of this test. + * @param alterer A function whose first argument is an iterator over the lines of {@code this}, whose second + * argument is the line most recently returned by that iterator, and whose return value is + * whether an alteration was successfully performed. This function is only applied within + * multiline code blocks. + */ + private void alter(BiFunction, String, Boolean> alterer) { + ListIterator it = lines.listIterator(); + boolean inCodeBlock = false; + int lineNumber = 0; + while (it.hasNext()) { + String current = it.next(); + if (current.contains("=}")) inCodeBlock = false; + if (inCodeBlock && alterer.apply(it, current)) badLines.add(lineNumber); + if (current.contains("{=")) inCodeBlock = true; + if (current.contains("{=") && current.contains("=}")) { + inCodeBlock = current.lastIndexOf("{=") > current.lastIndexOf("=}"); + } + lineNumber++; + } + } + + /** Return the swap file associated with {@code f}. */ + private static Path swapFile(Path p) { + return p.getParent().resolve("." + p.getFileName() + ".swp"); + } + } + + /** A builder for an error inserter. */ + public static class Builder { + private static class Node implements Iterable { + private final Node previous; + private final T item; + private Node(Node previous, T item) { + this.previous = previous; + this.item = item; + } + + @NotNull + @Override + public Iterator iterator() { + NodeIterator ret = new NodeIterator<>(); + ret.current = this; + return ret; + } + + private static class NodeIterator implements Iterator { + private Node current; + + @Override + public boolean hasNext() { + return current != null; + } + + @Override + public T next() { + T ret = current.item; + current = current.previous; + return ret; + } + } + } + private final Node> replacers; + private final Node insertables; + private final Predicate insertCondition; + + /** Initializes a builder for error inserters. */ + public Builder() { + this(null, null, s -> true); + } + + /** Construct a builder with the given replacers and insertables. */ + private Builder( + Node> replacers, + Node insertables, + Predicate insertCondition + ) { + this.replacers = replacers; + this.insertables = insertables; + this.insertCondition = insertCondition; + } + + /** + * Record that the resulting {@code ErrorInserter} may replace {@code phrase} with {@code alternativePhrase}. + * @param phrase A phrase in target language code. + * @param alternativePhrase A phrase that {@code phrase} may be replaced with in order to introduce an error. + * @return A {@code Builder} that knows about all the edits that {@code this} knows about, plus the edit that + * replaces {@code phrase} with {@code alternativePhrase}. + */ + public Builder replacer(String phrase, String alternativePhrase) { + return new Builder( + new Node<>( + replacers, + line -> { + int changeableEnd = line.length(); + for (String bad : new String[]{"#", "//", "\""}) { + if (line.contains(bad)) changeableEnd = Math.min(changeableEnd, line.indexOf(bad)); + } + return line.substring(0, changeableEnd).replace(phrase, alternativePhrase) + + line.substring(changeableEnd); + } + ), + insertables, + insertCondition + ); + } + + /** Record that {@code} line may be inserted in order to introduce an error. */ + public Builder insertable(String line) { + return new Builder(replacers, new Node<>(insertables, line), insertCondition); + } + + /** + * Record that for any line X, insertCondition(X) is a necessary condition that a line may be inserted after X. + */ + public Builder insertCondition(Predicate insertCondition) { + return new Builder(replacers, insertables, insertCondition.and(insertCondition)); + } + + /** Get the error inserter generated by {@code this}. */ + public ErrorInserter get(Random random) { + return new ErrorInserter( + random, + replacers == null ? ImmutableList.of() : ImmutableList.copyOf(replacers), + insertables == null ? ImmutableList.of() : ImmutableList.copyOf(insertables), + insertCondition + ); + } + } + + private static final int MAX_ALTERATION_ATTEMPTS = 100; + + private final Random random; + private final ImmutableList> replacers; + private final ImmutableList insertables; + private final Predicate insertCondition; + + private ErrorInserter( + Random random, + ImmutableList> replacers, + ImmutableList insertables, + Predicate insertCondition + ) { + this.random = random; + this.replacers = replacers; + this.insertables = insertables; + this.insertCondition = insertCondition; + } + + /** + * Alter the given test and return the altered version. + * @param test An LF file that can be used as a test. + * @return An {@code AlteredTest} that is based on {@code test}. + */ + public AlteredTest alterTest(Path test) throws IOException { + AlteredTest alterable = new AlteredTest(test, insertCondition); + int remainingAlterationAttempts = MAX_ALTERATION_ATTEMPTS; + while (alterable.getBadLines().isEmpty() && remainingAlterationAttempts-- > 0) { + if (random.nextBoolean() && !replacers.isEmpty()) { + alterable.replace(replacers.get(random.nextInt(replacers.size())), random); + } else if (!insertables.isEmpty()) { + alterable.insert(insertables.get(random.nextInt(insertables.size())), random); + } + } + alterable.write(); + return alterable; + } +} diff --git a/org.lflang.tests/src/org/lflang/tests/lsp/LspTests.java b/org.lflang.tests/src/org/lflang/tests/lsp/LspTests.java new file mode 100644 index 0000000000..bcf176b6bc --- /dev/null +++ b/org.lflang.tests/src/org/lflang/tests/lsp/LspTests.java @@ -0,0 +1,237 @@ +package org.lflang.tests.lsp; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.StreamSupport; + +import org.eclipse.lsp4j.Diagnostic; +import org.eclipse.emf.common.util.URI; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.lflang.LFRuntimeModule; +import org.lflang.LFStandaloneSetup; +import org.lflang.Target; +import org.lflang.generator.GeneratorResult; +import org.lflang.generator.GeneratorResult.Status; +import org.lflang.generator.IntegratedBuilder; +import org.lflang.generator.LanguageServerErrorReporter; +import org.lflang.tests.LFTest; +import org.lflang.tests.TestRegistry; +import org.lflang.tests.TestRegistry.TestCategory; +import org.lflang.tests.lsp.ErrorInserter.AlteredTest; + +/** + * Test the code generator features that are required by the language server. + * + * @author Peter Donovan + */ +class LspTests { + + /** The {@code Random} whose initial state determines the behavior of the set of all {@code LspTests} instances. */ + private static final Random RANDOM = new Random(2101); + /** The test categories that should be excluded from LSP tests. */ + private static final TestCategory[] EXCLUDED_CATEGORIES = { + TestCategory.EXAMPLE, TestCategory.DOCKER, TestCategory.DOCKER_FEDERATED + }; + private static final Predicate> NOT_SUPPORTED = diagnosticsHaveKeyword("supported"); + private static final Predicate> MISSING_DEPENDENCY = diagnosticsHaveKeyword("libprotoc") + .or(diagnosticsHaveKeyword("protoc-c")).or(diagnosticsIncludeText("could not be found")); + + /** The {@code IntegratedBuilder} instance whose behavior is to be tested. */ + private static final IntegratedBuilder builder = new LFStandaloneSetup(new LFRuntimeModule()) + .createInjectorAndDoEMFRegistration().getInstance(IntegratedBuilder.class); + + @Test + void lspWithDependenciesTestC() { buildAndRunTest(Target.C); } + @Test + void lspWithDependenciesTestCpp() { buildAndRunTest(Target.CPP); } + @Test + void lspWithDependenciesTestPython() { buildAndRunTest(Target.Python); } + @Test + void lspWithDependenciesTestTypeScript() { buildAndRunTest(Target.TS); } + @Test + void lspWithDependenciesTestRust() { buildAndRunTest(Target.Rust); } + + /** Test for false negatives in Python syntax-only validation. */ + @Test + void pythonSyntaxOnlyValidationTest() throws IOException { + targetLanguageValidationTest(Target.Python, ErrorInserter.PYTHON_SYNTAX_ONLY.get(RANDOM)); + } + + /** Test for false negatives in C++ validation. */ + @Test + void cppValidationTest() throws IOException { + targetLanguageValidationTest(Target.CPP, ErrorInserter.CPP.get(RANDOM)); + } + + /** Test for false negatives in Python validation. */ + @Test + void pythonValidationTest() throws IOException { + targetLanguageValidationTest(Target.Python, ErrorInserter.PYTHON.get(RANDOM)); + } + + /** Test for false negatives in Rust validation. */ + @Test + void rustValidationTest() throws IOException { + targetLanguageValidationTest(Target.Rust, ErrorInserter.RUST.get(RANDOM)); + } + + /** Test for false negatives in TypeScript validation. */ + @Test + void typescriptValidationTest() throws IOException { + targetLanguageValidationTest(Target.TS, ErrorInserter.TYPESCRIPT.get(RANDOM)); + } + + /** + * Test for false negatives in the validation of LF files with target {@code target} that have errors inserted by + * {@code errorInserter}. + */ + private void targetLanguageValidationTest(Target target, ErrorInserter errorInserter) throws IOException { + checkDiagnostics( + target, + alteredTest -> MISSING_DEPENDENCY.or(diagnostics -> alteredTest.getBadLines().stream().allMatch( + badLine -> { + System.out.print("Expecting an error to be reported at line " + badLine + "..."); + boolean result = NOT_SUPPORTED.test(diagnostics) || diagnostics.stream().anyMatch( + diagnostic -> diagnostic.getRange().getStart().getLine() == badLine + ); + System.out.println(result ? " Success." : " but the expected error could not be found."); + return result; + } + )), + errorInserter + ); + } + + /** + * Verify that the diagnostics that result from fully validating tests associated with {@code target} satisfy + * {@code requirementGetter}. + * @param target Any target language. + * @param requirementGetter A map from altered tests to the requirements that diagnostics regarding those tests + * must meet. + * @param alterer The means of inserting problems into the tests, or {@code null} if problems are not to be + * inserted. + * @throws IOException upon failure to write an altered copy of some test to storage. + */ + private void checkDiagnostics( + Target target, + Function>> requirementGetter, + ErrorInserter alterer + ) throws IOException { + MockLanguageClient client = new MockLanguageClient(); + LanguageServerErrorReporter.setClient(client); + for (LFTest test : allTests(target)) { + client.clearDiagnostics(); + if (alterer != null) { + try (AlteredTest altered = alterer.alterTest(test.srcFile)) { + runTest(altered.getPath(), false); + Assertions.assertTrue(requirementGetter.apply(altered).test(client.getReceivedDiagnostics())); + } + } else { + runTest(test.srcFile, false); + Assertions.assertTrue(requirementGetter.apply(null).test(client.getReceivedDiagnostics())); + } + } + } + + /** Test the "Build and Run" functionality of the language server. */ + private void buildAndRunTest(Target target) { + MockLanguageClient client = new MockLanguageClient(); + LanguageServerErrorReporter.setClient(client); + for (LFTest test : selectTests(target)) { + MockReportProgress reportProgress = new MockReportProgress(); + GeneratorResult result = runTest(test.srcFile, true); + if (NOT_SUPPORTED.or(MISSING_DEPENDENCY).test(client.getReceivedDiagnostics())) { + System.err.println("WARNING: Skipping \"Build and Run\" test due to lack of support or a missing " + + "requirement."); + } else { + Assertions.assertFalse(reportProgress.failed()); + Assertions.assertEquals(Status.COMPILED, result.getStatus()); + Assertions.assertNotNull(result.getCommand()); + Assertions.assertEquals(result.getCommand().run(), 0); + } + } + } + + /** + * Select {@code count} tests from each test category. + * @param target The target language of the desired tests. + * @return A sample of one integration test per target, per category. + */ + private Set selectTests(Target target) { + Set ret = new HashSet<>(); + for (TestCategory category : selectedCategories()) { + Set registeredTests = TestRegistry.getRegisteredTests(target, category, false); + if (registeredTests.size() == 0) continue; + int relativeIndex = RANDOM.nextInt(registeredTests.size()); + for (LFTest t : registeredTests) { + if (relativeIndex-- == 0) { + ret.add(t); + break; + } + } + } + return ret; + } + + /** Return all non-excluded tests whose target language is {@code target}. */ + private Set allTests(Target target) { + return StreamSupport.stream(selectedCategories().spliterator(), false) + .map(category -> TestRegistry.getRegisteredTests(target, category, false)) + .collect(HashSet::new, HashSet::addAll, HashSet::addAll); + } + + /** Return the non-excluded categories. */ + private Iterable selectedCategories() { + return () -> Arrays.stream(TestCategory.values()).filter( + category -> Arrays.stream(EXCLUDED_CATEGORIES).noneMatch(category::equals) + ).iterator(); + } + + /** + * Returns the predicate that a list of diagnostics contains the given keyword. + * @param keyword A keyword that a list of diagnostics should be searched for. + * @return The predicate, "X mentions {@code keyword}." + */ + private static Predicate> diagnosticsHaveKeyword(String keyword) { + return diagnostics -> diagnostics.stream().anyMatch( + d -> Arrays.asList(d.getMessage().toLowerCase().split("\\b")).contains(keyword) + ); + } + + /** + * Returns the predicate that a list of diagnostics contains the given text. + * @param requiredText A keyword that a list of diagnostics should be searched for. + * @return The predicate, "X includes {@code requiredText}." + */ + private static Predicate> diagnosticsIncludeText(String requiredText) { + return diagnostics -> diagnostics.stream().anyMatch( + d -> d.getMessage().toLowerCase().contains(requiredText) + ); + } + + /** + * Run the given test. + * @param test An integration test. + * @param mustComplete Whether the build must be complete. + * @return The result of running the test. + */ + private GeneratorResult runTest(Path test, boolean mustComplete) { + MockReportProgress reportProgress = new MockReportProgress(); + GeneratorResult result = builder.run( + URI.createFileURI(test.toString()), + mustComplete, reportProgress, + () -> false + ); + Assertions.assertFalse(reportProgress.failed()); + return result; + } +} diff --git a/org.lflang.tests/src/org/lflang/tests/lsp/MockLanguageClient.java b/org.lflang.tests/src/org/lflang/tests/lsp/MockLanguageClient.java new file mode 100644 index 0000000000..3e9ade8f14 --- /dev/null +++ b/org.lflang.tests/src/org/lflang/tests/lsp/MockLanguageClient.java @@ -0,0 +1,68 @@ +package org.lflang.tests.lsp; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.eclipse.lsp4j.Diagnostic; +import org.eclipse.lsp4j.DiagnosticSeverity; +import org.eclipse.lsp4j.MessageActionItem; +import org.eclipse.lsp4j.MessageParams; +import org.eclipse.lsp4j.PublishDiagnosticsParams; +import org.eclipse.lsp4j.ShowMessageRequestParams; +import org.eclipse.lsp4j.services.LanguageClient; + +/** + * A {@code MockLanguageClient} is a language client that should be used in language server tests. + * + * @author Peter Donovan + */ +public class MockLanguageClient implements LanguageClient { + + private List receivedDiagnostics = new ArrayList<>(); + + @Override + public void telemetryEvent(Object object) { + // Do nothing. + } + + @Override + public void publishDiagnostics(PublishDiagnosticsParams diagnostics) { + receivedDiagnostics.addAll(diagnostics.getDiagnostics()); + for (Diagnostic d : diagnostics.getDiagnostics()) { + ( + (d.getSeverity() == DiagnosticSeverity.Error || d.getSeverity() == DiagnosticSeverity.Warning) ? + System.err : System.out + ).println( + "Test client received diagnostic at line " + d.getRange().getStart().getLine() + ": " + d.getMessage() + ); + } + } + + @Override + public void showMessage(MessageParams messageParams) { + System.out.println("Test client received message: " + messageParams.getMessage()); + } + + @Override + public CompletableFuture showMessageRequest(ShowMessageRequestParams requestParams) { + showMessage(requestParams); + return null; + } + + @Override + public void logMessage(MessageParams message) { + showMessage(message); + } + + /** Return the diagnostics that {@code this} has received. */ + public List getReceivedDiagnostics() { + return Collections.unmodifiableList(receivedDiagnostics); + } + + /** Clear the diagnostics recorded by {@code this}. */ + public void clearDiagnostics() { + receivedDiagnostics.clear(); + } +} diff --git a/org.lflang.tests/src/org/lflang/tests/lsp/MockReportProgress.java b/org.lflang.tests/src/org/lflang/tests/lsp/MockReportProgress.java new file mode 100644 index 0000000000..bc2f4de3ea --- /dev/null +++ b/org.lflang.tests/src/org/lflang/tests/lsp/MockReportProgress.java @@ -0,0 +1,33 @@ +package org.lflang.tests.lsp; + +import org.lflang.generator.IntegratedBuilder; + +/** + * Collect progress reports and check that they have the expected properties. + * + * @author Peter Donovan + */ +public class MockReportProgress implements IntegratedBuilder.ReportProgress { + private int previousPercentProgress; + private boolean failed; + public MockReportProgress() { + previousPercentProgress = 0; + failed = false; + } + + @Override + public void apply(String message, Integer percentage) { + System.out.println(message); + if (percentage == null) return; + if (percentage < previousPercentProgress || percentage < 0 || percentage > 100) failed = true; + previousPercentProgress = percentage; + } + + /** + * Returns whether an invalid sequence of progress reports was received. + * @return whether an invalid sequence of progress reports was received + */ + public boolean failed() { + return failed; + } +} diff --git a/org.lflang/META-INF/MANIFEST.MF b/org.lflang/META-INF/MANIFEST.MF index 9f79e44af7..be7d48d067 100644 --- a/org.lflang/META-INF/MANIFEST.MF +++ b/org.lflang/META-INF/MANIFEST.MF @@ -19,7 +19,10 @@ Require-Bundle: org.eclipse.xtext, org.eclipse.xtend.lib.macro, org.apache.commons.cli;bundle-version="1.4", org.jetbrains.kotlin.bundled-compiler;resolution:=optional, - org.eclipse.lsp4j;bundle-version="0.10.0" + org.eclipse.lsp4j;bundle-version="0.10.0", + com.fasterxml.jackson.core.jackson-core, + com.fasterxml.jackson.core.jackson-annotations, + com.fasterxml.jackson.core.jackson-databind Bundle-RequiredExecutionEnvironment: JavaSE-11 Export-Package: org.lflang, org.lflang.generator, diff --git a/org.lflang/build.gradle b/org.lflang/build.gradle index 6898374ecc..d510254d3a 100644 --- a/org.lflang/build.gradle +++ b/org.lflang/build.gradle @@ -9,7 +9,9 @@ dependencies { implementation group: 'org.eclipse.emf', name: 'org.eclipse.emf.mwe2.launch', version: '2.12.1' // https://mvnrepository.com/artifact/org.eclipse.lsp4j/org.eclipse.lsp4j implementation group: 'org.eclipse.lsp4j', name: 'org.eclipse.lsp4j', version: '0.10.0' - + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.4' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.12.4' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.4' } configurations { diff --git a/org.lflang/src/lib/ts/.eslintrc.json b/org.lflang/src/lib/ts/.eslintrc.json new file mode 100644 index 0000000000..5cf28e0201 --- /dev/null +++ b/org.lflang/src/lib/ts/.eslintrc.json @@ -0,0 +1,15 @@ +{ + "parser": "@typescript-eslint/parser", + "plugins": [ + "@typescript-eslint" + ], + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/eslint-recommended", + "plugin:@typescript-eslint/recommended" + ], + "rules": { + "prefer-const": "warn", + "@typescript-eslint/no-inferrable-types": "warn" + } +} diff --git a/org.lflang/src/lib/ts/package.json b/org.lflang/src/lib/ts/package.json index 367a812ab7..39d8c64e52 100644 --- a/org.lflang/src/lib/ts/package.json +++ b/org.lflang/src/lib/ts/package.json @@ -26,6 +26,9 @@ "@babel/preset-typescript": "^7.8.3", "@types/google-protobuf": "^3.7.4", "@types/node": "^13.9.2", + "@typescript-eslint/eslint-plugin": "^5.8.1", + "@typescript-eslint/parser": "^5.8.1", + "eslint": "^8.5.0", "rimraf": "^3.0.2", "typescript": "^3.8.3", "ts-protoc-gen": "^0.12.0" diff --git a/org.lflang/src/org/lflang/ASTUtils.xtend b/org.lflang/src/org/lflang/ASTUtils.xtend index 7d9b3d8ce6..47180e25b4 100644 --- a/org.lflang/src/org/lflang/ASTUtils.xtend +++ b/org.lflang/src/org/lflang/ASTUtils.xtend @@ -575,6 +575,19 @@ class ASTUtils { * @return Textual representation of the given argument. */ def static String toText(Code code) { + return CodeMap.Correspondence.tag(code, toUntaggedText(code), true) + } + + /** + * Translate the given code into its textual representation + * without any {@code CodeMap.Correspondence} tags inserted. + * @param code AST node to render as string. + * @return Textual representation of the given argument. + */ + def private static String toUntaggedText(Code code) { + // FIXME: This function should not be necessary, but it is because we currently inspect the + // content of code blocks in the validator and generator (using regexes, etc.). See #810, #657. + var text = "" if (code !== null) { val node = NodeModelUtils.getNode(code) if (node !== null) { @@ -591,35 +604,17 @@ class ASTUtils { str = str.substring(start + 2, end) if (str.split('\n').length > 1) { // multi line code - return str.trimCodeBlock + text = str.trimCodeBlock } else { // single line code - return str.trim - } + text = str.trim + } } else if (code.body !== null) { // Code must have been added as a simple string. - return code.body.toString + text = code.body.toString } } - return "" - } - - /** - * Translate the given code into its textual representation, - * with a tag inserted to establish this representation's - * correspondence to the generated code. This tag is an - * implementation detail that is internal to the code - * generator. - * @param code the AST node to render as a string - * @return a textual representation of {@code code} - */ - def static String toTaggedText(Code code) { - // FIXME: Duplicates work already done in - // GeneratorBase::prSourceLineNumber. It does not - // make sense for both methods to persist in the - // code base at once. - val text = toText(code) - return CodeMap.Correspondence.tag(code, text, true) + return text } def static toText(TypeParm t) { @@ -683,10 +678,6 @@ class ASTUtils { // extract the whitespace prefix prefix = line.substring(0, firstCharacter) - } else if(!first) { - // Do not remove blank lines. They throw off #line directives. - buffer.append(line) - buffer.append('\n') } } first = false @@ -910,7 +901,7 @@ class ASTUtils { } def static boolean isZero(Code code) { - if (code !== null && code.toText.isZero) { + if (code !== null && code.toUntaggedText.isZero) { return true } return false @@ -951,7 +942,7 @@ class ASTUtils { * @return True if the given code is an integer, false otherwise. */ def static boolean isInteger(Code code) { - return code.toText.isInteger + return code.toUntaggedText.isInteger } /** diff --git a/org.lflang/src/org/lflang/AstExtensions.kt b/org.lflang/src/org/lflang/AstExtensions.kt index 6310f0ab31..ceb9184967 100644 --- a/org.lflang/src/org/lflang/AstExtensions.kt +++ b/org.lflang/src/org/lflang/AstExtensions.kt @@ -144,13 +144,6 @@ val StateVar.isOfTimeType: Boolean get() = JavaAstUtils.isOfTimeType(this) */ fun Code.toText(): String = ASTUtils.toText(this) -/** - * Translate this code element into its tagged textual - * representation. - * @see ASTUtils.toTaggedText - */ -fun Code.toTaggedText(): String = ASTUtils.toTaggedText(this) - /** * Translate this code element into its textual representation. * @see ASTUtils.toText diff --git a/org.lflang/src/org/lflang/ErrorReporter.java b/org.lflang/src/org/lflang/ErrorReporter.java index 29dc304fa4..a80cae75d2 100644 --- a/org.lflang/src/org/lflang/ErrorReporter.java +++ b/org.lflang/src/org/lflang/ErrorReporter.java @@ -3,6 +3,9 @@ import java.nio.file.Path; import org.eclipse.emf.ecore.EObject; +import org.eclipse.lsp4j.DiagnosticSeverity; + +import org.lflang.generator.Position; /** * Interface for reporting errors. @@ -55,7 +58,7 @@ public interface ErrorReporter { * Report an error at the specified line within a file. * * @param message The error message. - * @param line The line number to report at. + * @param line The one-based line number to report at. * @param file The file to report at. * @return a string that describes the error. */ @@ -66,12 +69,62 @@ public interface ErrorReporter { * Report a warning at the specified line within a file. * * @param message The error message. - * @param line The line number to report at. + * @param line The one-based line number to report at. * @param file The file to report at. * @return a string that describes the warning. */ String reportWarning(Path file, Integer line, String message); + /** + * Report a message of severity {@code severity}. + * @param file The file to which the message pertains, or {@code null} if the file is unknown. + * @param severity the severity of the message + * @param message the message to send to the IDE + * @return a string that describes the diagnostic + */ + default String report(Path file, DiagnosticSeverity severity, String message) { + switch (severity) { + case Error: + return reportError(message); + case Warning: + case Hint: + case Information: // FIXME: Information -> warning?? If this results in false alarms from LFC, we should expand API. + default: + return reportWarning(message); + } + } + + /** + * Report a message of severity {@code severity} that + * pertains to line {@code line} of an LF source file. + * @param file The file to which the message pertains, or {@code null} if the file is unknown. + * @param severity the severity of the message + * @param message the message to send to the IDE + * @param line the one-based line number associated + * with the message + * @return a string that describes the diagnostic + */ + default String report(Path file, DiagnosticSeverity severity, String message, int line) { + return report(file, severity, message); + } + + /** + * Report a message of severity {@code severity} that + * pertains to the range [{@code startPos}, {@code endPos}) + * of an LF source file. + * @param file The file to which the message pertains, or {@code null} if the file is unknown. + * @param severity the severity of the message + * @param message the message to send to the IDE + * @param startPos the position of the first character + * of the range of interest + * @param endPos the position immediately AFTER the + * final character of the range of + * interest + * @return a string that describes the diagnostic + */ + default String report(Path file, DiagnosticSeverity severity, String message, Position startPos, Position endPos) { + return report(file, severity, message); + } /** * Check if errors where reported. diff --git a/org.lflang/src/org/lflang/generator/ActionInstance.java b/org.lflang/src/org/lflang/generator/ActionInstance.java index 8e7a79b77d..fe8fef31d5 100644 --- a/org.lflang/src/org/lflang/generator/ActionInstance.java +++ b/org.lflang/src/org/lflang/generator/ActionInstance.java @@ -26,11 +26,9 @@ STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY package org.lflang.generator; -import org.lflang.JavaAstUtils; import org.lflang.TimeValue; import org.lflang.lf.Action; import org.lflang.lf.ActionOrigin; -import org.lflang.lf.Parameter; /** * Instance of an action. diff --git a/org.lflang/src/org/lflang/generator/CodeMap.java b/org.lflang/src/org/lflang/generator/CodeMap.java index b4956ebbd3..541135de91 100644 --- a/org.lflang/src/org/lflang/generator/CodeMap.java +++ b/org.lflang/src/org/lflang/generator/CodeMap.java @@ -30,7 +30,7 @@ public static class Correspondence { "/\\*Correspondence: (?%s) \\-> (?%s) \\(src=(?%s)\\)\\*/", Position.removeNamedCapturingGroups(Range.PATTERN), Position.removeNamedCapturingGroups(Range.PATTERN), - ".*" + ".*?" )); // TODO(peter): Add "private final boolean verbatim;" and make corresponding enhancements @@ -153,21 +153,12 @@ public static String tag(EObject astNode, String representation, boolean verbati Position lfStart = Position.fromOneBased( oneBasedLfLineAndColumn.getLine(), oneBasedLfLineAndColumn.getColumn() ); - Position lfDisplacement; - final Position generatedCodeDisplacement = Position.displacementOf(representation); final Path lfPath = Path.of(astNode.eResource().getURI().path()); - if (verbatim) { - lfStart = lfStart.plus(Position.displacementOf( - node.getText().substring(0, indexOf(node.getText(), representation)) - )); - lfDisplacement = generatedCodeDisplacement; - } else { - lfDisplacement = Position.displacementOf(node.getText()); - } + if (verbatim) lfStart = lfStart.plus(node.getText().substring(0, indexOf(node.getText(), representation))); return new Correspondence( lfPath, - new Range(lfStart, lfStart.plus(lfDisplacement)), - new Range(Position.ORIGIN, generatedCodeDisplacement) + new Range(lfStart, lfStart.plus(verbatim ? representation : node.getText())), + new Range(Position.ORIGIN, Position.displacementOf(representation)) ) + representation; } diff --git a/org.lflang/src/org/lflang/generator/CommandErrorReportingStrategy.java b/org.lflang/src/org/lflang/generator/CommandErrorReportingStrategy.java deleted file mode 100644 index 952eded4ab..0000000000 --- a/org.lflang/src/org/lflang/generator/CommandErrorReportingStrategy.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.lflang.generator; - -import java.nio.file.Path; -import java.util.Map; - -import org.lflang.ErrorReporter; - -/** - * Represents a strategy for parsing the output of a - * validator. - */ -@FunctionalInterface public interface CommandErrorReportingStrategy { - // Note: The expected use case is to parse the output of - // a target language validator that is run in a separate - // process. - // Note: This replaces the method GeneratorBase::reportCommandErrors, - // which uses the template design pattern. The justification is that - // 1. Not all validator output fits the template. In particular, ESLint - // uses JSON. It is also possible that discretion should be allowed - // on whether the entire validation message should be sent to the - // error reporter vs. just the first line of it, and that is why - // the implementation of GeneratorBase::reportCommandErrors is unsuitable - // for use of GCC with an IDE. There is also the inconvenience that - // different tools make inconsistent choices about what goes to stderr - // vs. stdout. - // 2. Using composition for code reuse in this way may lead to smaller, - // less tightly coupled modules than if all of the shared functionality - // of the code generators is inherited from GeneratorBase. - // FIXME: In order to avoid redundancy, it is necessary to - // delete either this interface or the methods - // GeneratorBase::reportCommandErrors and - // GeneratorBase::parseCommandOutput. - - /** - * Parses the validation output and reports any errors - * that it contains. - * @param validationOutput any validation output - * @param errorReporter any error reporter - * @param map the map from generated files to CodeMaps - */ - void report(String validationOutput, ErrorReporter errorReporter, Map map); -} diff --git a/org.lflang/src/org/lflang/generator/DiagnosticReporting.java b/org.lflang/src/org/lflang/generator/DiagnosticReporting.java new file mode 100644 index 0000000000..0beae982dc --- /dev/null +++ b/org.lflang/src/org/lflang/generator/DiagnosticReporting.java @@ -0,0 +1,60 @@ +package org.lflang.generator; + +import java.nio.file.Path; +import java.util.Map; + +import org.eclipse.lsp4j.DiagnosticSeverity; + +import org.lflang.ErrorReporter; + +/** + * {@code DiagnosticReporting} provides utilities for + * reporting validation output. + * + * @author Peter Donovan + */ +public class DiagnosticReporting { + + private DiagnosticReporting() { + // utility class + } + + /** + * A means of parsing the output of a validator. + */ + @FunctionalInterface public interface Strategy { + /** + * Parse the validation output and report any errors + * that it contains. + * @param validationOutput any validation output + * @param errorReporter any error reporter + * @param map the map from generated files to CodeMaps + */ + void report(String validationOutput, ErrorReporter errorReporter, Map map); + } + + /** + * Format the given data as a human-readable message. + * @param message An error message. + * @param path The path of the source of the message. + * @param position The position where the message originates. + * @return The given data as a human-readable message. + */ + public static String messageOf(String message, Path path, Position position) { + return String.format("%s [%s:%s:%s]", message, path.getFileName().toString(), position.getOneBasedLine(), position.getOneBasedColumn()); + } + + /** + * Convert {@code severity} into a {@code DiagnosticSeverity} using a heuristic that should be + * compatible with many tools. + * @param severity The string representation of a diagnostic severity. + * @return The {@code DiagnosticSeverity} representation of {@code severity}. + */ + public static DiagnosticSeverity severityOf(String severity) { + severity = severity.toLowerCase(); + if (severity.contains("error")) return DiagnosticSeverity.Error; + else if (severity.contains("warning")) return DiagnosticSeverity.Warning; + else if (severity.contains("hint") || severity.contains("help")) return DiagnosticSeverity.Hint; + else return DiagnosticSeverity.Information; + } +} diff --git a/org.lflang/src/org/lflang/generator/GeneratorBase.xtend b/org.lflang/src/org/lflang/generator/GeneratorBase.xtend index 9818ae1aad..d1e59e548a 100644 --- a/org.lflang/src/org/lflang/generator/GeneratorBase.xtend +++ b/org.lflang/src/org/lflang/generator/GeneratorBase.xtend @@ -469,9 +469,9 @@ abstract class GeneratorBase extends AbstractLFValidator implements TargetTypes // the definition of `Foo`. this.reactors = this.instantiationGraph.nodesInTopologicalOrder - // If there is no main reactor, then make sure the reactors list includes - // even reactors that are not instantiated anywhere. - if (mainDef === null) { + // If there is no main reactor or if all reactors in the file need to be validated, then make sure the reactors + // list includes even reactors that are not instantiated anywhere. + if (mainDef === null || fileConfig.context.mode == Mode.LSP_MEDIUM) { for (r : fileConfig.resource.allContents.toIterable.filter(Reactor)) { if (!this.reactors.contains(r)) { this.reactors.add(r); @@ -1205,9 +1205,9 @@ abstract class GeneratorBase extends AbstractLFValidator implements TargetTypes // statements to find which one matches and mark all the // import statements down the chain. But what a pain! if (severity == IMarker.SEVERITY_ERROR) { - errorReporter.reportError(originalPath, 0, "Error in imported file: " + path) + errorReporter.reportError(originalPath, 1, "Error in imported file: " + path) } else { - errorReporter.reportWarning(originalPath, 0, "Warning in imported file: " + path) + errorReporter.reportWarning(originalPath, 1, "Warning in imported file: " + path) } } } @@ -1257,9 +1257,9 @@ abstract class GeneratorBase extends AbstractLFValidator implements TargetTypes // statements to find which one matches and mark all the // import statements down the chain. But what a pain! if (severity == IMarker.SEVERITY_ERROR) { - errorReporter.reportError(originalPath, 0, "Error in imported file: " + path) + errorReporter.reportError(originalPath, 1, "Error in imported file: " + path) } else { - errorReporter.reportWarning(originalPath, 0, "Warning in imported file: " + path) + errorReporter.reportWarning(originalPath, 1, "Warning in imported file: " + path) } } } diff --git a/org.lflang/src/org/lflang/generator/GeneratorResult.java b/org.lflang/src/org/lflang/generator/GeneratorResult.java index 0c2a375692..0f28c3f018 100644 --- a/org.lflang/src/org/lflang/generator/GeneratorResult.java +++ b/org.lflang/src/org/lflang/generator/GeneratorResult.java @@ -26,8 +26,8 @@ public class GeneratorResult { public enum Status { NOTHING(result -> ""), // Code generation was not performed. CANCELLED(result -> "Code generation was cancelled."), - FAILED(result -> ""), // This may be due to a failed validation check, in which case the problem will be displayed - // in the editor. This makes a message unnecessary. + FAILED(result -> ""), // This may be due to a failed validation check, in which case the error should have been + // sent to the error reporter and handled there. This makes a message unnecessary. GENERATED(GetUserMessage.COMPLETED), COMPILED(GetUserMessage.COMPLETED); @@ -48,7 +48,7 @@ public interface GetUserMessage { } /** The {@code GetUserMessage} associated with this {@code Status}. */ - public final GetUserMessage gum; + private final GetUserMessage gum; /** Initializes a {@code Status} whose {@code GetUserMessage} is {@code gum}. */ Status(GetUserMessage gum) { @@ -97,6 +97,11 @@ public LFCommand getCommand() { return command; } + /** Return the exectuable produced by this build, or {@code null} if none exists. */ + public Path getExecutable() { + return executable; + } + /** * Return a message that can be relayed to the end user about this * {@code GeneratorResult}. diff --git a/org.lflang/src/org/lflang/generator/HumanReadableReportingStrategy.java b/org.lflang/src/org/lflang/generator/HumanReadableReportingStrategy.java new file mode 100644 index 0000000000..95617d6e8b --- /dev/null +++ b/org.lflang/src/org/lflang/generator/HumanReadableReportingStrategy.java @@ -0,0 +1,176 @@ +package org.lflang.generator; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.lsp4j.DiagnosticSeverity; +import org.eclipse.xtext.xbase.lib.Procedures.Procedure0; +import org.eclipse.xtext.xbase.lib.Procedures.Procedure2; + +import org.lflang.ErrorReporter; + +/** + * An error reporting strategy that parses human-readable + * output. + * + * @author Peter Donovan + */ +public class HumanReadableReportingStrategy implements DiagnosticReporting.Strategy { + + /** A pattern that matches lines that should be reported via this strategy. */ + private final Pattern diagnosticMessagePattern; + /** A pattern that matches labels that show the exact range to which the diagnostic pertains. */ + private final Pattern labelPattern; + /** The path against which any paths should be resolved. */ + private final Path relativeTo; + /** The next line to be processed, or {@code null}. */ + private String bufferedLine; + + /** + * Instantiate a reporting strategy for lines of + * validator output that match {@code diagnosticMessagePattern}. + * @param diagnosticMessagePattern A pattern that matches lines that should be + * reported via this strategy. This pattern + * must contain named capturing groups called + * "path", "line", "column", "message", and + * "severity". + * @param labelPattern A pattern that matches lines that act as labels, showing + * the location of the relevant piece of text. This pattern + * must contain two groups, the first of which must match + * characters that precede the location given by the "line" + * and "column" groups. + */ + public HumanReadableReportingStrategy(Pattern diagnosticMessagePattern, Pattern labelPattern) { + this(diagnosticMessagePattern, labelPattern, null); + } + + /** + * Instantiate a reporting strategy for lines of + * validator output that match {@code diagnosticMessagePattern}. + * @param diagnosticMessagePattern a pattern that matches lines that should be + * reported via this strategy. This pattern + * must contain named capturing groups called + * "path", "line", "column", "message", and + * "severity". + * @param labelPattern A pattern that matches lines that act as labels, showing + * the location of the relevant piece of text. This pattern + * must contain two groups, the first of which must match + * characters that precede the location given by the "line" + * and "column" groups. + * @param relativeTo The path against which any paths should be resolved. + */ + public HumanReadableReportingStrategy(Pattern diagnosticMessagePattern, Pattern labelPattern, Path relativeTo) { + for (String groupName : new String[]{"path", "line", "column", "message", "severity"}) { + assert diagnosticMessagePattern.pattern().contains(groupName) : String.format( + "Error line patterns must have a named capturing group called %s", groupName + ); + } + this.diagnosticMessagePattern = diagnosticMessagePattern; + this.labelPattern = labelPattern; + this.relativeTo = relativeTo; + this.bufferedLine = null; + } + + @Override + public void report(String validationOutput, ErrorReporter errorReporter, Map map) { + Iterator it = validationOutput.lines().iterator(); + while (it.hasNext() || bufferedLine != null) { + if (bufferedLine != null) { + reportErrorLine(bufferedLine, it, errorReporter, map); + bufferedLine = null; + } else { + reportErrorLine(it.next(), it, errorReporter, map); + } + } + } + + /** + * Report the validation message contained in the given line of text. + * @param line The current line. + * @param it An iterator over the lines that follow the current line. + * @param errorReporter An arbitrary ErrorReporter. + * @param maps A mapping from generated file paths to + * CodeMaps. + */ + private void reportErrorLine(String line, Iterator it, ErrorReporter errorReporter, Map maps) { + Matcher matcher = diagnosticMessagePattern.matcher(stripEscaped(line)); + if (matcher.matches()) { + final Path path = Paths.get(matcher.group("path")); + final Position generatedFilePosition = Position.fromOneBased( + Integer.parseInt(matcher.group("line")), + Integer.parseInt(matcher.group("column") != null ? matcher.group("column") : "0") // FIXME: Unreliable heuristic + ); + final String message = DiagnosticReporting.messageOf( + matcher.group("message"), path, generatedFilePosition + ); + final CodeMap map = maps.get(relativeTo != null ? relativeTo.resolve(path) : path); + final DiagnosticSeverity severity = DiagnosticReporting.severityOf(matcher.group("severity")); + if (map == null) { + errorReporter.report(null, severity, message); + return; + } + for (Path srcFile : map.lfSourcePaths()) { + Position lfFilePosition = map.adjusted(srcFile, generatedFilePosition); + if (matcher.group("column") != null) { + reportAppropriateRange( + (p0, p1) -> errorReporter.report(srcFile, severity, message, p0, p1), lfFilePosition, it + ); + } else { + errorReporter.report(srcFile, severity, message, lfFilePosition.getOneBasedLine()); + } + } + } + } + + /** + * Report the appropriate range to {@code report}. + * @param report A reporting method whose first and + * second parameters are the (included) + * start and (excluded) end of the + * relevant range. + * @param lfFilePosition The point about which the + * relevant range is anchored. + * @param it An iterator over the lines immediately + * following a diagnostic message. + */ + private void reportAppropriateRange( + Procedure2 report, Position lfFilePosition, Iterator it + ) { + Procedure0 failGracefully = () -> report.apply(lfFilePosition, lfFilePosition.plus(" ")); + if (!it.hasNext()) { + failGracefully.apply(); + return; + } + String line = it.next(); + Matcher labelMatcher = labelPattern.matcher(line); + if (labelMatcher.find()) { + report.apply( + Position.fromZeroBased( + lfFilePosition.getZeroBasedLine(), + lfFilePosition.getZeroBasedColumn() - labelMatcher.group(1).length() + ), + lfFilePosition.plus(labelMatcher.group(2)) + ); + return; + } + if (diagnosticMessagePattern.matcher(line).find()) { + failGracefully.apply(); + bufferedLine = line; + return; + } + reportAppropriateRange(report, lfFilePosition, it); + } + + /** + * Strip the ANSI escape sequences from {@code s}. + * @param s Any string. + * @return {@code s}, with any escape sequences removed. + */ + private static String stripEscaped(String s) { + return s.replaceAll("\u001B\\[[;\\d]*[ -/]*[@-~]", ""); + } +} diff --git a/org.lflang/src/org/lflang/generator/JavaGeneratorUtils.java b/org.lflang/src/org/lflang/generator/JavaGeneratorUtils.java index a667160e46..ff24cbcd50 100644 --- a/org.lflang/src/org/lflang/generator/JavaGeneratorUtils.java +++ b/org.lflang/src/org/lflang/generator/JavaGeneratorUtils.java @@ -31,4 +31,9 @@ public static void writeSourceCodeToFile(CharSequence code, String path) throws } } } + + /** Return whether the operating system is Windows. */ + public static boolean isHostWindows() { + return System.getProperty("os.name").toLowerCase().contains("win"); + } } diff --git a/org.lflang/src/org/lflang/generator/LFGeneratorContext.java b/org.lflang/src/org/lflang/generator/LFGeneratorContext.java index bf623d26d7..f5a5b1763d 100644 --- a/org.lflang/src/org/lflang/generator/LFGeneratorContext.java +++ b/org.lflang/src/org/lflang/generator/LFGeneratorContext.java @@ -89,12 +89,13 @@ default void finish( Map codeMaps, String interpreter ) { + final boolean isWindows = JavaGeneratorUtils.isHostWindows(); if (execName != null && binPath != null) { - Path executable = binPath.resolve(execName); + Path executable = binPath.resolve(execName + (isWindows && interpreter == null ? ".exe" : "")); String relativeExecutable = fileConfig.srcPkgPath.relativize(executable).toString(); LFCommand command = interpreter != null ? LFCommand.get(interpreter, List.of(relativeExecutable), fileConfig.srcPkgPath) : - LFCommand.get(relativeExecutable, List.of(), fileConfig.srcPkgPath); + LFCommand.get(isWindows ? executable.toString() : relativeExecutable, List.of(), fileConfig.srcPkgPath); finish(new GeneratorResult(status, executable, command, codeMaps)); } else { finish(new GeneratorResult(status, null, null, codeMaps)); @@ -122,7 +123,10 @@ default void finish( * Conclude this build and record that it was unsuccessful. */ default void unsuccessfulFinish() { - finish(getCancelIndicator().isCanceled() ? GeneratorResult.CANCELLED : GeneratorResult.FAILED); + finish( + getCancelIndicator() != null && getCancelIndicator().isCanceled() ? + GeneratorResult.CANCELLED : GeneratorResult.FAILED + ); } /** diff --git a/org.lflang/src/org/lflang/generator/LanguageServerErrorReporter.java b/org.lflang/src/org/lflang/generator/LanguageServerErrorReporter.java index d5adbad398..2cf81800c6 100644 --- a/org.lflang/src/org/lflang/generator/LanguageServerErrorReporter.java +++ b/org.lflang/src/org/lflang/generator/LanguageServerErrorReporter.java @@ -2,9 +2,12 @@ import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import org.eclipse.emf.common.util.URI; import org.eclipse.emf.ecore.EObject; import org.eclipse.lsp4j.Diagnostic; import org.eclipse.lsp4j.DiagnosticSeverity; @@ -15,6 +18,11 @@ import org.lflang.ErrorReporter; +/** + * Report diagnostics to the language client. + * + * @author Peter Donovan + */ public class LanguageServerErrorReporter implements ErrorReporter { /** @@ -27,12 +35,12 @@ public class LanguageServerErrorReporter implements ErrorReporter { /** The document for which this is a diagnostic acceptor. */ private final EObject parseRoot; /** The list of all diagnostics since the last reset. */ - private final List diagnostics; + private final Map> diagnostics; /* ------------------------ CONSTRUCTORS -------------------------- */ /** - * Initializes a {@code DiagnosticAcceptor} for the + * Initialize a {@code DiagnosticAcceptor} for the * document whose parse tree root node is * {@code parseRoot}. * @param parseRoot the root of the AST of the document @@ -40,19 +48,19 @@ public class LanguageServerErrorReporter implements ErrorReporter { */ public LanguageServerErrorReporter(EObject parseRoot) { this.parseRoot = parseRoot; - this.diagnostics = new ArrayList<>(); + this.diagnostics = new HashMap<>(); } /* ----------------------- PUBLIC METHODS ------------------------- */ @Override public String reportError(String message) { - return acceptDiagnostic(DiagnosticSeverity.Error, message); + return report(getMainFile(), DiagnosticSeverity.Error, message); } @Override public String reportWarning(String message) { - return acceptDiagnostic(DiagnosticSeverity.Warning, message); + return report(getMainFile(), DiagnosticSeverity.Warning, message); } @Override @@ -67,77 +75,84 @@ public String reportWarning(EObject object, String message) { @Override public String reportError(Path file, Integer line, String message) { - return acceptDiagnostic(DiagnosticSeverity.Error, message, line != null ? line - 1 : 0); // TODO: document one-basedness + return report(file, DiagnosticSeverity.Error, message, line != null ? line : 1); } @Override public String reportWarning(Path file, Integer line, String message) { - return acceptDiagnostic(DiagnosticSeverity.Warning, message, line != null ? line - 1 : 0); + return report(file, DiagnosticSeverity.Warning, message, line != null ? line : 1); } @Override public boolean getErrorsOccurred() { - return diagnostics.stream().anyMatch(diagnostic -> diagnostic.getSeverity() == DiagnosticSeverity.Error); + return diagnostics.values().stream().anyMatch( + it -> it.stream().anyMatch(diagnostic -> diagnostic.getSeverity() == DiagnosticSeverity.Error) + ); + } + + @Override + public String report(Path file, DiagnosticSeverity severity, String message) { + return report(file, severity, message, 1); + } + + @Override + public String report(Path file, DiagnosticSeverity severity, String message, int line) { + Optional text = getLine(line - 1); + return report( + file, + severity, + message, + Position.fromOneBased(line, 1), + Position.fromOneBased(line, 1 + (text.isEmpty() ? 0 : text.get().length())) + ); + } + + @Override + public String report(Path file, DiagnosticSeverity severity, String message, Position startPos, Position endPos) { + if (file == null) file = getMainFile(); + diagnostics.putIfAbsent(file, new ArrayList<>()); + diagnostics.get(file).add(new Diagnostic( + toRange(startPos, endPos), message, severity, "LF Language Server" + )); + return "" + severity + ": " + message; } /** - * Saves a reference to the language client. + * Save a reference to the language client. * @param client the language client */ public static void setClient(LanguageClient client) { LanguageServerErrorReporter.client = client; } - /* ----------------------- PRIVATE METHODS ------------------------ */ - /** - * Reports a message of severity {@code severity}. - * @param severity the severity of the message - * @param message the message to send to the IDE - * @return a string that describes the diagnostic + * Publish diagnostics by forwarding them to the + * language client. */ - private String acceptDiagnostic(DiagnosticSeverity severity, String message) { - return acceptDiagnostic(severity, message, 0); + public void publishDiagnostics() { + if (client == null) { + System.err.println( + "WARNING: Cannot publish diagnostics because the language client has not yet been found." + ); + return; + } + for (Path file : diagnostics.keySet()) { + PublishDiagnosticsParams publishDiagnosticsParams = new PublishDiagnosticsParams(); + publishDiagnosticsParams.setUri(URI.createFileURI(file.toString()).toString()); + publishDiagnosticsParams.setDiagnostics(diagnostics.get(file)); + client.publishDiagnostics(publishDiagnosticsParams); + } } - /** - * Reports a message of severity {@code severity}. - * @param severity the severity of the message - * @param message the message to send to the IDE - * @param line the zero-based line number associated - * with the message - * @return a string that describes the diagnostic - */ - private String acceptDiagnostic(DiagnosticSeverity severity, String message, int line) { - Optional text = getLine(line); - return acceptDiagnostic( - severity, - message, - Position.fromZeroBased(line, 0), - Position.fromZeroBased(line, text.isEmpty() ? 1 : text.get().length()) - ); - } + /* ----------------------- PRIVATE METHODS ------------------------ */ - /** - * Reports a message of severity {@code severity}. - * @param severity the severity of the message - * @param message the message to send to the IDE - * @param startPos the position of the first character - * of the range of interest - * @param endPos the position immediately AFTER the - * final character of the range of - * interest - * @return a string that describes the diagnostic - */ - private String acceptDiagnostic(DiagnosticSeverity severity, String message, Position startPos, Position endPos) { - diagnostics.add(new Diagnostic( - toRange(startPos, endPos), message, severity, "LF Language Server" - )); - return "" + severity + ": " + message; + /** Return the file on which the current validation process was triggered. */ + private Path getMainFile() { + return Path.of(parseRoot.eResource().getURI().toFileString()); } /** - * Returns the text of the document for which this is an + * Return the text of the document for which this is an * error reporter. * @return the text of the document for which this is an * error reporter @@ -147,7 +162,7 @@ private String getText() { } /** - * Returns the line at index {@code line} in the + * Return the line at index {@code line} in the * document for which this is an error reporter. * @param line the zero-based line index * @return the line located at the given index @@ -157,24 +172,7 @@ private Optional getLine(int line) { } /** - * Publishes diagnostics by forwarding them to the - * language client. - */ - public void publishDiagnostics() { - if (client == null) { - System.err.println( - "WARNING: Cannot publish diagnostics because the language client has not yet been found." - ); - return; - } - PublishDiagnosticsParams publishDiagnosticsParams = new PublishDiagnosticsParams(); - publishDiagnosticsParams.setUri(parseRoot.eResource().getURI().toString()); - publishDiagnosticsParams.setDiagnostics(diagnostics); - client.publishDiagnostics(publishDiagnosticsParams); - } - - /** - * Returns the Range that starts at {@code p0} and ends + * Return the Range that starts at {@code p0} and ends * at {@code p1}. * @param p0 an arbitrary Position * @param p1 a Position that is greater than {@code p0} diff --git a/org.lflang/src/org/lflang/generator/MainContext.java b/org.lflang/src/org/lflang/generator/MainContext.java index 201b3da40e..c786cf2200 100644 --- a/org.lflang/src/org/lflang/generator/MainContext.java +++ b/org.lflang/src/org/lflang/generator/MainContext.java @@ -73,7 +73,7 @@ public MainContext( Function constructErrorReporter ) { this.mode = mode; - this.cancelIndicator = cancelIndicator; + this.cancelIndicator = cancelIndicator == null ? () -> false : cancelIndicator; this.reportProgress = reportProgress; this.args = args; this.hierarchicalBin = hierarchicalBin; @@ -114,7 +114,7 @@ public ErrorReporter constructErrorReporter(FileConfig fileConfig) { public void finish(GeneratorResult result) { if (this.result != null) throw new IllegalStateException("A code generation process can only have one result."); this.result = result; - reportProgress("Build complete.", 100); + reportProgress(result.getUserMessage(), 100); } @Override diff --git a/org.lflang/src/org/lflang/generator/PerLineReportingStrategy.java b/org.lflang/src/org/lflang/generator/PerLineReportingStrategy.java deleted file mode 100644 index e94a16a62d..0000000000 --- a/org.lflang/src/org/lflang/generator/PerLineReportingStrategy.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.lflang.generator; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.lflang.ErrorReporter; - -/** - * An error reporting strategy that considers only one line - * of validator output at a time. - */ -public class PerLineReportingStrategy implements CommandErrorReportingStrategy { - - private final Pattern p; - - /** - * Instantiates a reporting strategy for lines of - * validator output that match {@code p}. - * @param p a pattern that matches lines that should be - * reported via this strategy. This pattern - * must contain named capturing groups called - * "path", "line", "column", "message", and - * "severity". - */ - public PerLineReportingStrategy(Pattern p) { - for (String groupName : new String[]{"path", "line", "column", "message", "severity"}) { - assert p.pattern().contains(groupName) : String.format( - "Error line patterns must have a named capturing group called %s", groupName - ); - } - this.p = p; - } - - @Override - public void report(String validationOutput, ErrorReporter errorReporter, Map map) { - validationOutput.lines().forEach(line -> reportErrorLine(line, errorReporter, map)); - } - - /** - * Reports the validation message contained in - * {@code line} if such a message exists. - * @param line a line of validator output - * @param errorReporter an arbitrary ErrorReporter - * @param maps a mapping from generated file paths to - * CodeMaps - */ - private void reportErrorLine(String line, ErrorReporter errorReporter, Map maps) { - Matcher matcher = p.matcher(line); - if (matcher.matches()) { - final Path path = Paths.get(matcher.group("path")); - final Position generatedFilePosition = Position.fromOneBased( - Integer.parseInt(matcher.group("line")), Integer.parseInt(matcher.group("column")) - ); - final String message = String.format( - "%s [%s:%s:%s]", - matcher.group("message"), path.getFileName(), - matcher.group("line"), matcher.group("column") - ); - final CodeMap map = maps.get(path); - final boolean isError = matcher.group("severity").toLowerCase().contains("error"); - if (map == null) { - if (isError) { - errorReporter.reportError(message); - } else { - errorReporter.reportWarning(message); - } - return; - } - for (Path srcFile : map.lfSourcePaths()) { - // FIXME: Is it desirable for the error to be reported to every single LF file associated - // with the generated file containing the error? Or is it best to be more selective? - Position lfFilePosition = map.adjusted(srcFile, generatedFilePosition); - if (isError) { - errorReporter.reportError(srcFile, lfFilePosition.getOneBasedLine(), message); - } else { - errorReporter.reportWarning(srcFile, lfFilePosition.getOneBasedLine(), message); - } - } - } - } -} diff --git a/org.lflang/src/org/lflang/generator/Position.java b/org.lflang/src/org/lflang/generator/Position.java index 87c6b2b091..9923785f2d 100644 --- a/org.lflang/src/org/lflang/generator/Position.java +++ b/org.lflang/src/org/lflang/generator/Position.java @@ -4,9 +4,11 @@ import java.util.regex.Pattern; /** - * Represents a position in a document, including line and - * column. This position may be relative to another + * A position in a document, including line and + * column. This position may be relative to some * position other than the origin. + * + * @author Peter Donovan */ public class Position implements Comparable { public static final Pattern PATTERN = Pattern.compile("\\((?[0-9]+), (?[0-9]+)\\)"); @@ -15,28 +17,13 @@ public class Position implements Comparable { private static final Pattern LINE_SEPARATOR = Pattern.compile("(\n)|(\r)|(\r\n)"); - /* - Implementation note: This class is designed to remove - all ambiguity wrt zero-based and one-based line and - column indexing. The motivating philosophy is that all - indexing should be zero-based in any programming - context, unless one is forced to use one-based indexing - in order to interface with someone else's software. This - justifies the apparent ambivalence here wrt zero vs. - one: Zero should be used when possible, but one can be - used when necessary. - This philosophy (and the need to be Comparable) - explains the choice not to use - org.eclipse.xtext.util.LineAndColumn. - */ - private final int line; private final int column; /* ------------------------ CONSTRUCTORS -------------------------- */ /** - * Returns the Position that describes the given + * Return the Position that describes the given * zero-based line and column numbers. * @param line the zero-based line number * @param column the zero-based column number @@ -48,7 +35,7 @@ public static Position fromZeroBased(int line, int column) { } /** - * Returns the Position that describes the given + * Return the Position that describes the given * one-based line and column numbers. * @param line the one-based line number * @param column the one-based column number @@ -60,8 +47,9 @@ public static Position fromOneBased(int line, int column) { } /** - * Returns the Position that equals the displacement - * caused by {@code text}. + * Return the Position that equals the displacement + * caused by {@code text}, assuming that {@code text} + * starts in column 0. * @param text an arbitrary string * @return the Position that equals the displacement * caused by {@code text} @@ -73,7 +61,7 @@ public static Position displacementOf(String text) { } /** - * Returns the Position that describes the same location + * Return the Position that describes the same location * in {@code content} as {@code offset}. * @param offset a location, expressed as an offset from * the beginning of {@code content} @@ -94,9 +82,8 @@ public static Position fromOffset(int offset, String content) { } /** - * Creates a new Position with the given line and column - * numbers. Private so that unambiguously named factory - * methods must be used instead. + * Create a new Position with the given line and column + * numbers. * @param line the zero-based line number * @param column the zero-based column number */ @@ -111,7 +98,7 @@ private Position(int line, int column) { /* ----------------------- PUBLIC METHODS ------------------------- */ /** - * Returns the one-based line number described by this + * Return the one-based line number described by this * {@code Position}. * @return the one-based line number described by this * {@code Position} @@ -121,7 +108,7 @@ public int getOneBasedLine() { } /** - * Returns the one-based column number described by this + * Return the one-based column number described by this * {@code Position}. * @return the one-based column number described by this * {@code Position} @@ -131,7 +118,7 @@ public int getOneBasedColumn() { } /** - * Returns the zero-based line number described by this + * Return the zero-based line number described by this * {@code Position}. * @return the zero-based line number described by this * {@code Position} @@ -141,7 +128,7 @@ public int getZeroBasedLine() { } /** - * Returns the zero-based column number described by this + * Return the zero-based column number described by this * {@code Position}. * @return the zero-based column number described by this * {@code Position} @@ -151,25 +138,25 @@ public int getZeroBasedColumn() { } /** - * Returns the offset of this {@code Position} from - * the beginning of the document whose content is - * {@code documentContent}. Silently returns an - * incorrect but valid offset in the case that this - * {@code Position} is not contained in - * {@code documentContent}. - * @param documentContent the content of the document - * in which this is a position - * @return the offset of this {@code Position} from - * the beginning of the document whose content is - * {@code documentContent} + * Return the Position that equals the displacement of + * ((text whose displacement equals {@code this}) + * concatenated with {@code text}). Note that this is + * not necessarily equal to + * ({@code this} + displacementOf(text)). + * @param text an arbitrary string + * @return the Position that equals the displacement + * caused by {@code text} */ - public int getOffset(String documentContent) { - return documentContent.lines().limit(getZeroBasedLine()).mapToInt(String::length).sum() - + getZeroBasedColumn() + getZeroBasedLine(); // Final term accounts for line breaks + public Position plus(String text) { + text += System.lineSeparator(); // Turn line separators into line terminators. + String[] lines = text.lines().toArray(String[]::new); + if (lines.length == 0) return this; // OK not to copy because Positions are immutable + int lastLineLength = lines[lines.length - 1].length(); + return new Position(line + lines.length - 1, lines.length > 1 ? lastLineLength : column + lastLineLength); } /** - * Returns the sum of this and another {@code Position}. + * Return the sum of this and another {@code Position}. * The result has meaning because Positions are * relative. * @param other another {@code Position} @@ -180,7 +167,7 @@ public Position plus(Position other) { } /** - * Returns the difference of this and another {@code + * Return the difference of this and another {@code * Position}. The result has meaning because * Positions are relative. * @param other another {@code Position} @@ -191,7 +178,7 @@ public Position minus(Position other) { } /** - * Compares two positions according to their order of + * Compare two positions according to their order of * appearance in a document (first according to line, * then according to column). */ @@ -214,7 +201,7 @@ public String toString() { } /** - * Returns the Position represented by {@code s}. + * Return the Position represented by {@code s}. * @param s a String that represents a Position, * formatted like the output of * {@code Position::toString}. @@ -237,7 +224,7 @@ public int hashCode() { } /** - * Removes the names from the named capturing groups + * Remove the names from the named capturing groups * that appear in {@code regex}. * @param regex an arbitrary regular expression * @return a string representation of {@code regex} diff --git a/org.lflang/src/org/lflang/generator/ValidationStrategy.java b/org.lflang/src/org/lflang/generator/ValidationStrategy.java new file mode 100644 index 0000000000..e2882bf41a --- /dev/null +++ b/org.lflang/src/org/lflang/generator/ValidationStrategy.java @@ -0,0 +1,46 @@ +package org.lflang.generator; + +import java.nio.file.Path; + +import org.lflang.util.LFCommand; + +/** + * A means of validating generated code. + * + * @author Peter Donovan + */ +public interface ValidationStrategy { + + /** + * Return the command that produces validation output in association + * with {@code generatedFile}, or {@code null} if this strategy has no + * command that will successfully produce validation output. + */ + LFCommand getCommand(Path generatedFile); + + /** + * Return a strategy for parsing the stderr of the validation command. + * @return A strategy for parsing the stderr of the validation command. + */ + DiagnosticReporting.Strategy getErrorReportingStrategy(); + + /** + * Return a strategy for parsing the stdout of the validation command. + * @return A strategy for parsing the stdout of the validation command. + */ + DiagnosticReporting.Strategy getOutputReportingStrategy(); + + /** + * Return whether this strategy validates all generated files, as + * opposed to just the given one. + * @return whether this strategy validates all generated files + */ + boolean isFullBatch(); + + /** + * Return the priority of this. Strategies with higher + * priorities are more likely to be used. + * @return The priority of this. + */ + int getPriority(); +} diff --git a/org.lflang/src/org/lflang/generator/Validator.java b/org.lflang/src/org/lflang/generator/Validator.java new file mode 100644 index 0000000000..718ee2a5d6 --- /dev/null +++ b/org.lflang/src/org/lflang/generator/Validator.java @@ -0,0 +1,154 @@ +package org.lflang.generator; + +import org.eclipse.xtext.util.CancelIndicator; + +import org.lflang.ErrorReporter; +import org.lflang.util.LFCommand; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableMap; + +/** + * Validate generated code. + * + * @author Peter Donovan + */ +public abstract class Validator { + + protected static class Pair { + public final S first; + public final T second; + public Pair(S first, T second) { + this.first = first; + this.second = second; + } + } + + protected final ErrorReporter errorReporter; + protected final ImmutableMap codeMaps; + + /** + * Initialize a {@code Validator} that reports errors to {@code errorReporter} and adjusts + * document positions using {@code codeMaps}. + */ + protected Validator(ErrorReporter errorReporter, Map codeMaps) { + this.errorReporter = errorReporter; + this.codeMaps = ImmutableMap.copyOf(codeMaps); + } + + /** + * Validate this Validator's group of generated files. + * @param cancelIndicator The cancel indicator for the + * current operation. + */ + public final void doValidate(CancelIndicator cancelIndicator) throws ExecutionException, InterruptedException { + final List>> tasks = getValidationStrategies().stream().map( + it -> (Callable>) () -> { + it.second.run(cancelIndicator, true); + return it; + } + ).collect(Collectors.toList()); + for (Future> f : getFutures(tasks)) { + f.get().first.getErrorReportingStrategy().report(f.get().second.getErrors().toString(), errorReporter, codeMaps); + f.get().first.getOutputReportingStrategy().report(f.get().second.getOutput().toString(), errorReporter, codeMaps); + } + } + + /** + * Invoke all the given tasks. + * @param tasks Any set of tasks. + * @param The return type of the tasks. + * @return Futures corresponding to each task, or an empty list upon failure. + * @throws InterruptedException If interrupted while waiting. + */ + private static List> getFutures(List> tasks) throws InterruptedException { + List> futures = List.of(); + switch (tasks.size()) { + case 0: + break; + case 1: + try { + futures = List.of(CompletableFuture.completedFuture(tasks.get(0).call())); + } catch (Exception e) { + System.err.println(e.getMessage()); // This should never happen + } + break; + default: + futures = Executors.newFixedThreadPool( + Math.min(Runtime.getRuntime().availableProcessors(), tasks.size()) + ).invokeAll(tasks); + } + return futures; + } + + /** + * Run the given command, report any messages produced using the reporting strategies + * given by {@code getBuildReportingStrategies}, and return its return code. + */ + public final int run(LFCommand command, CancelIndicator cancelIndicator) { + final int returnCode = command.run(cancelIndicator, true); + getBuildReportingStrategies().first.report(command.getErrors().toString(), errorReporter, codeMaps); + getBuildReportingStrategies().second.report(command.getOutput().toString(), errorReporter, codeMaps); + return returnCode; + } + + /** + * Return the validation strategies and validation + * commands corresponding to each generated file. + * @return the validation strategies and validation + * commands corresponding to each generated file + */ + private List> getValidationStrategies() { + final List> commands = new ArrayList<>(); + for (Path generatedFile : codeMaps.keySet()) { + final Pair p = getValidationStrategy(generatedFile); + if (p.first == null || p.second == null) continue; + commands.add(p); + if (p.first.isFullBatch()) break; + } + return commands; + } + + /** + * Return the validation strategy and command + * corresponding to the given file if such a strategy + * and command are available. + * @return the validation strategy and command + * corresponding to the given file if such a strategy + * and command are available + */ + private Pair getValidationStrategy(Path generatedFile) { + List sorted = getPossibleStrategies().stream() + .sorted(Comparator.comparingInt(vs -> -vs.getPriority())).collect(Collectors.toList()); + for (ValidationStrategy strategy : sorted) { + LFCommand validateCommand = strategy.getCommand(generatedFile); + if (validateCommand != null) { + return new Pair<>(strategy, validateCommand); + } + } + return new Pair<>(null, null); + } + + /** + * List all validation strategies that exist for the implementor + * without filtering by platform or availability. + */ + protected abstract Collection getPossibleStrategies(); + + /** + * Return the appropriate output and error reporting + * strategies for the main build process. + */ + protected abstract Pair getBuildReportingStrategies(); +} diff --git a/org.lflang/src/org/lflang/generator/c/CCmakeCompiler.java b/org.lflang/src/org/lflang/generator/c/CCmakeCompiler.java index 4a11d4965f..089a967cb7 100644 --- a/org.lflang/src/org/lflang/generator/c/CCmakeCompiler.java +++ b/org.lflang/src/org/lflang/generator/c/CCmakeCompiler.java @@ -38,6 +38,7 @@ import org.lflang.TargetConfig.Mode; import org.lflang.TargetConfig; import org.lflang.generator.GeneratorBase; +import org.lflang.generator.JavaGeneratorUtils; import org.lflang.generator.LFGeneratorContext; import org.lflang.util.LFCommand; @@ -198,7 +199,7 @@ public LFCommand compileCmakeCommand( FileConfig.toUnixString(fileConfig.getSrcGenPath()) )); - if (isHostWindows()) { + if (JavaGeneratorUtils.isHostWindows()) { arguments.add("-DCMAKE_SYSTEM_VERSION=\"10.0\""); } diff --git a/org.lflang/src/org/lflang/generator/c/CCompiler.java b/org.lflang/src/org/lflang/generator/c/CCompiler.java index 5c573e6612..fa7956d945 100644 --- a/org.lflang/src/org/lflang/generator/c/CCompiler.java +++ b/org.lflang/src/org/lflang/generator/c/CCompiler.java @@ -240,14 +240,4 @@ static String getTargetFileName(String fileName, boolean CppMode) { } return fileName + ".c"; } - - - - /** Return true if the operating system is Windows. */ - public static boolean isHostWindows() { - String OS = System.getProperty("os.name").toLowerCase(); - if (OS.indexOf("win") >= 0) { return true; } - return false; - } - } diff --git a/org.lflang/src/org/lflang/generator/c/CGenerator.xtend b/org.lflang/src/org/lflang/generator/c/CGenerator.xtend index 4239088838..3a6111aa49 100644 --- a/org.lflang/src/org/lflang/generator/c/CGenerator.xtend +++ b/org.lflang/src/org/lflang/generator/c/CGenerator.xtend @@ -434,7 +434,7 @@ class CGenerator extends GeneratorBase { * otherwise report an error and return false. */ protected def boolean isOSCompatible() { - if (CCompiler.isHostWindows) { + if (JavaGeneratorUtils.isHostWindows) { if (isFederated) { errorReporter.reportError( "Federated LF programs with a C target are currently not supported on Windows. " + @@ -582,8 +582,8 @@ class CGenerator extends GeneratorBase { val compileThreadPool = Executors.newFixedThreadPool(numOfCompileThreads); System.out.println("******** Using "+numOfCompileThreads+" threads."); var federateCount = 0; - val LFGeneratorContext compilingContext = new SubContext( - context, IntegratedBuilder.VALIDATED_PERCENT_PROGRESS, 100 + val LFGeneratorContext generatingContext = new SubContext( + context, IntegratedBuilder.VALIDATED_PERCENT_PROGRESS, IntegratedBuilder.GENERATED_PERCENT_PROGRESS ) for (federate : federates) { federateCount++; @@ -891,7 +891,7 @@ class CGenerator extends GeneratorBase { val threadFileConfig = fileConfig; val generator = this; // FIXME: currently only passed to report errors with line numbers in the Eclipse IDE val CppMode = CCppMode; - compilingContext.reportProgress( + generatingContext.reportProgress( String.format("Generated code for %d/%d executables. Compiling...", federateCount, federates.size()), 100 * federateCount / federates.size() ); @@ -5988,12 +5988,12 @@ class CGenerator extends GeneratorBase { // Regular expression pattern for array types with specified length. // \s is whitespace, \w is a word character (letter, number, or underscore). // For example, for "foo[10]", the first match will be "foo" and the second "[10]". - static final Pattern arrayPatternFixed = Pattern.compile("^\\s*+(\\w+)\\s*(\\[[0-9]+\\])\\s*$"); + static final Pattern arrayPatternFixed = Pattern.compile("^\\s*(?:/\\*.*?\\*/)?\\s*(\\w+)\\s*(\\[[0-9]+\\])\\s*$"); // Regular expression pattern for array types with unspecified length. // \s is whitespace, \w is a word character (letter, number, or underscore). // For example, for "foo[]", the first match will be "foo". - static final Pattern arrayPatternVariable = Pattern.compile("^\\s*+(\\w+)\\s*\\[\\]\\s*$"); + static final Pattern arrayPatternVariable = Pattern.compile("^\\s*(?:/\\*.*?\\*/)?\\s*(\\w+)\\s*\\[\\]\\s*$"); // Regular expression pattern for shared_ptr types. static final Pattern sharedPointerVariable = Pattern.compile("^std::shared_ptr<(\\S+)>$"); diff --git a/org.lflang/src/org/lflang/generator/cpp/CppCmakeGenerator.kt b/org.lflang/src/org/lflang/generator/cpp/CppCmakeGenerator.kt index 923becf13b..1a3089bcef 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppCmakeGenerator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppCmakeGenerator.kt @@ -35,6 +35,7 @@ class CppCmakeGenerator(private val targetConfig: TargetConfig, private val file companion object { const val includesVarName: String = "TARGET_INCLUDE_DIRECTORIES" + const val compilerIdName: String = "CXX_COMPILER_ID" } /** Convert a log level to a severity number understood by the reactor-cpp runtime. */ @@ -148,6 +149,7 @@ class CppCmakeGenerator(private val targetConfig: TargetConfig, private val file |get_target_property(TARGET_INCLUDE_DIRECTORIES $S{LF_MAIN_TARGET} INCLUDE_DIRECTORIES) |list(APPEND TARGET_INCLUDE_DIRECTORIES $S{SOURCE_DIR}/include) |set(${includesVarName} $S{TARGET_INCLUDE_DIRECTORIES} CACHE STRING "Directories included in the main target." FORCE) + |set(${compilerIdName} $S{CMAKE_CXX_COMPILER_ID} CACHE STRING "The name of the C++ compiler." FORCE) | ${" |"..(includeFiles?.joinToString("\n") { "include(\"$it\")" } ?: "") } """.trimMargin() diff --git a/org.lflang/src/org/lflang/generator/cpp/CppGenerator.kt b/org.lflang/src/org/lflang/generator/cpp/CppGenerator.kt index fb250585cd..13d5c9d4d2 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppGenerator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppGenerator.kt @@ -127,8 +127,10 @@ class CppGenerator( if (!r.isGeneric) cppSources.add(sourceFile) codeMaps[fileConfig.srcGenPath.resolve(sourceFile)] = reactorCodeMap + val headerCodeMap = CodeMap.fromGeneratedCode(generator.generateHeader()) + codeMaps[fileConfig.srcGenPath.resolve(headerFile)] = headerCodeMap - fsa.generateFile(relSrcGenPath.resolve(headerFile).toString(), generator.generateHeader()) + fsa.generateFile(relSrcGenPath.resolve(headerFile).toString(), headerCodeMap.generatedCode) fsa.generateFile(relSrcGenPath.resolve(sourceFile).toString(), reactorCodeMap.generatedCode) } @@ -140,8 +142,10 @@ class CppGenerator( val preambleCodeMap = CodeMap.fromGeneratedCode(generator.generateSource()) cppSources.add(sourceFile) codeMaps[fileConfig.srcGenPath.resolve(sourceFile)] = preambleCodeMap + val headerCodeMap = CodeMap.fromGeneratedCode(generator.generateHeader()) + codeMaps[fileConfig.srcGenPath.resolve(headerFile)] = headerCodeMap - fsa.generateFile(relSrcGenPath.resolve(headerFile).toString(), generator.generateHeader()) + fsa.generateFile(relSrcGenPath.resolve(headerFile).toString(), headerCodeMap.generatedCode) fsa.generateFile(relSrcGenPath.resolve(sourceFile).toString(), preambleCodeMap.generatedCode) } diff --git a/org.lflang/src/org/lflang/generator/cpp/CppMethodGenerator.kt b/org.lflang/src/org/lflang/generator/cpp/CppMethodGenerator.kt index 3497a964c5..17aa30032c 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppMethodGenerator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppMethodGenerator.kt @@ -29,7 +29,7 @@ import org.lflang.generator.PrependOperator import org.lflang.lf.Method import org.lflang.lf.MethodArgument import org.lflang.lf.Reactor -import org.lflang.toTaggedText +import org.lflang.toText /** A C++ code generator for state variables */ class CppMethodGenerator(private val reactor: Reactor) { @@ -45,7 +45,7 @@ class CppMethodGenerator(private val reactor: Reactor) { """ |${reactor.templateLine} |$targetType ${reactor.templateName}::Inner::$name(${cppArgs.joinToString(", ")})$constQualifier { - ${" | "..code.toTaggedText()} + ${" | "..code.toText()} |} """.trimMargin() } diff --git a/org.lflang/src/org/lflang/generator/cpp/CppPreambleGenerator.kt b/org.lflang/src/org/lflang/generator/cpp/CppPreambleGenerator.kt index 17e130edb7..9ec9b31b3e 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppPreambleGenerator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppPreambleGenerator.kt @@ -30,7 +30,7 @@ import org.lflang.generator.PrependOperator import org.lflang.lf.Preamble import org.lflang.model import org.lflang.scoping.LFGlobalScopeProvider -import org.lflang.toTaggedText +import org.lflang.toText import org.lflang.toUnixString @@ -60,7 +60,7 @@ class CppPreambleGenerator( |#include "reactor-cpp/reactor-cpp.hh" ${" |"..includes.joinToString(separator = "\n", prefix = "// include the preambles from imported files \n")} | - ${" |"..publicPreambles.joinToString(separator = "\n") { it.code.toTaggedText() }} + ${" |"..publicPreambles.joinToString(separator = "\n") { it.code.toText() }} """.trimMargin() } } @@ -79,7 +79,7 @@ class CppPreambleGenerator( |using namespace std::chrono_literals; |using namespace reactor::operators; | - ${" |"..privatePreambles.joinToString(separator = "\n") { it.code.toTaggedText() }} + ${" |"..privatePreambles.joinToString(separator = "\n") { it.code.toText() }} """.trimMargin() } } diff --git a/org.lflang/src/org/lflang/generator/cpp/CppReactionGenerator.kt b/org.lflang/src/org/lflang/generator/cpp/CppReactionGenerator.kt index 1aacc3659a..f418255c35 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppReactionGenerator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppReactionGenerator.kt @@ -29,7 +29,7 @@ import org.lflang.isBank import org.lflang.label import org.lflang.lf.* import org.lflang.priority -import org.lflang.toTaggedText +import org.lflang.toText /** A C++ code generator for reactions and their function bodies */ class CppReactionGenerator( @@ -144,7 +144,7 @@ class CppReactionGenerator( |// reaction ${reaction.label} |${reactor.templateLine} ${" |"..getFunctionDefinitionSignature(reaction, "body")} { - ${" | "..reaction.code.toTaggedText()} + ${" | "..reaction.code.toText()} |} | """.trimMargin() @@ -155,7 +155,7 @@ class CppReactionGenerator( return """ |${reactor.templateLine} ${" |"..getFunctionDefinitionSignature(reaction, "deadline_handler")} { - ${" | "..reaction.deadline.code.toTaggedText()} + ${" | "..reaction.deadline.code.toText()} |} | """.trimMargin() diff --git a/org.lflang/src/org/lflang/generator/cpp/CppReactorGenerator.kt b/org.lflang/src/org/lflang/generator/cpp/CppReactorGenerator.kt index 4fde1709af..85556a83a9 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppReactorGenerator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppReactorGenerator.kt @@ -29,7 +29,6 @@ import org.lflang.generator.PrependOperator import org.lflang.isGeneric import org.lflang.lf.Reactor import org.lflang.toText -import org.lflang.toTaggedText import org.lflang.toUnixString /** @@ -62,11 +61,11 @@ class CppReactorGenerator(private val reactor: Reactor, fileConfig: CppFileConfi private fun publicPreamble() = reactor.preambles.filter { it.isPublic } - .joinToString(separator = "\n", prefix = "// public preamble\n") { it.code.toTaggedText() } + .joinToString(separator = "\n", prefix = "// public preamble\n") { it.code.toText() } private fun privatePreamble() = reactor.preambles.filter { it.isPrivate } - .joinToString(separator = "\n", prefix = "// private preamble\n") { it.code.toTaggedText() } + .joinToString(separator = "\n", prefix = "// private preamble\n") { it.code.toText() } /** Generate a C++ header file declaring the given reactor. */ fun generateHeader() = with(PrependOperator) { diff --git a/org.lflang/src/org/lflang/generator/cpp/CppValidator.kt b/org.lflang/src/org/lflang/generator/cpp/CppValidator.kt index 48e05d4881..d91c759f1a 100644 --- a/org.lflang/src/org/lflang/generator/cpp/CppValidator.kt +++ b/org.lflang/src/org/lflang/generator/cpp/CppValidator.kt @@ -1,175 +1,164 @@ package org.lflang.generator.cpp -import org.eclipse.xtext.util.CancelIndicator import org.lflang.ErrorReporter +import org.lflang.generator.ValidationStrategy import org.lflang.generator.CodeMap -import org.lflang.generator.CommandErrorReportingStrategy -import org.lflang.generator.PerLineReportingStrategy +import org.lflang.generator.DiagnosticReporting +import org.lflang.generator.HumanReadableReportingStrategy +import org.lflang.generator.Validator import org.lflang.util.LFCommand -import java.nio.file.Files +import java.io.File import java.nio.file.Path -import java.util.concurrent.Callable -import java.util.concurrent.Executors +import java.nio.file.Paths import java.util.regex.Pattern +/** + * A validator for generated C++. + * + * @author Peter Donovan + */ class CppValidator( private val fileConfig: CppFileConfig, - private val errorReporter: ErrorReporter, - private val codeMaps: Map -) { + errorReporter: ErrorReporter, + codeMaps: Map +): Validator(errorReporter, codeMaps) { companion object { - /** This matches the line in the CMake cache that states the C++ standard. */ - private val cmakeCxxStandard: Pattern = Pattern.compile("CMAKE_CXX_STANDARD:STRING=(?.*)") - /** This matches the line in the CMake cache that states the compiler includes for a given target. */ - private val cmakeIncludes: Pattern = Pattern.compile("${CppCmakeGenerator.includesVarName}:STRING=(?.*)") + /** This matches a line in the CMake cache. */ + private val CMAKE_CACHED_VARIABLE: Pattern = Pattern.compile("(?[\\w_]+):(?[\\w_]+)=(?.*)") /** This matches a line of error reports from g++. */ - private val gxxErrorLine: Pattern = Pattern.compile( + private val GXX_ERROR_LINE: Pattern = Pattern.compile( "(?.+\\.((cc)|(hh))):(?\\d+):(?\\d+): (?(error)|(warning)): (?.*?) ?(?(\\[.*])?)" ) - // Happily, the two tools seem to produce errors that follow the same format. - /** This matches a line of error reports from Clang-Tidy. */ - private val clangTidyErrorLine: Pattern = gxxErrorLine + private val GXX_LABEL: Pattern = Pattern.compile("(~*)(\\^~*)") + // Happily, multiple tools seem to produce errors that follow the same format. + /** This matches a line of error reports from Clang. */ + private val CLANG_ERROR_LINE: Pattern = GXX_ERROR_LINE + private val CLANG_LABEL: Pattern = GXX_LABEL + /** This matches a line of error reports from MSVC. */ + private val MSVC_ERROR_LINE: Pattern = Pattern.compile( + "(?.+\\.((cc)|(hh)))\\((?\\d+)(,\\s*(?\\d+))?\\)\\s*:.*?(?(error)|(warning)) [A-Z]+\\d+:\\s*(?.*?)" + ) + private val MSVC_LABEL: Pattern = Pattern.compile("(?<=\\s)\\^(?=\\s)") // Unused with the current MSVC settings + } + + private class CppValidationStrategy( + private val errorReportingStrategy: DiagnosticReporting.Strategy, + private val outputReportingStrategy: DiagnosticReporting.Strategy, + private val time: Int, + private val getCommand: (p: Path) -> LFCommand? + ): ValidationStrategy { + + override fun getCommand(generatedFile: Path) = getCommand.invoke(generatedFile) + + override fun getErrorReportingStrategy() = errorReportingStrategy + + override fun getOutputReportingStrategy() = outputReportingStrategy + + override fun isFullBatch() = false + + override fun getPriority() = -time } /** - * This describes a strategy for validating a C++ source document. - * @param errorReportingStrategy a strategy for parsing the stderr of the validation command - * @param outputReportingStrategy a strategy for parsing the stdout of the validation command - * @param time a number that is large for strategies that take a long time + * [CppValidationStrategyFactory] instances map validator instances to validation strategy instances. + * @param compilerIds The CMake compiler IDs of the compilers that are closely related to {@code this}. + * @param create The function that creates a strategy from a validator. */ - private enum class CppValidationStrategy( - val errorReportingStrategy: CommandErrorReportingStrategy, - val outputReportingStrategy: CommandErrorReportingStrategy, - val time: Int - ) { + private enum class CppValidationStrategyFactory(val compilerIds: List, val create: ((CppValidator) -> CppValidationStrategy)) { + // Note: Clang-tidy is slow (on the order of tens of seconds) for checking C++ files. - CLANG_TIDY({ _, _, _ -> }, PerLineReportingStrategy(clangTidyErrorLine), 5) { - override fun getCommand(validator: CppValidator, generatedFile: Path): LFCommand? { - val args = mutableListOf(generatedFile.toString(), "--checks=*", "--quiet", "--", "-std=c++${validator.cppStandard}") - validator.includes.forEach { args.add("-I$it") } - return LFCommand.get("clang-tidy", args, validator.fileConfig.outPath) + CLANG_TIDY(listOf(), { cppValidator -> CppValidationStrategy( + { _, _, _ -> }, + HumanReadableReportingStrategy(CLANG_ERROR_LINE, CLANG_LABEL), + 5, + { generatedFile: Path -> + val args = mutableListOf(generatedFile.toString(), "--checks=*", "--quiet", "--", "-std=c++${cppValidator.cppStandard}") + cppValidator.includes.forEach { args.add("-I$it") } + LFCommand.get("clang-tidy", args, cppValidator.fileConfig.srcGenPkgPath) } - }, - GXX(PerLineReportingStrategy(gxxErrorLine), { _, _, _ -> }, 1) { - override fun getCommand(validator: CppValidator, generatedFile: Path): LFCommand? { - val args: MutableList = mutableListOf("-fsyntax-only", "-Wall", "-std=c++${validator.cppStandard}") - validator.includes.forEach { args.add("-I$it") } + )}), + CLANG(listOf("Clang", "AppleClang"), { cppValidator -> CppValidationStrategy( + HumanReadableReportingStrategy(CLANG_ERROR_LINE, CLANG_LABEL), + { _, _, _ -> }, + 0, + { generatedFile: Path -> + val args: MutableList = mutableListOf("-fsyntax-only", "-Wall", "-std=c++${cppValidator.cppStandard}") + cppValidator.includes.forEach { args.add("-I$it") } args.add(generatedFile.toString()) - return LFCommand.get("g++", args, validator.fileConfig.outPath) + LFCommand.get("clang++", args, cppValidator.fileConfig.srcGenPkgPath) } - }; - - /** - * Returns the command that produces validation - * output in association with `generatedFile`. - * @param validator the C++ validator instance - * corresponding to the relevant group of generated - * files - */ - abstract fun getCommand(validator: CppValidator, generatedFile: Path): LFCommand? - } - - /** - * Validates this Validator's group of generated files. - * @param cancelIndicator the cancel indicator for the - * current operation - */ - fun doValidate(cancelIndicator: CancelIndicator) { - if (!cmakeCachePath.toFile().exists()) return - val futures = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()) - .invokeAll(getValidationStrategies().map { Callable {it.second.run(cancelIndicator); it} }) - for (f in futures) { - val (strategy, command) = f.get() - strategy.errorReportingStrategy.report(command.errors.toString(), errorReporter, codeMaps) - strategy.outputReportingStrategy.report(command.output.toString(), errorReporter, codeMaps) - } - } - - /** - * Runs the given command, reports any messages it - * produces, and returns its return code. - */ - fun run(compileCommand: LFCommand, cancelIndicator: CancelIndicator): Int { - val returnCode = compileCommand.run(cancelIndicator) - val (errorReportingStrategy, outputReportingStrategy) = getBuildReportingStrategies() - errorReportingStrategy.report(compileCommand.errors.toString(), errorReporter, codeMaps) - outputReportingStrategy.report(compileCommand.output.toString(), errorReporter, codeMaps) - return returnCode + )}), + GXX(listOf("GNU"), { cppValidator -> CppValidationStrategy( + HumanReadableReportingStrategy(GXX_ERROR_LINE, GXX_LABEL), + { _, _, _ -> }, + 1, + { generatedFile: Path -> + val args: MutableList = mutableListOf("-fsyntax-only", "-Wall", "-std=c++${cppValidator.cppStandard}") + cppValidator.includes.forEach { args.add("-I$it") } + args.add(generatedFile.toString()) + LFCommand.get("g++", args, cppValidator.fileConfig.srcGenPkgPath) + } + )}), + MSVC(listOf("MSVC"), { cppValidator -> CppValidationStrategy( + { _, _, _ -> }, + HumanReadableReportingStrategy(MSVC_ERROR_LINE, MSVC_LABEL), + 3, + { generatedFile: Path -> + cppValidator.cmakeGeneratorInstance?.let { path -> + val setUpDeveloperEnvironment: Path = Paths.get(path) + .resolve("Common7${File.separator}Tools${File.separator}VsDevCmd.bat") + val args: MutableList = mutableListOf("&", "cl", "/Zs", "/diagnostics:column", "/std:c++${cppValidator.cppStandard}") + cppValidator.includes.forEach { args.add("/I$it") } + args.add(generatedFile.toString()) + LFCommand.get(setUpDeveloperEnvironment.toString(), args, cppValidator.fileConfig.srcGenPkgPath) + } + } + )}); } /** - * Returns the appropriate output and error reporting + * Return the appropriate output and error reporting * strategies for the build process carried out by * CMake and Make. */ - private fun getBuildReportingStrategies(): Pair { - // This is a rather silly function, but it is left as-is because the appropriate reporting strategy - // could in principle be a function of the build process carried out by CMake. It just so happens - // that the compilers that are supported in the current version seem to use the same reporting format, - // so this ends up being a constant function. - return Pair(CppValidationStrategy.GXX.errorReportingStrategy, CppValidationStrategy.GXX.errorReportingStrategy) - } - - /** - * Returns the validation strategies and validation - * commands corresponding to each generated file. - * @return the validation strategies and validation - * commands corresponding to each generated file - */ - private fun getValidationStrategies(): List> { - val commands = mutableListOf>() - for (generatedFile: Path in codeMaps.keys) { - val p = getValidationStrategy(generatedFile) - val (strategy, command) = p - if (strategy == null || command == null) continue - commands.add(Pair(strategy, command)) + override fun getBuildReportingStrategies(): Pair { + val compilerId: String = getFromCache(CppCmakeGenerator.compilerIdName) ?: "GNU" // This is just a guess. + val mostSimilarValidationStrategy = CppValidationStrategyFactory.values().find { it.compilerIds.contains(compilerId) } + if (mostSimilarValidationStrategy === null) { + return Pair(DiagnosticReporting.Strategy { _, _, _ -> }, DiagnosticReporting.Strategy { _, _, _ -> }) } - return commands + return Pair( + mostSimilarValidationStrategy.create(this).errorReportingStrategy, + mostSimilarValidationStrategy.create(this).outputReportingStrategy, + ) } - /** - * Returns the validation strategy and command - * corresponding to the given file, if such a strategy - * and command are available. - * @return the validation strategy and command - * corresponding to the given file, if such a strategy - * and command are available - */ - private fun getValidationStrategy(generatedFile: Path): Pair { - for (strategy in CppValidationStrategy.values().sortedBy {strategy -> strategy.time}) { - val validateCommand = strategy.getCommand(this, generatedFile) // - if (validateCommand != null) { - return Pair(strategy, validateCommand) + private fun getFromCache(variableName: String): String? { + if (cmakeCache.exists()) cmakeCache.useLines { + for (line in it) { + val matcher = CMAKE_CACHED_VARIABLE.matcher(line) + if (matcher.matches() && matcher.group("name") == variableName) return matcher.group("value") } } - return Pair(null, null) + return null } /** The include directories required by the generated files. */ private val includes: List - get() { - for (line in cmakeCache) { - val matcher = cmakeIncludes.matcher(line) - if (matcher.matches()) return matcher.group("includes").split(';') - } - return listOf() - } + get() = getFromCache(CppCmakeGenerator.includesVarName)?.split(';') ?: listOf() /** The C++ standard used by the generated files. */ - private val cppStandard: String - get() { - for (line in cmakeCache) { - val matcher = cmakeCxxStandard.matcher(line) - if (matcher.matches()) return matcher.group("cppStandard") - } - return "" - } + private val cppStandard: String? + get() = getFromCache("CMAKE_CXX_STANDARD") + + /** The desired instance of the C++ build system. */ + private val cmakeGeneratorInstance: String? + get() = getFromCache("CMAKE_GENERATOR_INSTANCE") - /** The content of the CMake cache. */ // FIXME: Most of this data will never be used. Should it really be cached? - private val cmakeCache: List by lazy { Files.readAllLines(cmakeCachePath) } + /** The CMake cache. */ + private val cmakeCache: File = fileConfig.buildPath.resolve("CMakeCache.txt").toFile() - /** The path to the CMake cache. */ - private val cmakeCachePath: Path = fileConfig.buildPath.resolve("CMakeCache.txt") -} \ No newline at end of file + override fun getPossibleStrategies(): Collection = CppValidationStrategyFactory.values().map { it.create(this) } +} diff --git a/org.lflang/src/org/lflang/generator/python/PythonGenerator.xtend b/org.lflang/src/org/lflang/generator/python/PythonGenerator.xtend index 0949d2f470..7632876132 100644 --- a/org.lflang/src/org/lflang/generator/python/PythonGenerator.xtend +++ b/org.lflang/src/org/lflang/generator/python/PythonGenerator.xtend @@ -27,8 +27,10 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package org.lflang.generator.python import java.io.File -import java.nio.file.Paths +import java.nio.file.Path import java.util.ArrayList +import java.util.HashMap +import java.util.HashSet import java.util.LinkedHashSet import java.util.List import java.util.regex.Pattern @@ -40,6 +42,7 @@ import org.lflang.FileConfig import org.lflang.InferredType import org.lflang.Target import org.lflang.TargetConfig +import org.lflang.TargetConfig.Mode import org.lflang.TargetProperty.CoordinationType import org.lflang.federated.FedFileConfig import org.lflang.federated.FederateInstance @@ -47,6 +50,7 @@ import org.lflang.federated.PythonGeneratorExtension import org.lflang.federated.launcher.FedPyLauncher import org.lflang.federated.serialization.FedNativePythonSerialization import org.lflang.federated.serialization.SupportedSerializers +import org.lflang.generator.CodeMap import org.lflang.generator.GeneratorResult import org.lflang.generator.IntegratedBuilder import org.lflang.generator.JavaGeneratorUtils @@ -72,7 +76,6 @@ import org.lflang.lf.StateVar import org.lflang.lf.TriggerRef import org.lflang.lf.Value import org.lflang.lf.VarRef -import org.lflang.util.LFCommand import static extension org.lflang.ASTUtils.* import static extension org.lflang.JavaAstUtils.* @@ -163,6 +166,8 @@ class PythonGenerator extends CGenerator { // Regular expression pattern for pointer types. The star at the end has to be visible. static final Pattern pointerPatternVariable = Pattern.compile("^\\s*+(\\w+)\\s*\\*\\s*$"); + + val protoNames = new HashSet() //////////////////////////////////////////// //// Public methods @@ -760,10 +765,20 @@ class PythonGenerator extends CGenerator { * @return the code body */ def generatePythonCode(FederateInstance federate) ''' - from LinguaFranca«topLevelName» import * - from LinguaFrancaBase.constants import * #Useful constants - from LinguaFrancaBase.functions import * #Useful helper functions - from LinguaFrancaBase.classes import * #Useful classes + # List imported names, but do not use pylint's --extension-pkg-allow-list option + # so that these names will be assumed present without having to compile and install. + from LinguaFranca«topLevelName» import ( # pylint: disable=no-name-in-module + Tag, action_capsule_t, compare_tags, get_current_tag, get_elapsed_logical_time, + get_elapsed_physical_time, get_logical_time, get_microstep, get_physical_time, + get_start_time, port_capsule, port_instance_token, request_stop, schedule_copy, + start + ) + from LinguaFrancaBase.constants import BILLION, FOREVER, NEVER, instant_t, interval_t + from LinguaFrancaBase.functions import ( + DAY, DAYS, HOUR, HOURS, MINUTE, MINUTES, MSEC, MSECS, NSEC, NSECS, SEC, SECS, USEC, + USECS, WEEK, WEEKS + ) + from LinguaFrancaBase.classes import Make import sys import copy @@ -810,7 +825,8 @@ class PythonGenerator extends CGenerator { if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } - JavaGeneratorUtils.writeSourceCodeToFile(generatePythonCode(federate), file.absolutePath) + val codeMaps = #{file.toPath -> CodeMap.fromGeneratedCode(generatePythonCode(federate).toString)} + JavaGeneratorUtils.writeSourceCodeToFile(codeMaps.get(file.toPath).generatedCode, file.absolutePath) val setupPath = fileConfig.getSrcGenPath.resolve("setup.py") // Handle Python setup @@ -824,7 +840,7 @@ class PythonGenerator extends CGenerator { // Create the setup file JavaGeneratorUtils.writeSourceCodeToFile(generatePythonSetupFile, setupPath.toString) - + return codeMaps } /** @@ -836,7 +852,7 @@ class PythonGenerator extends CGenerator { '''python3''', #["-m", "pip", "install", "--force-reinstall", "."], fileConfig.srcGenPath) - + if (installCmd === null) { errorReporter.reportError( "The Python target requires Python >= 3.6, pip >= 20.0.2, and setuptools >= 45.2.0-1 to compile the generated code. " + @@ -964,6 +980,7 @@ class PythonGenerator extends CGenerator { pythonPreamble.append(''' import «rootFilename»_pb2 as «rootFilename» ''') + protoNames.add(rootFilename) } } case ROS2: { @@ -989,6 +1006,7 @@ class PythonGenerator extends CGenerator { //val protoc = createCommand("protoc", #['''--python_out=src-gen/«topLevelName»''', topLevelName], codeGenConfig.outPath) if (protoc === null) { errorReporter.reportError("Processing .proto files requires libprotoc >= 3.6.1") + return } val returnCode = protoc.run(cancelIndicator) if (returnCode == 0) { @@ -1215,7 +1233,7 @@ class PythonGenerator extends CGenerator { * otherwise report an error and return false. */ override isOSCompatible() { - if (CCompiler.isHostWindows) { + if (JavaGeneratorUtils.isHostWindows) { if (isFederated) { errorReporter.reportError( "Federated LF programs with a Python target are currently not supported on Windows. Exiting code generation." @@ -1255,17 +1273,17 @@ class PythonGenerator extends CGenerator { targetConfig.noCompile = compileStatus - if (errorsOccurred) return; + if (errorsOccurred) { + context.unsuccessfulFinish() + return; + } var baseFileName = topLevelName // Keep a separate file config for each federate val oldFileConfig = fileConfig; var federateCount = 0; + val codeMaps = new HashMap for (federate : federates) { - compilingFederatesContext.reportProgress( - String.format("Installing Python modules. %d/%d complete...", federateCount, federates.size()), - 100 * federateCount / federates.size() - ) federateCount++ if (isFederated) { topLevelName = baseFileName + '_' + federate.name @@ -1273,10 +1291,22 @@ class PythonGenerator extends CGenerator { } // Don't generate code if there is no main reactor if (this.main !== null) { - generatePythonFiles(fsa, federate) + val codeMapsForFederate = generatePythonFiles(fsa, federate) + codeMaps.putAll(codeMapsForFederate) if (!targetConfig.noCompile) { + compilingFederatesContext.reportProgress( + String.format("Validating %d/%d sets of generated files...", federateCount, federates.size()), + 100 * federateCount / federates.size() + ) // If there are no federates, compile and install the generated code - pythonCompileCode(context) + new PythonValidator(fileConfig, errorReporter, codeMaps, protoNames).doValidate(context.cancelIndicator) + if (!errorsOccurred() && context.mode != Mode.LSP_MEDIUM) { + compilingFederatesContext.reportProgress( + String.format("Validation complete. Compiling and installing %d/%d Python modules...", federateCount, federates.size()), + 100 * federateCount / federates.size() + ) + pythonCompileCode(context) // Why is this invoked here if the current federate is not a parameter? + } } else { printSetupInfo(); } @@ -1292,14 +1322,12 @@ class PythonGenerator extends CGenerator { } // Restore filename topLevelName = baseFileName - if (context.getCancelIndicator().isCanceled()) { - context.finish(GeneratorResult.CANCELLED) - } else if (errorReporter.getErrorsOccurred()) { - context.finish(GeneratorResult.FAILED) + if (errorReporter.getErrorsOccurred()) { + context.unsuccessfulFinish() } else if (!isFederated) { - context.finish(GeneratorResult.Status.COMPILED, '''«topLevelName».py''', fileConfig.srcGenPath, fileConfig, null, "python3") + context.finish(GeneratorResult.Status.COMPILED, '''«topLevelName».py''', fileConfig.srcGenPath, fileConfig, codeMaps, "python3") } else { - context.finish(GeneratorResult.Status.COMPILED, fileConfig.name, fileConfig.binPath, fileConfig, null, "bash") + context.finish(GeneratorResult.Status.COMPILED, fileConfig.name, fileConfig.binPath, fileConfig, codeMaps, "bash") } } diff --git a/org.lflang/src/org/lflang/generator/python/PythonValidator.java b/org.lflang/src/org/lflang/generator/python/PythonValidator.java new file mode 100644 index 0000000000..dbbd60fa89 --- /dev/null +++ b/org.lflang/src/org/lflang/generator/python/PythonValidator.java @@ -0,0 +1,335 @@ +package org.lflang.generator.python; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.lsp4j.DiagnosticSeverity; + +import org.lflang.ErrorReporter; +import org.lflang.FileConfig; +import org.lflang.generator.CodeMap; +import org.lflang.generator.DiagnosticReporting; +import org.lflang.generator.DiagnosticReporting.Strategy; +import org.lflang.generator.Position; +import org.lflang.generator.ValidationStrategy; +import org.lflang.generator.Validator; +import org.lflang.util.LFCommand; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +/** + * A validator for generated Python. + * + * @author Peter Donovan + */ +public class PythonValidator extends Validator { + + /** The pattern that diagnostics from the Python compiler typically follow. */ + private static final Pattern DIAGNOSTIC_MESSAGE_PATTERN = Pattern.compile( + "(\\*\\*\\*)?\\s*File \"(?.*?\\.py)\", line (?\\d+)" + ); + /** The pattern typically followed by the message that typically follows the main diagnostic line. */ + private static final Pattern MESSAGE = Pattern.compile("\\w*Error: .*"); + /** An alternative pattern that at least some diagnostics from the Python compiler may follow. */ + private static final Pattern ALT_DIAGNOSTIC_MESSAGE_PATTERN = Pattern.compile( + ".*Error:.*line (?\\d+)\\)" + ); + + /** The JSON parser. */ + private static final ObjectMapper mapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private final Set protoNames; + + /** + * The message format of Pylint's JSON output. + */ + @SuppressWarnings( {"FieldCanBeLocal", "unused"}) // Unused fields are included for completeness. + private static final class PylintMessage { + private String type; + private String module; + private String obj; + private int line; + private int column; + private int endLine; + private int endColumn; + private Path path; + private String symbol; + private String message; + private String messageId; + public void setType(String type) { this.type = type; } + public void setModule(String module) { this.module = module; } + public void setObj(String obj) { this.obj = obj; } + public void setLine(int line) { this.line = line; } + public void setColumn(int column) { this.column = column; } + public void setEndLine(int endLine) { this.endLine = endLine; } + public void setEndColumn(int endColumn) { this.endColumn = endColumn; } + public void setPath(String path) { this.path = Path.of(path); } + public void setSymbol(String symbol) { this.symbol = symbol; } + public void setMessage(String message) { this.message = message; } + @JsonProperty("message-id") + public void setMessageId(String messageId) { this.messageId = messageId; } + public Position getStart() { return Position.fromZeroBased(line - 1, column); } + public Position getEnd() { return Position.fromZeroBased(endLine - 1, endColumn); } + public Path getPath(Path relativeTo) { return relativeTo.resolve(path); } + public DiagnosticSeverity getSeverity() { + // The following is consistent with VS Code's default behavior for pure Python: + // https://code.visualstudio.com/docs/python/linting#_pylint + switch (type.toLowerCase()) { + case "refactor": + return DiagnosticSeverity.Hint; + case "warning": + return DiagnosticSeverity.Warning; + case "error": + case "fatal": + return DiagnosticSeverity.Error; + case "convention": + default: + return DiagnosticSeverity.Information; + } + } + } + + private static final Pattern PylintNoNamePattern = Pattern.compile("Instance of '(?\\w+)' has no .*"); + + private final FileConfig fileConfig; + private final ErrorReporter errorReporter; + private final ImmutableMap codeMaps; + + /** + * Initialize a {@code PythonValidator} for a build process using {@code fileConfig} and + * report errors to {@code errorReporter}. + * @param fileConfig The file configuration of this build. + * @param errorReporter The reporter to which diagnostics should be sent. + * @param codeMaps A mapping from generated file paths to code maps that map them back to + * LF sources. + * @param protoNames The names of any protocol buffer message types that are used in the LF + * program being built. + */ + public PythonValidator( + FileConfig fileConfig, + ErrorReporter errorReporter, + Map codeMaps, + Set protoNames + ) { + super(errorReporter, codeMaps); + this.fileConfig = fileConfig; + this.errorReporter = errorReporter; + this.codeMaps = ImmutableMap.copyOf(codeMaps); + this.protoNames = ImmutableSet.copyOf(protoNames); + } + + @Override + protected Collection getPossibleStrategies() { return List.of( + new ValidationStrategy() { + @Override + public LFCommand getCommand(Path generatedFile) { + return LFCommand.get( + "python3", + List.of("-c", "import compileall; compileall.compile_dir('.', quiet=1)"), + fileConfig.getSrcGenPkgPath() + ); + } + + @Override + public Strategy getErrorReportingStrategy() { + return (a, b, c) -> {}; + } + + @Override + public Strategy getOutputReportingStrategy() { + return (String validationOutput, ErrorReporter errorReporter, Map map) -> { + String[] lines = (validationOutput + "\n\n\n").lines().toArray(String[]::new); + for (int i = 0; i < lines.length - 3; i++) { + if (!tryReportTypical(lines, i)) { + tryReportAlternative(lines, i); + } + } + }; + } + + /** + * Try to report a typical error message from the Python compiler. + * + * @param lines The lines of output from the compiler. + * @param i The current index at which a message may start. Guaranteed to be less + * than + * {@code lines.length - 3}. + * @return Whether an error message was reported. + */ + private boolean tryReportTypical(String[] lines, int i) { + Matcher main = DIAGNOSTIC_MESSAGE_PATTERN.matcher(lines[i]); + Matcher messageMatcher = MESSAGE.matcher(lines[i + 3]); + String message = messageMatcher.matches() ? messageMatcher.group() : "Syntax Error"; + if (main.matches()) { + int line = Integer.parseInt(main.group("line")); + CodeMap map = codeMaps.get(fileConfig.getSrcGenPkgPath().resolve(Path.of(main.group("path"))).normalize()); + Position genPosition = Position.fromOneBased(line, Integer.MAX_VALUE); // Column is just a placeholder. + if (map == null) { + errorReporter.report(null, DiagnosticSeverity.Error, message, 1); // Undesirable fallback + } else { + for (Path lfFile : map.lfSourcePaths()) { + Position lfPosition = map.adjusted(lfFile, genPosition); + // TODO: We could be more precise than just getting the right line, but the way the output + // is formatted (with leading whitespace possibly trimmed) does not make it easy. + errorReporter.report(lfFile, DiagnosticSeverity.Error, message, lfPosition.getOneBasedLine()); + } + } + return true; + } + return false; + } + + /** + * Try to report an alternative error message from the Python compiler. + * + * @param lines The lines of output from the compiler. + * @param i The current index at which a message may start. + */ + private void tryReportAlternative(String[] lines, int i) { + Matcher main = ALT_DIAGNOSTIC_MESSAGE_PATTERN.matcher(lines[i]); + if (main.matches()) { + int line = Integer.parseInt(main.group("line")); + Iterable relevantMaps = codeMaps.keySet().stream() + .filter(p -> main.group().contains(p.getFileName().toString())) + .map(codeMaps::get)::iterator; + for (CodeMap map : relevantMaps) { // There should almost always be exactly one of these + for (Path lfFile : map.lfSourcePaths()) { + errorReporter.report( + lfFile, + DiagnosticSeverity.Error, + main.group().replace("*** ", "").replace("Sorry: ", ""), + map.adjusted(lfFile, Position.fromOneBased(line, 1)).getOneBasedLine() + ); + } + } + } + } + + @Override + public boolean isFullBatch() { + return true; + } + + @Override + public int getPriority() { + return 0; + } + }, + new ValidationStrategy() { + @Override + public LFCommand getCommand(Path generatedFile) { + return LFCommand.get( + "pylint", + List.of("--output-format=json", generatedFile.getFileName().toString()), + fileConfig.getSrcGenPath() + ); + } + + @Override + public Strategy getErrorReportingStrategy() { + return (a, b, c) -> {}; + } + + @Override + public Strategy getOutputReportingStrategy() { + return (validationOutput, errorReporter, codeMaps) -> { + try { + for (PylintMessage message : mapper.readValue(validationOutput, PylintMessage[].class)) { + if (shouldIgnore(message)) continue; + CodeMap map = codeMaps.get(message.getPath(fileConfig.getSrcGenPath())); + if (map != null) { + for (Path lfFile : map.lfSourcePaths()) { + Function adjust = p -> map.adjusted(lfFile, p); + String humanMessage = DiagnosticReporting.messageOf( + message.message, + message.getPath(fileConfig.getSrcGenPath()), + message.getStart() + ); + Position lfStart = adjust.apply(message.getStart()); + Position lfEnd = adjust.apply(message.getEnd()); + bestEffortReport( + errorReporter, + adjust, + lfStart, + lfEnd, + lfFile, + message.getSeverity(), + humanMessage + ); + } + } + } + } catch (JsonProcessingException e) { + // This should be impossible unless Pylint's API changes. Maybe it's fine to fail quietly + // like this in case that happens -- this will go to stderr, so you can see it if you look. + e.printStackTrace(); + } + }; + } + + /** + * Return whether the given message should be ignored. + * @param message A Pylint message that is a candidate to be reported. + * @return whether {@code message} should be reported. + */ + private boolean shouldIgnore(PylintMessage message) { + // Code generation does not preserve whitespace, so this check is unreliable. + if (message.symbol.equals("trailing-whitespace")) return true; + // This filters out Pylint messages concerning missing members in types defined by protocol buffers. + // FIXME: Make this unnecessary, perhaps using https://github.com/nelfin/pylint-protobuf. + Matcher matcher = PylintNoNamePattern.matcher(message.message); + return message.symbol.equals("no-member") + && matcher.matches() && protoNames.contains(matcher.group("name")); + } + + /** Make a best-effort attempt to place the diagnostic on the correct line. */ + private void bestEffortReport( + ErrorReporter errorReporter, + Function adjust, + Position lfStart, + Position lfEnd, + Path file, + DiagnosticSeverity severity, + String humanMessage + ) { + if (!lfEnd.equals(Position.ORIGIN) && !lfStart.equals(Position.ORIGIN)) { // Ideal case + errorReporter.report(file, severity, humanMessage, lfStart, lfEnd); + } else { // Fallback: Try to report on the correct line, or failing that, just line 1. + if (lfStart.equals(Position.ORIGIN)) lfStart = adjust.apply( + Position.fromZeroBased(lfStart.getZeroBasedLine(), Integer.MAX_VALUE) + ); + // FIXME: It might be better to improve style of generated code instead of quietly returning here. + if (lfStart.equals(Position.ORIGIN) && severity != DiagnosticSeverity.Error) return; + errorReporter.report(file, severity, humanMessage, lfStart.getOneBasedLine()); + } + } + + @Override + public boolean isFullBatch() { + return false; + } + + @Override + public int getPriority() { + return 1; + } + } + ); } + + @Override + protected Pair getBuildReportingStrategies() { + return new Pair<>((a, b, c) -> {}, (a, b, c) -> {}); + } +} diff --git a/org.lflang/src/org/lflang/generator/rust/RustEmitter.kt b/org.lflang/src/org/lflang/generator/rust/RustEmitter.kt index 713fc13ac4..b9339986f4 100644 --- a/org.lflang/src/org/lflang/generator/rust/RustEmitter.kt +++ b/org.lflang/src/org/lflang/generator/rust/RustEmitter.kt @@ -25,9 +25,11 @@ package org.lflang.generator.rust import org.lflang.FileConfig +import org.lflang.generator.CodeMap import org.lflang.generator.PrependOperator import org.lflang.generator.rust.RustEmitter.generateRustProject import java.nio.file.Files +import java.nio.file.Path /** @@ -39,30 +41,31 @@ import java.nio.file.Files */ object RustEmitter : RustEmitterBase() { - fun generateRustProject(fileConfig: RustFileConfig, gen: GenerationInfo) { + fun generateRustProject(fileConfig: RustFileConfig, gen: GenerationInfo): Map { + val codeMaps = mutableMapOf() - fileConfig.emit("Cargo.toml") { + fileConfig.emit(codeMaps, "Cargo.toml") { with(RustCargoTomlEmitter) { makeCargoTomlFile(gen) } } // this file determines the default toolchain for Cargo, useful for CLion too - fileConfig.emit("rust-toolchain") { + fileConfig.emit(codeMaps, "rust-toolchain") { this += "nightly" } // if singleFile, this file will contain every module. - fileConfig.emit("src/main.rs") { + fileConfig.emit(codeMaps, "src/main.rs") { with(RustMainFileEmitter) { makeMainFile(gen) } } if (!gen.properties.singleFile) { - fileConfig.emit("src/reactors/mod.rs") { makeReactorsAggregateModule(gen) } + fileConfig.emit(codeMaps, "src/reactors/mod.rs") { makeReactorsAggregateModule(gen) } for (reactor in gen.reactors) { - fileConfig.emit("src/reactors/${reactor.names.modFileName}.rs") { + fileConfig.emit(codeMaps, "src/reactors/${reactor.names.modFileName}.rs") { with(RustReactorEmitter) { emitReactorModule(reactor) } @@ -79,6 +82,7 @@ object RustEmitter : RustEmitterBase() { FileConfig.copyFile(modPath, target) } } + return codeMaps } private fun Emitter.makeReactorsAggregateModule(gen: GenerationInfo) { diff --git a/org.lflang/src/org/lflang/generator/rust/RustFileConfig.kt b/org.lflang/src/org/lflang/generator/rust/RustFileConfig.kt index c32329cf07..f558580c70 100644 --- a/org.lflang/src/org/lflang/generator/rust/RustFileConfig.kt +++ b/org.lflang/src/org/lflang/generator/rust/RustFileConfig.kt @@ -27,6 +27,7 @@ package org.lflang.generator.rust import org.eclipse.emf.ecore.resource.Resource import org.eclipse.xtext.generator.IFileSystemAccess2 import org.lflang.FileConfig +import org.lflang.generator.CodeMap import org.lflang.generator.LFGeneratorContext import java.io.Closeable import java.io.IOException @@ -46,16 +47,14 @@ class RustFileConfig(resource: Resource, fsa: IFileSystemAccess2, context: LFGen deleteDirectory(outPath.resolve("target")) } - inline fun emit(p: Path, f: Emitter.() -> Unit) { - //System.err.println("Generating file ${srcGenPath.relativize(p)}...") - //val milliTime = + inline fun emit(codeMaps: MutableMap, p: Path, f: Emitter.() -> Unit) { measureTimeMillis { - Emitter(p).use { it.f() } + Emitter(codeMaps, p).use { it.f() } } - //System.err.println("Done in $milliTime ms.") } - inline fun emit(pathRelativeToOutDir: String, f: Emitter.() -> Unit): Unit = emit(srcGenPath.resolve(pathRelativeToOutDir), f) + inline fun emit(codeMaps: MutableMap, pathRelativeToOutDir: String, f: Emitter.() -> Unit): Unit + = emit(codeMaps, srcGenPath.resolve(pathRelativeToOutDir), f) } @@ -64,6 +63,7 @@ class RustFileConfig(resource: Resource, fsa: IFileSystemAccess2, context: LFGen * the object writes to the file. */ class Emitter( + private val codeMaps: MutableMap, /** File to which this emitter should write. */ private val output: Path, ) : Closeable { @@ -105,7 +105,9 @@ class Emitter( override fun close() { Files.createDirectories(output.parent) - Files.writeString(output, sb) + val codeMap = CodeMap.fromGeneratedCode(sb.toString()) + codeMaps[output] = codeMap + Files.writeString(output, codeMap.generatedCode) } } diff --git a/org.lflang/src/org/lflang/generator/rust/RustGenerator.kt b/org.lflang/src/org/lflang/generator/rust/RustGenerator.kt index 7960bd8f7a..1ff693f01f 100644 --- a/org.lflang/src/org/lflang/generator/rust/RustGenerator.kt +++ b/org.lflang/src/org/lflang/generator/rust/RustGenerator.kt @@ -28,8 +28,10 @@ import org.eclipse.emf.ecore.resource.Resource import org.eclipse.xtext.generator.IFileSystemAccess2 import org.lflang.ErrorReporter import org.lflang.Target +import org.lflang.TargetConfig import org.lflang.TargetProperty.BuildType import org.lflang.generator.canGenerate +import org.lflang.generator.CodeMap import org.lflang.generator.GeneratorBase import org.lflang.generator.GeneratorResult import org.lflang.generator.IntegratedBuilder @@ -40,6 +42,7 @@ import org.lflang.lf.Action import org.lflang.lf.VarRef import org.lflang.scoping.LFGlobalScopeProvider import java.nio.file.Files +import java.nio.file.Path /** * Generator for the Rust target language. The generation is @@ -72,10 +75,10 @@ class RustGenerator( Files.createDirectories(fileConfig.srcGenPath) val gen = RustModelBuilder.makeGenerationInfo(targetConfig, reactors) - RustEmitter.generateRustProject(fileConfig, gen) + val codeMaps: Map = RustEmitter.generateRustProject(fileConfig, gen) if (targetConfig.noCompile || errorsOccurred()) { - context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(null)) + context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(codeMaps)) println("Exiting before invoking target compiler.") } else { context.reportProgress( @@ -83,11 +86,12 @@ class RustGenerator( ) val exec = fileConfig.binPath.toAbsolutePath().resolve(gen.executableName) Files.deleteIfExists(exec) // cleanup, cargo doesn't do it - invokeRustCompiler(context, gen.executableName) + if (context.mode == TargetConfig.Mode.LSP_MEDIUM) RustValidator(fileConfig, errorReporter, codeMaps).doValidate(context.cancelIndicator) + else invokeRustCompiler(context, gen.executableName, codeMaps) } } - private fun invokeRustCompiler(context: LFGeneratorContext, executableName: String) { + private fun invokeRustCompiler(context: LFGeneratorContext, executableName: String, codeMaps: Map) { val args = mutableListOf().apply { this += listOf( @@ -113,6 +117,10 @@ class RustGenerator( } this += targetConfig.compilerFlags + + if (context.mode != TargetConfig.Mode.STANDALONE) { + this += listOf("--message-format", "json") + } } val cargoCommand = commandFactory.createCommand( @@ -120,17 +128,17 @@ class RustGenerator( fileConfig.srcGenPath.toAbsolutePath() ) ?: return - val cargoReturnCode = cargoCommand.run(context.cancelIndicator) + val cargoReturnCode = RustValidator(fileConfig, errorReporter, codeMaps).run(cargoCommand, context.cancelIndicator) if (cargoReturnCode == 0) { println("SUCCESS (compiling generated Rust code)") println("Generated source code is in ${fileConfig.srcGenPath}") println("Compiled binary is in ${fileConfig.binPath}") - context.finish(GeneratorResult.Status.COMPILED, executableName, fileConfig, null) + context.finish(GeneratorResult.Status.COMPILED, executableName, fileConfig, codeMaps) } else if (context.cancelIndicator.isCanceled) { context.finish(GeneratorResult.CANCELLED) } else { - errorReporter.reportError("cargo failed with error code $cargoReturnCode") + if (!errorsOccurred()) errorReporter.reportError("cargo failed with error code $cargoReturnCode") context.finish(GeneratorResult.FAILED) } } diff --git a/org.lflang/src/org/lflang/generator/rust/RustModel.kt b/org.lflang/src/org/lflang/generator/rust/RustModel.kt index 988e038019..012384c3d6 100644 --- a/org.lflang/src/org/lflang/generator/rust/RustModel.kt +++ b/org.lflang/src/org/lflang/generator/rust/RustModel.kt @@ -658,7 +658,7 @@ fun Reactor.instantiateType(formalType: TargetCode, typeArgs: List): T else { val typeArgsByName = typeParams.mapIndexed { i, tp -> Pair(tp.identifier, typeArgs[i].toText()) }.toMap() - formalType.replace(IDENT_REGEX) { + CodeMap.fromGeneratedCode(formalType).generatedCode.replace(IDENT_REGEX) { typeArgsByName[it.value] ?: it.value } } @@ -669,7 +669,7 @@ fun Reactor.instantiateType(formalType: TargetCode, typeArgs: List): T */ private val TypeParm.identifier: String get() { - val targetCode = toText() + val targetCode = CodeMap.fromGeneratedCode(toText()).generatedCode return IDENT_REGEX.find(targetCode.trimStart())?.value ?: throw InvalidLfSourceException( "No identifier in type param `$targetCode`", diff --git a/org.lflang/src/org/lflang/generator/rust/RustValidator.kt b/org.lflang/src/org/lflang/generator/rust/RustValidator.kt new file mode 100644 index 0000000000..67f4a1c2b8 --- /dev/null +++ b/org.lflang/src/org/lflang/generator/rust/RustValidator.kt @@ -0,0 +1,173 @@ +package org.lflang.generator.rust + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.core.JsonProcessingException +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import org.eclipse.lsp4j.DiagnosticSeverity +import org.lflang.ErrorReporter +import org.lflang.FileConfig +import org.lflang.generator.CodeMap +import org.lflang.generator.DiagnosticReporting +import org.lflang.generator.Position +import org.lflang.generator.ValidationStrategy +import org.lflang.generator.Validator +import org.lflang.util.LFCommand +import java.nio.file.Path +import java.nio.file.Paths + +/** + * A validator for generated Rust. + * + * @author Peter Donovan + */ +@Suppress("ArrayInDataClass") // Data classes here must not be used in data structures such as hashmaps. +class RustValidator( + private val fileConfig: FileConfig, + errorReporter: ErrorReporter, + codeMaps: Map +): Validator(errorReporter, codeMaps) { + companion object { + private val mapper = ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private const val COMPILER_MESSAGE_REASON = "compiler-message" + } + // See the following reference for details on cargo metadata: https://doc.rust-lang.org/cargo/commands/cargo-metadata.html + private data class RustMetadata( + // Other fields exist, but we don't need them. The mapper is configured not to fail on unknown properties. + @JsonProperty("workspace_root") private val _workspaceRoot: String + ) { + val workspaceRoot: Path + get() = Paths.get(_workspaceRoot) + } + // See the following references for details on these data classes: + // * https://doc.rust-lang.org/cargo/reference/external-tools.html#json-messages + // * https://doc.rust-lang.org/rustc/json.html + private data class RustOutput( + @JsonProperty("reason") val reason: String, + ) + private data class RustCompilerMessage( + @JsonProperty("reason") val reason: String, + @JsonProperty("package_id") val packageId: String, + @JsonProperty("manifest_path") val manifestPath: String, + @JsonProperty("target") val target: RustCompilerTarget, + @JsonProperty("message") val message: RustDiagnostic + ) + private data class RustCompilerTarget( + @Suppress("ArrayInDataClass") @JsonProperty("kind") val kind: Array, + @Suppress("ArrayInDataClass") @JsonProperty("crate_types") val crateTypes: Array, + @JsonProperty("name") val name: String, + @JsonProperty("src_path") val srcPath: String, + @JsonProperty("edition") val edition: String, + @Suppress("ArrayInDataClass") @JsonProperty("required_features") val requiredFeatures: Array?, + @JsonProperty("doctest") val doctest: Boolean + ) + private data class RustDiagnostic( + @JsonProperty("message") val message: String, + @JsonProperty("code") val code: RustDiagnosticCode?, + @JsonProperty("level") val level: String, + @Suppress("ArrayInDataClass") @JsonProperty("spans") val spans: Array, // Ignore warning. This will not be used in hashmaps etc. + @Suppress("ArrayInDataClass") @JsonProperty("children") val children: Array, + @JsonProperty("rendered") val rendered: String? + ) { + val severity: DiagnosticSeverity + get() = when (level) { + "error" -> DiagnosticSeverity.Error + "warning" -> DiagnosticSeverity.Warning + "note" -> DiagnosticSeverity.Information + "help" -> DiagnosticSeverity.Hint + "failure-note" -> DiagnosticSeverity.Information + "error: internal compiler error" -> DiagnosticSeverity.Error + else -> DiagnosticSeverity.Warning // Should be impossible + } + } + private data class RustDiagnosticCode( + @JsonProperty("code") val code: String, + @JsonProperty("explanation") val explanation: String? + ) + private data class RustSpan( + @JsonProperty("file_name") val fileName: String, + @JsonProperty("byte_start") val byteStart: Int, + @JsonProperty("byte_end") val byteEnd: Int, + @JsonProperty("line_start") val lineStart: Int, + @JsonProperty("line_end") val lineEnd: Int, + @JsonProperty("column_start") val columnStart: Int, + @JsonProperty("column_end") val columnEnd: Int, + @JsonProperty("is_primary") val isPrimary: Boolean, + @JsonProperty("text") val text: Array, // Ignore warning. This will not be used in hashmaps etc. + @JsonProperty("label") val label: String?, + @JsonProperty("suggested_replacement") val suggestedReplacement: String?, + @JsonProperty("suggestion_applicability") val suggestionApplicability: String?, + @JsonProperty("expansion") val expansion: RustSpanExpansion? + ) { + val start: Position + get() = Position.fromOneBased(lineStart, columnStart) + val end: Position + get() = Position.fromOneBased(lineEnd, columnEnd) + } + private data class RustSpanExpansion( + @JsonProperty("span") val span: RustSpan, + @JsonProperty("macro_decl_name") val macroDeclName: String, + @JsonProperty("def_site_span") val defSiteSpan: RustSpan? + ) + private data class RustSpanText( + @JsonProperty("text") val text: String, + @JsonProperty("highlight_start") val highlightStart: Int, + @JsonProperty("highlight_end") val highlightEnd: Int + ) + + private var _metadata: RustMetadata? = null + + private fun getMetadata(): RustMetadata? { + val nullableCommand = LFCommand.get("cargo", listOf("metadata", "--format-version", "1"), fileConfig.srcGenPkgPath) + _metadata = _metadata ?: nullableCommand?.let { command -> + command.run({false}, true) + command.output.toString().lines().filter { it.startsWith("{") }.mapNotNull { + try { + mapper.readValue(it, RustMetadata::class.java) + } catch (e: JsonProcessingException) { + null + } + }.firstOrNull() + } + return _metadata + } + + override fun getPossibleStrategies(): Collection = listOf(object: ValidationStrategy { + override fun getCommand(generatedFile: Path?): LFCommand { + return LFCommand.get("cargo", listOf("clippy", "--message-format", "json"), fileConfig.srcGenPkgPath) + } + + override fun getErrorReportingStrategy() = DiagnosticReporting.Strategy { _, _, _ -> } + + override fun getOutputReportingStrategy() = DiagnosticReporting.Strategy { + validationOutput, errorReporter, map -> validationOutput.lines().forEach { messageLine -> + if (messageLine.isNotBlank() && mapper.readValue(messageLine, RustOutput::class.java).reason == COMPILER_MESSAGE_REASON) { + val message = mapper.readValue(messageLine, RustCompilerMessage::class.java).message + if (message.spans.isEmpty()) errorReporter.report(null, message.severity, message.message) + for (s: RustSpan in message.spans) { + val p: Path? = getMetadata()?.workspaceRoot?.resolve(s.fileName) + map[p]?.let { + for (lfSourcePath: Path in it.lfSourcePaths()) { + errorReporter.report( + lfSourcePath, + message.severity, + DiagnosticReporting.messageOf(message.message, p, s.start), + it.adjusted(lfSourcePath, s.start), + it.adjusted(lfSourcePath, s.end), + ) + } + } + } + } + } + } + + override fun getPriority(): Int = 0 + + override fun isFullBatch(): Boolean = true + }) + + override fun getBuildReportingStrategies(): Pair = Pair( + possibleStrategies.first().errorReportingStrategy, possibleStrategies.first().outputReportingStrategy + ) +} diff --git a/org.lflang/src/org/lflang/generator/ts/TSGenerator.kt b/org.lflang/src/org/lflang/generator/ts/TSGenerator.kt index e9cc01ac34..7c3669a909 100644 --- a/org.lflang/src/org/lflang/generator/ts/TSGenerator.kt +++ b/org.lflang/src/org/lflang/generator/ts/TSGenerator.kt @@ -50,11 +50,21 @@ import java.nio.file.Files import java.util.LinkedList import org.lflang.federated.serialization.SupportedSerializers import org.lflang.generator.canGenerate +import org.lflang.generator.CodeMap import org.lflang.generator.GeneratorBase import org.lflang.generator.GeneratorResult import org.lflang.generator.IntegratedBuilder +import org.lflang.generator.JavaGeneratorUtils import org.lflang.generator.LFGeneratorContext import org.lflang.generator.PrependOperator +import org.lflang.generator.SubContext +import java.nio.file.Path +import java.nio.file.StandardCopyOption +import kotlin.collections.HashMap + +private const val NO_NPM_MESSAGE = "The TypeScript target requires npm >= 6.14.4. " + + "For installation instructions, see: https://www.npmjs.com/get-npm. \n" + + "Auto-compiling can be disabled using the \"no-compile: true\" target property." /** * Generator for TypeScript target. @@ -79,7 +89,7 @@ class TSGenerator( * Names of the configuration files to check for and copy to the generated * source package root if they cannot be found in the source directory. */ - val CONFIG_FILES = arrayOf("package.json", "tsconfig.json", "babel.config.js") + val CONFIG_FILES = arrayOf("package.json", "tsconfig.json", "babel.config.js", ".eslintrc.json") /** * Files to be copied from the reactor-ts submodule into the generated @@ -89,6 +99,12 @@ class TSGenerator( "command-line-usage.d.ts", "component.ts", "federation.ts", "reaction.ts", "reactor.ts", "microtime.d.ts", "nanotimer.d.ts", "time.ts", "ulog.d.ts", "util.ts") + + /** + * The percent progress associated with having collected all JS/TS dependencies. + */ + private const val COLLECTED_DEPENDENCIES_PERCENT_PROGRESS + = (IntegratedBuilder.GENERATED_PERCENT_PROGRESS + IntegratedBuilder.COMPILED_PERCENT_PROGRESS) / 2 } init { @@ -115,14 +131,14 @@ class TSGenerator( * generation. * @param resource The resource containing the source code. * @param fsa The file system access (used to write the result). - * @param context FIXME: Undocumented argument. No idea what this is. + * @param context The context of this build. */ - // TODO(hokeun): Split this method into smaller methods. override fun doGenerate(resource: Resource, fsa: IFileSystemAccess2, context: LFGeneratorContext) { super.doGenerate(resource, fsa, context) if (!canGenerate(errorsOccurred(), mainDef, errorReporter, context)) return + if (!isOsCompatible()) return // FIXME: The following operation must be done after levels are assigned. // Removing these ports before that will cause incorrect levels to be assigned. @@ -131,123 +147,153 @@ class TSGenerator( // assigning levels. removeRemoteFederateConnectionPorts(null); - fileConfig.deleteDirectory(fileConfig.srcGenPath) + clean(context) + copyRuntime() + copyConfigFiles() + + val codeMaps = HashMap() + for (federate in federates) generateCode(fsa, federate, codeMaps) + // For small programs, everything up until this point is virtually instantaneous. This is the point where cancellation, + // progress reporting, and IDE responsiveness become real considerations. + if (targetConfig.noCompile) { + context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(null)) + } else { + context.reportProgress( + "Code generation complete. Collecting dependencies...", IntegratedBuilder.GENERATED_PERCENT_PROGRESS + ) + if (shouldCollectDependencies(context)) collectDependencies(resource, context) + if (errorsOccurred()) { + context.unsuccessfulFinish(); + return; + } + if (targetConfig.protoFiles.size != 0) { + protoc() + } else { + println("No .proto files have been imported. Skipping protocol buffer compilation.") + } + val parsingContext = SubContext(context, COLLECTED_DEPENDENCIES_PERCENT_PROGRESS, 100) + if ( + !context.cancelIndicator.isCanceled + && passesChecks(TSValidator(tsFileConfig, errorReporter, codeMaps), parsingContext) + ) { + if (context.mode == Mode.LSP_MEDIUM) { + context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(codeMaps)) + } else { + compile(parsingContext) + concludeCompilation(context, codeMaps) + } + } else { + context.unsuccessfulFinish() + } + } + } + + /** + * Clean up the src-gen directory as needed to prepare for code generation. + */ + private fun clean(context: LFGeneratorContext) { + // Dirty shortcut for integrated mode: Delete nothing, saving the node_modules directory to avoid re-running pnpm. + if (context.mode != Mode.LSP_MEDIUM) fileConfig.deleteDirectory(fileConfig.srcGenPath) + } + + /** + * Copy the TypeScript runtime so that it is accessible to the generated code. + */ + private fun copyRuntime() { for (runtimeFile in RUNTIME_FILES) { fileConfig.copyFileFromClassPath( "$LIB_PATH/reactor-ts/src/core/$runtimeFile", - tsFileConfig.tsCoreGenPath().resolve(runtimeFile).toString()) + tsFileConfig.tsCoreGenPath().resolve(runtimeFile).toString() + ) } + } - /** - * Check whether configuration files are present in the same directory - * as the source file. For those that are missing, install a default - * If the given filename is not present in the same directory as the source - * file, copy a default version of it from $LIB_PATH/. - */ + /** + * For each configuration file that is not present in the same directory + * as the source file, copy a default version from $LIB_PATH/. + */ + private fun copyConfigFiles() { for (configFile in CONFIG_FILES) { - val configFileDest = fileConfig.srcGenPath.resolve(configFile).toString() - val configFileInSrc = fileConfig.srcPath.resolve(configFile).toString() - if (fsa.isFile(configFileInSrc)) { - // TODO(hokeun): Check if this logic is still necessary. + val configFileDest = fileConfig.srcGenPath.resolve(configFile) + val configFileInSrc = fileConfig.srcPath.resolve(configFile) + if (configFileInSrc.toFile().exists()) { println("Copying '" + configFile + "' from " + fileConfig.srcPath) - fileConfig.copyFileFromClassPath(configFileInSrc, configFileDest) + Files.copy(configFileInSrc, configFileDest, StandardCopyOption.REPLACE_EXISTING) } else { println( "No '" + configFile + "' exists in " + fileConfig.srcPath + ". Using default configuration." ) - fileConfig.copyFileFromClassPath("$LIB_PATH/$configFile", configFileDest) + fileConfig.copyFileFromClassPath("$LIB_PATH/$configFile", configFileDest.toString()) } } + } - for (federate in federates) { - var tsFileName = fileConfig.name - // TODO(hokeun): Consider using FedFileConfig when enabling federated execution for TypeScript. - // For details, see https://github.com/icyphy/lingua-franca/pull/431#discussion_r676302102 - if (isFederated) { - tsFileName += '_' + federate.name - } + /** + * Generate the code corresponding to [federate], recording any resulting mappings in [codeMaps]. + */ + private fun generateCode(fsa: IFileSystemAccess2, federate: FederateInstance, codeMaps: MutableMap) { + var tsFileName = fileConfig.name + // TODO(hokeun): Consider using FedFileConfig when enabling federated execution for TypeScript. + // For details, see https://github.com/icyphy/lingua-franca/pull/431#discussion_r676302102 + if (isFederated) { + tsFileName += '_' + federate.name + } - val tsFilePath = tsFileConfig.tsSrcGenPath().resolve("$tsFileName.ts") + val tsFilePath = tsFileConfig.tsSrcGenPath().resolve("$tsFileName.ts") - val tsCode = StringBuilder() + val tsCode = StringBuilder() - val preambleGenerator = TSImportPreambleGenerator(fileConfig.srcFile, - targetConfig.protoFiles) - tsCode.append(preambleGenerator.generatePreamble()) + val preambleGenerator = TSImportPreambleGenerator(fileConfig.srcFile, + targetConfig.protoFiles) + tsCode.append(preambleGenerator.generatePreamble()) - val parameterGenerator = TSParameterPreambleGenerator(this, fileConfig, targetConfig, reactors) - val (mainParameters, parameterCode) = parameterGenerator.generateParameters() - tsCode.append(parameterCode) + val parameterGenerator = TSParameterPreambleGenerator(this, fileConfig, targetConfig, reactors) + val (mainParameters, parameterCode) = parameterGenerator.generateParameters() + tsCode.append(parameterCode) - val reactorGenerator = TSReactorGenerator(this, errorReporter) - for (reactor in reactors) { - tsCode.append(reactorGenerator.generateReactor(reactor, federate)) - } - tsCode.append(reactorGenerator.generateReactorInstanceAndStart(this.mainDef, mainParameters)) - fsa.generateFile(fileConfig.srcGenBasePath.relativize(tsFilePath).toString(), - tsCode.toString()) - - if (targetConfig.dockerOptions != null && isFederated) { - println("WARNING: Federated Docker file generation is not supported on the Typescript target. No docker file is generated."); - } else if (targetConfig.dockerOptions != null) { - val dockerFilePath = fileConfig.srcGenPath.resolve("$tsFileName.Dockerfile"); - val dockerComposeFile = fileConfig.srcGenPath.resolve("docker-compose.yml"); - val dockerGenerator = TSDockerGenerator(tsFileName) - println("docker file written to $dockerFilePath") - fsa.generateFile(fileConfig.srcGenBasePath.relativize(dockerFilePath).toString(), dockerGenerator.generateDockerFileContent()) - fsa.generateFile(fileConfig.srcGenBasePath.relativize(dockerComposeFile).toString(), dockerGenerator.generateDockerComposeFileContent()) - } + val reactorGenerator = TSReactorGenerator(this, errorReporter) + for (reactor in reactors) { + tsCode.append(reactorGenerator.generateReactor(reactor, federate)) + } + tsCode.append(reactorGenerator.generateReactorInstanceAndStart(this.mainDef, mainParameters)) + val codeMap = CodeMap.fromGeneratedCode(tsCode.toString()) + codeMaps[tsFilePath] = codeMap + fsa.generateFile(fileConfig.srcGenBasePath.relativize(tsFilePath).toString(), codeMap.generatedCode) + + if (targetConfig.dockerOptions != null && isFederated) { + println("WARNING: Federated Docker file generation is not supported on the Typescript target. No docker file is generated.") + } else if (targetConfig.dockerOptions != null) { + val dockerFilePath = fileConfig.srcGenPath.resolve("$tsFileName.Dockerfile") + val dockerComposeFile = fileConfig.srcGenPath.resolve("docker-compose.yml") + val dockerGenerator = TSDockerGenerator(tsFileName) + println("docker file written to $dockerFilePath") + fsa.generateFile(fileConfig.srcGenBasePath.relativize(dockerFilePath).toString(), dockerGenerator.generateDockerFileContent()) + fsa.generateFile(fileConfig.srcGenBasePath.relativize(dockerComposeFile).toString(), dockerGenerator.generateDockerComposeFileContent()) } - // The following check is omitted for Mode.LSP_FAST because this code is unreachable in LSP_FAST mode. - if (!targetConfig.noCompile && context.mode != Mode.LSP_MEDIUM) compile(resource, context) - else context.finish(GeneratorResult.GENERATED_NO_EXECUTABLE.apply(null)) } - private fun compile(resource: Resource, context: LFGeneratorContext) { - if (!context.cancelIndicator.isCanceled) { - context.reportProgress( - "Code generation complete. Collecting dependencies...", - IntegratedBuilder.GENERATED_PERCENT_PROGRESS - ) - collectDependencies(resource, context) - } + private fun compile(parsingContext: LFGeneratorContext) { refreshProject() - if (!context.cancelIndicator.isCanceled && targetConfig.protoFiles.size != 0) { - if (context.cancelIndicator.isCanceled) return - context.reportProgress( - "Compiling protocol buffers...", - IntegratedBuilder.GENERATED_PERCENT_PROGRESS * 2 / 3 - + IntegratedBuilder.COMPILED_PERCENT_PROGRESS * 1 / 3 - ) - protoc() - } else { - println("No .proto files have been imported. Skipping protocol buffer compilation.") - } - - if (!context.cancelIndicator.isCanceled) { - context.reportProgress( - "Transpiling to JavaScript...", - IntegratedBuilder.GENERATED_PERCENT_PROGRESS * 1 / 3 - + IntegratedBuilder.COMPILED_PERCENT_PROGRESS * 2 / 3 - ) - transpile(context.cancelIndicator) - } + if (parsingContext.cancelIndicator.isCanceled) return + parsingContext.reportProgress("Transpiling to JavaScript...", 70) + transpile(parsingContext.cancelIndicator) - if (!context.cancelIndicator.isCanceled && isFederated) { - context.reportProgress( - "Generating federation infrastructure...", - IntegratedBuilder.GENERATED_PERCENT_PROGRESS * 1 / 6 - + IntegratedBuilder.COMPILED_PERCENT_PROGRESS * 5 / 6 - ) + if (parsingContext.cancelIndicator.isCanceled) return + if (isFederated) { + parsingContext.reportProgress("Generating federation infrastructure...", 90) generateFederationInfrastructure() } - - concludeCompilation(context) } + /** + * Return whether it is advisable to install dependencies. + */ + private fun shouldCollectDependencies(context: LFGeneratorContext): Boolean + = context.mode != Mode.LSP_MEDIUM || !fileConfig.srcGenPkgPath.resolve("node_modules").toFile().exists() + /** * Collect the dependencies in package.json and their * transitive dependencies. @@ -281,10 +327,7 @@ class TSGenerator( val npmInstall = commandFactory.createCommand("npm", listOf("install"), fileConfig.srcGenPkgPath) if (npmInstall == null) { - errorReporter.reportError( - "The TypeScript target requires npm >= 6.14.4. " + - "For installation instructions, see: https://www.npmjs.com/get-npm. \n" + - "Auto-compiling can be disabled using the \"no-compile: true\" target property.") + errorReporter.reportError(NO_NPM_MESSAGE) return } @@ -341,43 +384,34 @@ class TSGenerator( } /** - * Transpiles TypeScript to JavaScript. + * Run checks on the generated TypeScript. + * @return whether the checks pass. + */ + private fun passesChecks(validator: TSValidator, parsingContext: LFGeneratorContext): Boolean { + parsingContext.reportProgress("Linting generated code...", 0) + validator.doLint(parsingContext.cancelIndicator) + if (errorsOccurred()) return false + parsingContext.reportProgress("Validating generated code...", 25) + validator.doValidate(parsingContext.cancelIndicator) + return !errorsOccurred() + } + + /** + * Transpile TypeScript to JavaScript. */ private fun transpile(cancelIndicator: CancelIndicator) { - val errorMessage = "The TypeScript target requires npm >= 6.14.1 to compile the generated code. " + - "Auto-compiling can be disabled using the \"no-compile: true\" target property." - - // Invoke the compiler on the generated code. - println("Type Checking") - val tsc = commandFactory.createCommand("npm", listOf("run", "check-types"), fileConfig.srcGenPkgPath) - if (tsc == null) { - errorReporter.reportError(errorMessage); + println("Compiling") + val babel = commandFactory.createCommand("npm", listOf("run", "build"), fileConfig.srcGenPkgPath) + + if (babel == null) { + errorReporter.reportError(NO_NPM_MESSAGE) return } - if (tsc.run(cancelIndicator) == 0) { - // Babel will compile TypeScript to JS even if there are type errors - // so only run compilation if tsc found no problems. - //val babelPath = codeGenConfig.outPath + File.separator + "node_modules" + File.separator + ".bin" + File.separator + "babel" - // Working command $./node_modules/.bin/babel src-gen --out-dir js --extensions '.ts,.tsx' - println("Compiling") - val babel = commandFactory.createCommand("npm", listOf("run", "build"), fileConfig.srcGenPkgPath) - - if (babel == null) { - errorReporter.reportError(errorMessage); - return - } - - if (babel.run(cancelIndicator) == 0) { - println("SUCCESS (compiling generated TypeScript code)") - } else { - errorReporter.reportError("Compiler failed.") - } + if (babel.run(cancelIndicator) == 0) { + println("SUCCESS (compiling generated TypeScript code)") } else { - val errors: String = tsc.output.toString().lines().filter { it.contains("error") }.joinToString("\n") - errorReporter.reportError( - "Type checking failed" + if (errors.isBlank()) "." else " with the following errors:\n$errors" - ) + errorReporter.reportError("Compiler failed.") } } @@ -412,15 +446,29 @@ class TSGenerator( * Inform the context of the results of a compilation. * @param context The context of the compilation. */ - private fun concludeCompilation(context: LFGeneratorContext) { + private fun concludeCompilation(context: LFGeneratorContext, codeMaps: Map) { if (errorReporter.errorsOccurred) { context.unsuccessfulFinish() } else { - context.finish( - GeneratorResult.Status.COMPILED, fileConfig.name + ".js", - fileConfig.srcGenPkgPath.resolve("dist"), fileConfig, null, "node" + if (isFederated) { + context.finish(GeneratorResult.Status.COMPILED, fileConfig.name, fileConfig, codeMaps) + } else { + context.finish( + GeneratorResult.Status.COMPILED, fileConfig.name + ".js", + fileConfig.srcGenPkgPath.resolve("dist"), fileConfig, codeMaps, "node" + ) + } + } + } + + private fun isOsCompatible(): Boolean { + if (isFederated && JavaGeneratorUtils.isHostWindows()) { + errorReporter.reportError( + "Federated LF programs with a TypeScript target are currently not supported on Windows. Exiting code generation." ) + return false } + return true } /** diff --git a/org.lflang/src/org/lflang/generator/ts/TSValidator.kt b/org.lflang/src/org/lflang/generator/ts/TSValidator.kt new file mode 100644 index 0000000000..e217c9d975 --- /dev/null +++ b/org.lflang/src/org/lflang/generator/ts/TSValidator.kt @@ -0,0 +1,160 @@ +package org.lflang.generator.ts + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectMapper +import org.eclipse.lsp4j.DiagnosticSeverity +import org.eclipse.xtext.util.CancelIndicator +import org.lflang.ErrorReporter +import org.lflang.generator.CodeMap +import org.lflang.generator.DiagnosticReporting +import org.lflang.generator.HumanReadableReportingStrategy +import org.lflang.generator.Position +import org.lflang.generator.ValidationStrategy +import org.lflang.generator.Validator +import org.lflang.util.LFCommand +import java.nio.file.Path +import java.util.regex.Pattern + +private val TSC_OUTPUT_LINE: Pattern = Pattern.compile( + "(?[^:]*):(?\\d+):(?\\d+) - (?\\w+).*: (?.*)" +) +private val TSC_LABEL: Pattern = Pattern.compile("((?<=\\s))(~+)") + +/** + * A validator for generated TypeScript. + * + * @author Peter Donovan + */ +@Suppress("ArrayInDataClass") // Data classes here must not be used in data structures such as hashmaps. +class TSValidator( + private val fileConfig: TSFileConfig, + errorReporter: ErrorReporter, + codeMaps: Map +): Validator(errorReporter, codeMaps) { + + private class TSLinter( + private val fileConfig: TSFileConfig, + errorReporter: ErrorReporter, + codeMaps: Map + ): Validator(errorReporter, codeMaps) { + companion object { + private val mapper = ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + } + + // See https://eslint.org/docs/user-guide/formatters/#json for the best documentation available on + // the following data classes. + private data class ESLintOutput( + @JsonProperty("filePath") val filePath: String, + @JsonProperty("messages") val messages: Array, + @JsonProperty("errorCount") val errorCount: Int, + @JsonProperty("fatalErrorCount") val fatalErrorCount: Int, + @JsonProperty("warningCount") val warningCount: Int, + @JsonProperty("fixableErrorCount") val fixableErrorCount: Int, + @JsonProperty("fixableWarningCount") val fixableWarningCount: Int, + @JsonProperty("source") val source: String + ) + private data class ESLintMessage( + @JsonProperty("ruleId") val ruleId: String?, + @JsonProperty("severity") val _severity: Int, + @JsonProperty("message") val message: String, + @JsonProperty("line") val line: Int, + @JsonProperty("column") val column: Int, + @JsonProperty("nodeType") val nodeType: String?, + @JsonProperty("messageId") val messageId: String?, + @JsonProperty("endLine") val endLine: Int, + @JsonProperty("endColumn") val endColumn: Int, + @JsonProperty("fix") val fix: ESLintFix? + ) { + val start: Position = Position.fromOneBased(line, column) + val end: Position = if (endLine >= line) Position.fromOneBased(endLine, endColumn) else start.plus(" ") + val severity: DiagnosticSeverity = when (_severity) { + 0 -> DiagnosticSeverity.Information + 1 -> DiagnosticSeverity.Warning + 2 -> DiagnosticSeverity.Error + else -> DiagnosticSeverity.Warning // This should never happen + } + } + private data class ESLintFix( + @JsonProperty("range") val range: Array, + @JsonProperty("text") val text: String + ) + + override fun getPossibleStrategies(): Collection = listOf(object: ValidationStrategy { + override fun getCommand(generatedFile: Path?): LFCommand? { + return generatedFile?.let { + LFCommand.get( + "npx", + listOf("eslint", "--format", "json", fileConfig.srcGenPkgPath.relativize(it).toString()), + fileConfig.srcGenPkgPath + ) + } + } + + override fun getErrorReportingStrategy() = DiagnosticReporting.Strategy { _, _, _ -> } + + override fun getOutputReportingStrategy() = DiagnosticReporting.Strategy { + validationOutput, errorReporter, map -> validationOutput.lines().filter { it.isNotBlank() }.forEach { + line: String -> mapper.readValue(line, Array::class.java).forEach { + output: ESLintOutput -> output.messages.forEach { + message: ESLintMessage -> + val genPath: Path = fileConfig.srcGenPkgPath.resolve(output.filePath) + map[genPath]?.let { + codeMap -> + codeMap.lfSourcePaths().forEach { + val lfStart = codeMap.adjusted(it, message.start) + val lfEnd = codeMap.adjusted(it, message.end) + if (!lfStart.equals(Position.ORIGIN)) { // Ignore linting errors in non-user-supplied code. + errorReporter.report( + it, + message.severity, + DiagnosticReporting.messageOf(message.message, genPath, message.start), + lfStart, + if (lfEnd > lfStart) lfEnd else lfStart.plus(" "), + ) + } + } + } + } + } + } + } + + override fun isFullBatch(): Boolean = false // ESLint permits glob patterns. We could make this full-batch if desired. + + override fun getPriority(): Int = 0 + + }) + override fun getBuildReportingStrategies(): Pair + = Pair(DiagnosticReporting.Strategy { _, _, _ -> }, DiagnosticReporting.Strategy { _, _, _ -> }) // Not applicable + } + + override fun getPossibleStrategies(): Collection + = listOf(object: ValidationStrategy { + override fun getCommand(generatedFile: Path?): LFCommand? { // FIXME: Add "--incremental" argument if we update to TypeScript 4 + return LFCommand.get("npx", listOf("tsc", "--pretty", "--noEmit"), fileConfig.srcGenPkgPath) + } + + override fun getErrorReportingStrategy() = DiagnosticReporting.Strategy { _, _, _ -> } + + override fun getOutputReportingStrategy() = HumanReadableReportingStrategy( + TSC_OUTPUT_LINE, TSC_LABEL, fileConfig.srcGenPkgPath + ) + + override fun isFullBatch(): Boolean = true + + override fun getPriority(): Int = 0 + + }) + + override fun getBuildReportingStrategies(): Pair + = Pair(possibleStrategies.first().errorReportingStrategy, possibleStrategies.first().outputReportingStrategy) + + /** + * Run a relatively fast linter on the generated code. + * @param cancelIndicator The indicator of whether this build process is cancelled. + */ + fun doLint(cancelIndicator: CancelIndicator) { + TSLinter(fileConfig, errorReporter, codeMaps).doValidate(cancelIndicator) + } +} diff --git a/org.lflang/src/org/lflang/util/LFCommand.java b/org.lflang/src/org/lflang/util/LFCommand.java index d0a6bfb634..eb198c9854 100644 --- a/org.lflang/src/org/lflang/util/LFCommand.java +++ b/org.lflang/src/org/lflang/util/LFCommand.java @@ -71,7 +71,7 @@ public class LFCommand { /** - * Constructor + * Construct an LFCommand that executes the command carried by {@code pb}. */ protected LFCommand(ProcessBuilder pb) { processBuilder = pb; } @@ -100,11 +100,10 @@ public class LFCommand { /** - * Collects as much output as possible from in - * without blocking, prints it to print - * , and stores it in store. + * Collect as much output as possible from {@code in} without blocking, print it to + * {@code print} if not {@code quiet}, and store it in {@code store}. */ - private void collectOutput(InputStream in, ByteArrayOutputStream store, PrintStream print) { + private void collectOutput(InputStream in, ByteArrayOutputStream store, PrintStream print, boolean quiet) { byte[] buffer = new byte[64]; int len; do { @@ -119,7 +118,7 @@ private void collectOutput(InputStream in, ByteArrayOutputStream store, PrintStr len = in.read(buffer, 0, Math.min(in.available(), buffer.length)); if (len > 0) { store.write(buffer, 0, len); - print.write(buffer, 0, len); + if (!quiet) print.write(buffer, 0, len); } } catch (IOException e) { e.printStackTrace(); @@ -131,21 +130,22 @@ private void collectOutput(InputStream in, ByteArrayOutputStream store, PrintStr } /** - * Handles the user cancellation if one exists, and - * handles any output from process + * Handle user cancellation if necessary, and handle any output from {@code process} * otherwise. - * @param process a Process + * @param process a {@code Process} * @param cancelIndicator a flag indicating whether a - * cancellation of process - * is requested + * cancellation of {@code process} + * is requested + * @param quiet Whether output from {@code pb} should be silenced (i.e., not forwarded + * directly to stderr and stdout). */ - private void poll(Process process, CancelIndicator cancelIndicator) { + private void poll(Process process, CancelIndicator cancelIndicator, boolean quiet) { if (cancelIndicator != null && cancelIndicator.isCanceled()) { process.descendants().forEach(ProcessHandle::destroyForcibly); process.destroyForcibly(); } else { - collectOutput(process.getInputStream(), output, System.out); - collectOutput(process.getErrorStream(), errors, System.err); + collectOutput(process.getInputStream(), output, System.out, quiet); + collectOutput(process.getErrorStream(), errors, System.err, quiet); } } @@ -167,10 +167,14 @@ private void poll(Process process, CancelIndicator cancelIndicator) { * point are still collected. *

* + * @param cancelIndicator The indicator of whether the underlying process + * should be terminated. + * @param quiet Whether output from {@code pb} should be silenced (i.e., not forwarded + * directly to stderr and stdout). * @return the process' return code * @author {Christian Menard poll(process, cancelIndicator), + () -> poll(process, cancelIndicator, quiet), 0, PERIOD_MILLISECONDS, TimeUnit.MILLISECONDS ); @@ -191,7 +195,7 @@ public int run(CancelIndicator cancelIndicator) { poller.shutdown(); poller.awaitTermination(READ_TIMEOUT_MILLISECONDS, TimeUnit.MILLISECONDS); // Finish collecting any remaining data - poll(process, cancelIndicator); + poll(process, cancelIndicator, quiet); return returnCode; } catch (InterruptedException e) { e.printStackTrace(); @@ -199,13 +203,23 @@ public int run(CancelIndicator cancelIndicator) { } } + /** + * Execute the command while forwarding output and error streams. + * @param cancelIndicator The indicator of whether the underlying process + * should be terminated. + * @return the process' return code + */ + public int run(CancelIndicator cancelIndicator) { + return run(cancelIndicator, false); + } + /** * Execute the command while forwarding output and error * streams. Do not allow user cancellation. * @return the process' return code */ public int run() { - return run(null); + return run(null, false); } diff --git a/test/Python/src/HelloWorld.lf b/test/Python/src/HelloWorld.lf index e7627a0fd3..7b260fc474 100644 --- a/test/Python/src/HelloWorld.lf +++ b/test/Python/src/HelloWorld.lf @@ -10,8 +10,8 @@ reactor HelloWorld2 { reaction(shutdown) {= print("Shutdown invoked.") if not self.success: - self.sys.stderr.write("ERROR: startup reaction not executed.\n") - self.sys.exit(1) + sys.stderr.write("ERROR: startup reaction not executed.\n") + sys.exit(1) =} } main reactor HelloWorld { diff --git a/test/Python/src/PingPong.lf b/test/Python/src/PingPong.lf index d54ab2ad72..d4c8077620 100644 --- a/test/Python/src/PingPong.lf +++ b/test/Python/src/PingPong.lf @@ -50,13 +50,13 @@ reactor Pong(expected(10)) { =} reaction(shutdown) {= if self.count != self.expected: - self.sys.stderr.write( + sys.stderr.write( "ERROR: Pong expected to receive {} inputs, but it received {}.\n".format( self.expected, self.count ) ) - self.sys.exit(1) + sys.exit(1) print("Success.") =} } diff --git a/test/Python/src/multiport/MultiportToBankHierarchy.lf b/test/Python/src/multiport/MultiportToBankHierarchy.lf index d0d49656d0..fe473c39e0 100644 --- a/test/Python/src/multiport/MultiportToBankHierarchy.lf +++ b/test/Python/src/multiport/MultiportToBankHierarchy.lf @@ -19,7 +19,7 @@ reactor Destination( =} reaction(shutdown) {= if self.received is not True: - fprintf(stderr, "ERROR: Destination {:d} received no input!\n".format(self.bank_index)) + sys.stderr.write("ERROR: Destination {:d} received no input!\n".format(self.bank_index)) exit(1) print("Success.") =} diff --git a/test/TypeScript/src/ActionDelay.lf b/test/TypeScript/src/ActionDelay.lf index deeb178822..44b69587af 100644 --- a/test/TypeScript/src/ActionDelay.lf +++ b/test/TypeScript/src/ActionDelay.lf @@ -26,11 +26,11 @@ reactor Source { reactor Sink { input x:number; reaction(x) {= - var elapsed_logical = util.getElapsedLogicalTime(); - var logical = util.getCurrentLogicalTime(); - var physical = util.getCurrentPhysicalTime(); + const elapsed_logical = util.getElapsedLogicalTime(); + const logical = util.getCurrentLogicalTime(); + const physical = util.getCurrentPhysicalTime(); console.log("Logical, physical, and elapsed logical: " + logical + physical + elapsed_logical); - var oneHundredMsec = TimeValue.msec(100); + const oneHundredMsec = TimeValue.msec(100); if (!elapsed_logical.isEqualTo(oneHundredMsec)) { util.requestErrorStop("Expected " + oneHundredMsec + " but got " + elapsed_logical); } else { @@ -45,4 +45,4 @@ main reactor ActionDelay { source.out -> g.y_in; g.y_out -> sink.x; -} \ No newline at end of file +} diff --git a/test/TypeScript/src/CompositionAfter.lf b/test/TypeScript/src/CompositionAfter.lf index a2569c10ab..e774d9d69b 100644 --- a/test/TypeScript/src/CompositionAfter.lf +++ b/test/TypeScript/src/CompositionAfter.lf @@ -1,32 +1,32 @@ // This test connects a simple counting source // that checks against its own count. target TypeScript { - fast: true, - timeout: 10 sec + fast: true, + timeout: 10 sec }; reactor Source(period:time(2 sec)) { - output y:number; - timer t(1 sec, period); - state count:number(0); - reaction(t) -> y {= - count++; - y = count; - =} + output y:number; + timer t(1 sec, period); + state count:number(0); + reaction(t) -> y {= + count++; + y = count; + =} } reactor Test { - input x:number; - state count:number(0); - reaction(x) {= - count++; - console.log("Received " + x); - if (x != count) { - util.requestErrorStop("FAILURE: Expected " + count); - } - =} + input x:number; + state count:number(0); + reaction(x) {= + count++; + console.log("Received " + x); + if (x != count) { + util.requestErrorStop("FAILURE: Expected " + count); + } + =} } main reactor (delay:time(5 sec)) { - s = new Source(); - d = new Test(); - s.y -> d.x after delay; -} \ No newline at end of file + s = new Source(); + d = new Test(); + s.y -> d.x after delay; +} diff --git a/test/TypeScript/src/DoubleReaction.lf b/test/TypeScript/src/DoubleReaction.lf index 2c042684ca..970001a05b 100644 --- a/test/TypeScript/src/DoubleReaction.lf +++ b/test/TypeScript/src/DoubleReaction.lf @@ -6,38 +6,38 @@ target TypeScript { fast: true }; reactor Clock(offset:time(0), period:time(1 sec)) { - output y:number; - timer t(offset, period); - state count:number(0); - reaction(t) -> y {= - count++; - y = count; - =} + output y:number; + timer t(offset, period); + state count:number(0); + reaction(t) -> y {= + count++; + y = count; + =} } reactor Destination { - input x:number; - input w:number; - state s:number(2); - reaction(x, w) {= - let sum = 0; - if (x) { - sum += x; - } - if (w) { - sum += w; - } - console.log("Sum of inputs is: " + sum); - if (sum != s) { - util.requestErrorStop("FAILURE: Expected sum to be " + s + ", but it was " + sum) - } - s += 2; - =} + input x:number; + input w:number; + state s:number(2); + reaction(x, w) {= + let sum = 0; + if (x) { + sum += x; + } + if (w) { + sum += w; + } + console.log("Sum of inputs is: " + sum); + if (sum != s) { + util.requestErrorStop("FAILURE: Expected sum to be " + s + ", but it was " + sum) + } + s += 2; + =} } main reactor DoubleReaction { - c1 = new Clock(); - c2 = new Clock(); - d = new Destination(); - c1.y -> d.x; - c2.y -> d.w; -} \ No newline at end of file + c1 = new Clock(); + c2 = new Clock(); + d = new Destination(); + c1.y -> d.x; + c2.y -> d.w; +} diff --git a/test/TypeScript/src/FloatLiteral.lf b/test/TypeScript/src/FloatLiteral.lf index b05d795d23..161529a105 100644 --- a/test/TypeScript/src/FloatLiteral.lf +++ b/test/TypeScript/src/FloatLiteral.lf @@ -7,7 +7,7 @@ main reactor { state minus_epsilon:number(-.01e0) state expected:number(.964853323188E5) reaction(startup) {= - var F: number = - N * charge; + const F: number = - N * charge; if (Math.abs(F - expected) < Math.abs(minus_epsilon)) { console.log("The Faraday constant is roughly " + F + "."); } else { diff --git a/test/TypeScript/src/Hello.lf b/test/TypeScript/src/Hello.lf index 69fffb401c..2107303cf0 100644 --- a/test/TypeScript/src/Hello.lf +++ b/test/TypeScript/src/Hello.lf @@ -8,36 +8,36 @@ target TypeScript { fast: true }; reactor Reschedule(period:time(2 secs), message:string("Hello TypeScript")) { - state count:number(0); - state previous_time:time(0); - timer t(1 secs, period); - logical action a; - reaction(t) -> a {= - console.log(message); - actions.a.schedule(TimeValue.msec(200), null); - // Print the current time. - previous_time = util.getCurrentLogicalTime(); - console.log("Current time is " + previous_time); - =} - reaction(a) {= - count++; - console.log("***** action " + count + " at time " + util.getCurrentLogicalTime()); - // Check the a_has_value variable. - if (a) { - util.requestErrorStop("FAILURE: Expected a to be null (not present), but it was non-null."); - } - let currentTime = util.getCurrentLogicalTime(); - if (! currentTime.subtract(previous_time).isEqualTo(TimeValue.msec(200))) { - util.requestErrorStop("FAILURE: Expected 200ms of logical time to elapse but got " + - currentTime.subtract(previous_time)); - } - =} + state count:number(0); + state previous_time:time(0); + timer t(1 secs, period); + logical action a; + reaction(t) -> a {= + console.log(message); + actions.a.schedule(TimeValue.msec(200), null); + // Print the current time. + previous_time = util.getCurrentLogicalTime(); + console.log("Current time is " + previous_time); + =} + reaction(a) {= + count++; + console.log("***** action " + count + " at time " + util.getCurrentLogicalTime()); + // Check the a_has_value variable. + if (a) { + util.requestErrorStop("FAILURE: Expected a to be null (not present), but it was non-null."); + } + let currentTime = util.getCurrentLogicalTime(); + if (! currentTime.subtract(previous_time).isEqualTo(TimeValue.msec(200))) { + util.requestErrorStop("FAILURE: Expected 200ms of logical time to elapse but got " + + currentTime.subtract(previous_time)); + } + =} } reactor Inside(period:time(1 sec), message:string("Composite default message.")) { - third_instance = new Reschedule(period = period, message = message); + third_instance = new Reschedule(period = period, message = message); } main reactor Hello { - first_instance = new Reschedule(period = 4 sec, message = "Hello from first_instance."); - second_instance = new Reschedule(message = "Hello from second_instance."); - composite_instance = new Inside(message = "Hello from composite_instance."); + first_instance = new Reschedule(period = 4 sec, message = "Hello from first_instance."); + second_instance = new Reschedule(message = "Hello from second_instance."); + composite_instance = new Inside(message = "Hello from composite_instance."); } \ No newline at end of file diff --git a/test/TypeScript/src/concurrent/AsyncCallback.lf b/test/TypeScript/src/concurrent/AsyncCallback.lf index 961c5d5130..485c786efe 100644 --- a/test/TypeScript/src/concurrent/AsyncCallback.lf +++ b/test/TypeScript/src/concurrent/AsyncCallback.lf @@ -1,49 +1,48 @@ // Test asynchronous callbacks that trigger a physical action. target TypeScript { - timeout: 2 sec, - keepalive: true // Not really needed here because there is a timer. + timeout: 2 sec, + keepalive: true // Not really needed here because there is a timer. }; main reactor AsyncCallback { - - preamble {= - - function callback(a : Sched) { - // Schedule twice. If the action is not physical, these should + + preamble {= + function callback(a : Sched) { + // Schedule twice. If the action is not physical, these should // get consolidated into a single action triggering. If it is, // then they cause two separate triggerings with close but not // equal time stamps. The minimum time between these is determined // by the argument in the physical action definition. - a.schedule(0, null); - a.schedule(0, null); - } - =} - timer t(0, 200 msec); - state expected_time:time(100 msec); - state toggle:boolean(false); - + a.schedule(0, null); + a.schedule(0, null); + } + =} + timer t(0, 200 msec); + state expected_time:time(100 msec); + state toggle:boolean(false); + physical action a(100 msec):number; state i:number(0); - - reaction(t) -> a {= - // set a timeout for the callback - setTimeout(callback, 1000, actions.a); - =} - - reaction(a) {= - let elapsed_time = util.getElapsedLogicalTime(); + + reaction(t) -> a {= + // set a timeout for the callback + setTimeout(callback, 1000, actions.a); + =} + + reaction(a) {= + let elapsed_time = util.getElapsedLogicalTime(); console.log("Asynchronous callback " + i + ": Assigned logical time greater than start time by " + elapsed_time + " nsec." ); - if (elapsed_time.isEarlierThan(expected_time)) { - util.requestErrorStop("ERROR: Expected logical time to be larger than " + expected_time + ".") - } + if (elapsed_time.isEarlierThan(expected_time)) { + util.requestErrorStop("ERROR: Expected logical time to be larger than " + expected_time + ".") + } if (toggle) { toggle = false; expected_time.add(TimeValue.msec(200)); } else { toggle = true; } - =} + =} }