Skip to content

Commit

Permalink
reimplement external intrinsic idea to make it embeded into final tra…
Browse files Browse the repository at this point in the history 
…nspiled source code
  • Loading branch information
Viktor Yastrebov committed Jun 17, 2024
1 parent 96b0912 commit adece8c
Show file tree
Hide file tree
Showing 20 changed files with 103 additions and 207 deletions.
102 changes: 34 additions & 68 deletions lib/core/intrinsics/external_intrinsics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <clang/Frontend/CompilerInstance.h>
#include "core/transpiler_session/session_stage.h"
#include "core/transpiler_session/transpiler_session.h"
#include "oklt/util/io_helper.h"
#include "util/string_utils.hpp"

#include <algorithm>
Expand Down Expand Up @@ -73,10 +74,9 @@ std::optional<fs::path> getExternalInstrincisInclude(TranspilerSession& session,
return std::nullopt;
}

tl::expected<std::unique_ptr<llvm::MemoryBuffer>, std::string> getExternalIntrinsicSource(
TargetBackend backend,
const fs::path& intrinsicPath,
clang::SourceManager& sm) {
tl::expected<std::string, std::string> getExternalIntrinsicSource(TargetBackend backend,
const fs::path& intrinsicPath,
clang::SourceManager& sm) {
auto implPathResult = getIntrincisImplSourcePath(backend, intrinsicPath);
if (!implPathResult) {
return tl::make_unexpected(implPathResult.error());
Expand All @@ -93,119 +93,85 @@ tl::expected<std::unique_ptr<llvm::MemoryBuffer>, std::string> getExternalIntrin
}

auto it = std::find_if(files.cbegin(), files.cend(), [](const fs::path& p) -> bool {
return p.extension().string() == std::string(".h");
return p.extension().string() == std::string(".cpp");
});

if (it == files.cend()) {
std::string error = "Can't' find implementation file with path: " + sourceFolder.string();
return tl::make_unexpected(error);
}

auto& fm = sm.getFileManager();
auto backendFilePath = it->string();
auto maybeReplacedFile = fm.getFile(backendFilePath);
if (!maybeReplacedFile) {
std::string error = "Can't open file: " + backendFilePath;
auto contentResult = util::readFileAsStr(*it);
if (!contentResult) {
std::string error = "Can't get memory buffer for: " + it->string();
return tl::make_unexpected(error);
}
auto maybeBuffer = fm.getBufferForFile(maybeReplacedFile.get());
if (!maybeBuffer) {
std::string error = "Can't get memory buffer for: " + backendFilePath;
return tl::make_unexpected(error);
}
return std::move(maybeBuffer.get());
return contentResult.value();
}

bool overrideExternalIntrinsic(TranspilerSession& session,
bool overrideExternalIntrinsic(SessionStage& stage,
HeaderDepsInfo& deps,
const std::string& includedFileName,
clang::OptionalFileEntryRef includedFile,
clang::SourceManager& sourceManager) {
clang::OptionalFileEntryRef includedFile) {
auto& session = stage.getSession();
auto& sourceManager = stage.getCompiler().getSourceManager();
const auto& userIntrinsics = session.getInput().userIntrinsics;
if (!userIntrinsics.empty()) {
auto maybeIntrinsicPath = getExternalInstrincisInclude(session, includedFileName);
if (!maybeIntrinsicPath) {
return false;
}
auto intrinsicPath = maybeIntrinsicPath.value();
auto infoResult =
getExternalIntrinsicSource(session.getInput().backend, intrinsicPath, sourceManager);
if (!infoResult) {
session.pushError(std::error_code(), infoResult.error());
auto intrinsicResult =
getExternalIntrinsicSource(stage.getBackend(), intrinsicPath, sourceManager);
if (!intrinsicResult) {
session.pushError(std::error_code(), intrinsicResult.error());
return false;
}
auto buffer = std::move(infoResult.value());
auto& fm = sourceManager.getFileManager();
deps.externalIntrinsicsSources[includedFileName] = std::move(intrinsicResult.value());

auto emptyExternalIntrinsic = MemoryBuffer::getMemBuffer("");
if (includedFile) {
auto fileRef = includedFile;
const auto& fileEntry = fileRef->getFileEntry();
sourceManager.overrideFileContents(&fileEntry, std::move(buffer));
sourceManager.overrideFileContents(&fileEntry, std::move(emptyExternalIntrinsic));
} else {
// INFO: case when the file can be found by relative path
// it happens when the include path is relative to WORKING DIR path
auto& fm = sourceManager.getFileManager();
auto maybeFileRef = fm.getFileRef(includedFileName);
if (maybeFileRef) {
auto foundFileRef = maybeFileRef.get();
sourceManager.overrideFileContents(foundFileRef, std::move(buffer));
sourceManager.overrideFileContents(foundFileRef, std::move(emptyExternalIntrinsic));
}
}
return true;
}
return false;
}

void nullyLauncherExternalIntrinsics(TransformedFiles& inputs, SessionStage& stage) {
if (stage.getBackend() != TargetBackend::_LAUNCHER) {
return;
}

auto& session = stage.getSession();
const auto& intrinsics = session.getInput().userIntrinsics;
if (intrinsics.empty()) {
void updateExternalIntrinsicMap(SessionStage& stage, HeaderDepsInfo& deps) {
if (deps.externalIntrinsicsSources.empty()) {
return;
}

for (auto& mappedFile : inputs.fileMap) {
auto fileName = normalizedFileName(mappedFile.first);
for (const auto& includePath : intrinsics) {
auto folderPrefix = includePath.filename().string();
if (util::startsWith(fileName, folderPrefix)) {
mappedFile.second.clear();
}
}
}
}

void embedLauncherExternalIntrinsics(std::string& input,
const HeaderDepsInfo& info,
SessionStage& stage) {
auto backend = stage.getBackend();
if (backend != TargetBackend::_LAUNCHER) {
return;
}

const auto& usedIntrinsics = info.externalIntrinsics;
if (usedIntrinsics.empty()) {
return;
}

auto session = stage.getSession();
auto& session = stage.getSession();
auto& sm = stage.getCompiler().getSourceManager();
for (const auto& includeIntrinsic : usedIntrinsics) {
auto maybeIntrinsicPath = getExternalInstrincisInclude(session, includeIntrinsic);
for (auto& mappedIntrinsic : deps.externalIntrinsicsSources) {
auto maybeIntrinsicPath = getExternalInstrincisInclude(session, mappedIntrinsic.first);
if (!maybeIntrinsicPath) {
std::string error = "Count not find implementation for " + includeIntrinsic;
std::string error = "Count not find implementation for " + mappedIntrinsic.first;
session.pushError(std::error_code(), error);
return;
}
auto intrinsicPath = maybeIntrinsicPath.value();
auto infoResult = getExternalIntrinsicSource(backend, intrinsicPath, sm);
if (!infoResult) {
session.pushError(std::error_code(), infoResult.error());
auto intrinsicResult = getExternalIntrinsicSource(backend, intrinsicPath, sm);
if (!intrinsicResult) {
session.pushError(std::error_code(), intrinsicResult.error());
return;
}
auto buffer = std::move(infoResult.value());
// INFO: make temporary buffer because pointer could be not null terminated
input.insert(0, buffer->getBuffer().str());
mappedIntrinsic.second = std::move(intrinsicResult.value());
}
}
} // namespace oklt
20 changes: 4 additions & 16 deletions lib/core/intrinsics/external_intrinsics.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,15 @@
#include <clang/Basic/FileEntry.h>
#include <string>

namespace clang {
class CompilerInstance;
class SourceManager;
} // namespace clang

namespace oklt {

class TranspilerSession;
class TransformedFiles;
class SessionStage;
class HeaderDepsInfo;

bool overrideExternalIntrinsic(TranspilerSession& session,
bool overrideExternalIntrinsic(SessionStage& stage,
HeaderDepsInfo& deps,
const std::string& includedFileName,
clang::OptionalFileEntryRef includedFile,
clang::SourceManager& sourceManager);

void nullyLauncherExternalIntrinsics(TransformedFiles& inputs, SessionStage& stage);

void embedLauncherExternalIntrinsics(std::string& input,
const HeaderDepsInfo& info,
SessionStage& stage);
clang::OptionalFileEntryRef includedFile);

void updateExternalIntrinsicMap(SessionStage& stage, HeaderDepsInfo& deps);
} // namespace oklt
22 changes: 7 additions & 15 deletions lib/core/transpiler_session/code_generator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@
#include "core/transpiler_session/transpiler_session.h"

#include "core/intrinsics/builtin_intrinsics.h"
#include "core/intrinsics/external_intrinsics.h"
// #include "core/intrinsics/external_intrinsics.h"

#include "core/handler_manager/handler_manager.h"

#include "core/utils/attributes.h"
// #include "core/utils/attributes.h"
#include "core/vfs/overlay_fs.h"

#include <clang/AST/Attr.h>
Expand Down Expand Up @@ -86,10 +86,8 @@ void removeSystemHeaders(SessionStage& stage, const HeaderDepsInfo& deps) {
rewriter.RemoveText({dep.hashLoc, dep.filenameRange.getEnd()});
}

for (const auto& intrinsicDep : deps.externalIntrinsicDeps) {
SPDLOG_TRACE(
"remove system include {} {}", intrinsicDep.relativePath, intrinsicDep.fileName);
rewriter.RemoveText({intrinsicDep.hashLoc, intrinsicDep.filenameRange.getEnd()});
for (const auto& intrinsic : deps.externalIntrinsicHeaders) {
rewriter.RemoveText({intrinsic.hashLoc, intrinsic.filenameRange.getEnd()});
}
}

Expand Down Expand Up @@ -169,18 +167,14 @@ std::string restoreSystemAndBackendHeaders(SessionStage& stage,
}
}

embedLauncherExternalIntrinsics(input, deps, stage);
for (const auto& externalIntrinsicSource : deps.externalIntrinsicsSources) {
input.insert(0, externalIntrinsicSource.second);
}

for (auto it = deps.backendHeaders.rbegin(); it < deps.backendHeaders.rend(); ++it) {
input.insert(0, *it);
}

if (backend != TargetBackend::_LAUNCHER) {
for (const auto& intrinsicDep : deps.externalIntrinsicDeps) {
input.insert(0, "#include <" + intrinsicDep.fileName + ">\n");
}
}

// restore system headers
for (const auto& dep : deps.topLevelDeps) {
if (!clang::SrcMgr::isSystem(dep.fileType)) {
Expand All @@ -197,8 +191,6 @@ tl::expected<std::string, Error> fuseIncludeDeps(SessionStage& stage, const Head

auto inputs = gatherTransformedFiles(stage);

nullyLauncherExternalIntrinsics(inputs, stage);

auto preprocessedResult = preprocesseInputs(stage, inputs);
if (!preprocessedResult) {
return preprocessedResult;
Expand Down
83 changes: 20 additions & 63 deletions lib/core/transpiler_session/header_info.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
#include "core/transpiler_session/header_info.h"
#include "core/intrinsics/builtin_intrinsics.h"
#include "core/intrinsics/external_intrinsics.h"
#include "core/transpiler_session/transpiler_session.h"
#include "core/transpiler_session/session_stage.h"

namespace {}
namespace oklt {

using namespace llvm;

InclusionDirectiveCallback::InclusionDirectiveCallback(TranspilerSession& session,
InclusionDirectiveCallback::InclusionDirectiveCallback(SessionStage& stage,
HeaderDepsInfo& deps_,
clang::SourceManager& sm_)
: deps(deps_),
sm(sm_),
_session(session),
_isInExternalIntrinsic(false) {}
_stage(stage) {}

void InclusionDirectiveCallback::InclusionDirective(clang::SourceLocation hashLoc,
const clang::Token& includeTok,
Expand All @@ -36,67 +35,25 @@ void InclusionDirectiveCallback::InclusionDirective(clang::SourceLocation hashLo
}

auto fileNameStr = fileName.str();
auto isIntrinsic = overrideExternalIntrinsic(_session, fileNameStr, file, sm);
if (isIntrinsic) {
deps.externalIntrinsics.push_back(fileNameStr);
_extIntrinsicFID = sm.getFileID(hashLoc);
}
bool isIntrinsic = overrideExternalIntrinsic(_stage, deps, fileNameStr, file);

auto dep = HeaderDep{
.hashLoc = hashLoc,
.includeTok = includeTok,
.fileName = fileNameStr,
.isAngled = isAngled,
.filenameRange = filenameRange,
.file = file,
.searchPath = searchPath.str(),
.relativePath = relativePath.str(),
.imported = imported,
.fileType = fileType,
};

// INFO: force instrinsic includes to be system includes
// they will be removed & attached to final transpiled source
if (_isInExternalIntrinsic) {
fileType = clang::SrcMgr::CharacteristicKind::C_System;
deps.externalIntrinsicDeps.push_back(HeaderDep{
.hashLoc = hashLoc,
.includeTok = includeTok,
.fileName = fileNameStr,
.isAngled = isAngled,
.filenameRange = filenameRange,
.file = file,
.searchPath = searchPath.str(),
.relativePath = relativePath.str(),
.imported = imported,
.fileType = fileType,
});
if (isIntrinsic) {
deps.externalIntrinsicHeaders.push_back(std::move(dep));
} else {
deps.topLevelDeps.push_back(HeaderDep{
.hashLoc = hashLoc,
.includeTok = includeTok,
.fileName = fileNameStr,
.isAngled = isAngled,
.filenameRange = filenameRange,
.file = file,
.searchPath = searchPath.str(),
.relativePath = relativePath.str(),
.imported = imported,
.fileType = fileType,
});
deps.topLevelDeps.push_back(std::move(dep));
}
}

void InclusionDirectiveCallback::FileChanged(clang::SourceLocation Loc,
FileChangeReason Reason,
clang::SrcMgr::CharacteristicKind FileType,
clang::FileID PrevFID) {
if (Reason == FileChangeReason::EnterFile) {
if (_extIntrinsicFID.isValid()) {
_isInExternalIntrinsic = PrevFID == _extIntrinsicFID;
return;
}
}

if (Reason == FileChangeReason::ExitFile) {
auto thisFID = sm.getFileID(Loc);
if (_extIntrinsicFID.isValid() && thisFID == _extIntrinsicFID) {
_extIntrinsicFID = clang::FileID();
_isInExternalIntrinsic = false;
return;
}
}
}

bool InclusionDirectiveCallback::FileNotFound(clang::StringRef FileName) {
return _isInExternalIntrinsic;
}

} // namespace oklt
Loading

0 comments on commit adece8c

Please sign in to comment.