Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle large file uploads #62

Merged
merged 2 commits into from
Jan 12, 2019
Merged

Handle large file uploads #62

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e99ef32
Handle large file uploads
PeterJohnson Jan 12, 2019
24e8804
Fix Save function
PeterJohnson Jan 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 61 additions & 52 deletions deps/tools/rpiConfigServer_src/Application.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#include "Application.h"

#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>

Expand Down Expand Up @@ -80,86 +81,94 @@ void Application::Set(wpi::StringRef appType,
os << "exec " << appCommand << '\n';
}

m_appType = appType;

// terminate vision process so it reloads
VisionStatus::GetInstance()->Terminate(onFail);

UpdateStatus();
}

void Application::Upload(wpi::ArrayRef<uint8_t> contents,
std::function<void(wpi::StringRef)> onFail) {
int Application::StartUpload(wpi::StringRef appType, char* filename,
std::function<void(wpi::StringRef)> onFail) {
int fd = mkstemp(filename);
if (fd < 0) {
wpi::SmallString<64> msg;
msg = "could not open temporary file: ";
msg += std::strerror(errno);
onFail(msg);
}
return fd;
}

void Application::Upload(int fd, bool text, wpi::ArrayRef<uint8_t> contents) {
// write contents
wpi::raw_fd_ostream out(fd, false);
if (text) {
wpi::StringRef str(reinterpret_cast<const char*>(contents.data()),
contents.size());
// convert any Windows EOL to Unix
for (;;) {
size_t idx = str.find("\r\n");
if (idx == wpi::StringRef::npos) break;
out << str.slice(0, idx) << '\n';
str = str.slice(idx + 2, wpi::StringRef::npos);
}
out << str;
// ensure file ends with EOL
if (!str.empty() && str.back() != '\n') out << '\n';
} else {
out << contents;
}
}

void Application::FinishUpload(wpi::StringRef appType, int fd,
const char* tmpFilename,
std::function<void(wpi::StringRef)> onFail) {
wpi::StringRef filename;
bool text = false;
if (m_appType == "upload-java") {
if (appType == "upload-java") {
filename = "/uploaded.jar";
} else if (m_appType == "upload-cpp") {
} else if (appType == "upload-cpp") {
filename = "/uploaded";
} else if (m_appType == "upload-python") {
} else if (appType == "upload-python") {
filename = "/uploaded.py";
text = true;
} else {
wpi::SmallString<64> msg;
msg = "cannot upload application type '";
msg += m_appType;
msg += appType;
msg += "'";
onFail(msg);
::close(fd);
return;
}

wpi::SmallString<64> pathname;
pathname = EXEC_HOME;
pathname += filename;

// remove old file (need to do this as we can't overwrite a running exe)
if (unlink(pathname.c_str()) == -1) {
wpi::errs() << "could not remove app executable: " << std::strerror(errno)
// change ownership
if (fchown(fd, APP_UID, APP_GID) == -1) {
wpi::errs() << "could not change app ownership: " << std::strerror(errno)
<< '\n';
}

{
// open file for writing
std::error_code ec;
int fd;
if (wpi::sys::fs::openFileForWrite(pathname, fd, wpi::sys::fs::F_None)) {
wpi::SmallString<64> msg;
msg = "could not write ";
msg += pathname;
onFail(msg);
return;
}
// set file to be executable
if (fchmod(fd, 0775) == -1) {
wpi::errs() << "could not change app permissions: " << std::strerror(errno)
<< '\n';
}

// change ownership
if (fchown(fd, APP_UID, APP_GID) == -1) {
wpi::errs() << "could not change app ownership: " << std::strerror(errno)
<< '\n';
}
// close temporary file
::close(fd);

// set file to be executable
if (fchmod(fd, 0775) == -1) {
wpi::errs() << "could not change app permissions: "
<< std::strerror(errno) << '\n';
}
// remove old file (need to do this as we can't overwrite a running exe)
if (unlink(pathname.c_str()) == -1) {
wpi::errs() << "could not remove app executable: " << std::strerror(errno)
<< '\n';
}

// write contents and close file
wpi::raw_fd_ostream out(fd, true);
if (text) {
wpi::StringRef str(reinterpret_cast<const char*>(contents.data()),
contents.size());
// convert any Windows EOL to Unix
for (;;) {
size_t idx = str.find("\r\n");
if (idx == wpi::StringRef::npos) break;
out << str.slice(0, idx) << '\n';
str = str.slice(idx + 2, wpi::StringRef::npos);
}
out << str;
// ensure file ends with EOL
if (!str.empty() && str.back() != '\n') out << '\n';
} else {
out << contents;
}
// rename temporary file to new file
if (rename(tmpFilename, pathname.c_str()) == -1) {
wpi::errs() << "could not rename to app executable: "
<< std::strerror(errno) << '\n';
}

// terminate vision process so it reloads
Expand Down
10 changes: 5 additions & 5 deletions deps/tools/rpiConfigServer_src/Application.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,11 @@ class Application {

void Set(wpi::StringRef appType, std::function<void(wpi::StringRef)> onFail);

void Upload(wpi::ArrayRef<uint8_t> contents,
std::function<void(wpi::StringRef)> onFail);
int StartUpload(wpi::StringRef appType, char* filename,
std::function<void(wpi::StringRef)> onFail);
void Upload(int fd, bool text, wpi::ArrayRef<uint8_t> contents);
void FinishUpload(wpi::StringRef appType, int fd, const char* tmpFilename,
std::function<void(wpi::StringRef)> onFail);

void UpdateStatus();

Expand All @@ -40,9 +43,6 @@ class Application {
wpi::sig::Signal<const wpi::json&> status;

static std::shared_ptr<Application> GetInstance();

private:
std::string m_appType;
};

#endif // RPICONFIGSERVER_APPLICATION_H_
44 changes: 36 additions & 8 deletions deps/tools/rpiConfigServer_src/WebSocketHandlers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@

#include "WebSocketHandlers.h"

#include <unistd.h>

#include <cstring>
#include <memory>

#include <wpi/SmallVector.h>
Expand All @@ -29,7 +32,14 @@ namespace uv = wpi::uv;
#define SERVICE "/service/camera"

struct WebSocketData {
~WebSocketData() {
if (uploadFd != -1) ::close(uploadFd);
}

bool visionLogEnabled = false;
int uploadFd = -1;
bool uploadText = false;
char uploadFilename[128];

wpi::sig::ScopedConnection sysStatusConn;
wpi::sig::ScopedConnection sysWritableConn;
Expand Down Expand Up @@ -260,25 +270,43 @@ void ProcessWsText(wpi::WebSocket& ws, wpi::StringRef msg) {
wpi::errs() << "could not read networkSave value: " << e.what() << '\n';
return;
}
} else if (t == "applicationSave") {
} else if (t.startswith("application")) {
wpi::StringRef subType = t.substr(11);

auto statusFunc = [s = ws.shared_from_this()](wpi::StringRef msg) {
SendWsText(*s, {{"type", "status"}, {"message", msg}});
};

std::string appType;
try {
Application::GetInstance()->Set(
j.at("applicationType").get_ref<const std::string&>(), statusFunc);
appType = j.at("applicationType").get<std::string>();
} catch (const wpi::json::exception& e) {
wpi::errs() << "could not read applicationSave value: " << e.what()
<< '\n';
return;
}

if (subType == "Save") {
Application::GetInstance()->Set(appType, statusFunc);
} else if (subType == "StartUpload") {
auto d = ws.GetData<WebSocketData>();
std::strcpy(d->uploadFilename, EXEC_HOME "/appUploadXXXXXX");
d->uploadFd = Application::GetInstance()->StartUpload(
appType, d->uploadFilename, statusFunc);
d->uploadText = wpi::StringRef(appType).endswith("python");
} else if (subType == "FinishUpload") {
auto d = ws.GetData<WebSocketData>();
if (d->uploadFd != -1)
Application::GetInstance()->FinishUpload(appType, d->uploadFd,
d->uploadFilename, statusFunc);
d->uploadFd = -1;
SendWsText(ws, {{"type", "applicationSaveComplete"}});
}
}
}

void ProcessWsBinary(wpi::WebSocket& ws, wpi::ArrayRef<uint8_t> msg) {
auto statusFunc = [s = ws.shared_from_this()](wpi::StringRef msg) {
SendWsText(*s, {{"type", "status"}, {"message", msg}});
};
Application::GetInstance()->Upload(msg, statusFunc);
SendWsText(ws, {{"type", "applicationSaveComplete"}});
auto d = ws.GetData<WebSocketData>();
if (d->uploadFd != -1)
Application::GetInstance()->Upload(d->uploadFd, d->uploadText, msg);
}
36 changes: 32 additions & 4 deletions deps/tools/rpiConfigServer_src/resources/frcvision.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,12 +588,40 @@ $('#applicationSave').click(function() {
if (applicationFiles.length <= 0) {
return;
}

$('#applicationSave').button('loading');
var fr = new FileReader();
fr.onload = function(e) {
connection.send(e.target.result);

var msg = {
type: 'applicationStartUpload',
applicationType: $('#applicationType').val()
};
fr.readAsArrayBuffer(applicationFiles.item(0));
connection.send(JSON.stringify(msg));

var reader = new FileReader();
var file = applicationFiles.item(0);

function uploadFile(start) {
var nextSlice = start + (64 * 1024) + 1;
reader.onloadend = function(e) {
if (e.target.readyState !== FileReader.DONE) {
return;
}
connection.send(e.target.result);
if (nextSlice < file.size) {
// more to go
uploadFile(nextSlice);
} else {
// done
var msg = {
type: 'applicationFinishUpload',
applicationType: $('#applicationType').val()
};
connection.send(JSON.stringify(msg));
}
}
reader.readAsArrayBuffer(file.slice(start, nextSlice));
}
uploadFile(0);
});

// Start with display disconnected and start initial connection attempt
Expand Down