Skip to content
This repository has been archived by the owner on Sep 18, 2024. It is now read-only.

fix no log from subprocess on trial #3653

Merged
merged 10 commits into from
May 27, 2021
Merged

fix no log from subprocess on trial #3653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ee40d6c
fix no log from subprocess on trial
May 19, 2021
e0eaef9
fix no log view from web UI
May 25, 2021
437aaac
fix ut
May 26, 2021
750e468
fix ut
May 26, 2021
dba74d5
fix ut
May 26, 2021
16f327b
fix ut
May 26, 2021
d72aa02
fix ut
May 26, 2021
c2beb9f
fix -encoding not working on win32
May 26, 2021
8a5115a
add new type(TRIAL_STDOUT) for log
May 27, 2021
a00aa31
revert ut
May 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ts/nni_manager/common/trainingService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
*/
type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED';

type LogType = 'TRIAL_LOG' | 'TRIAL_ERROR';
type LogType = 'TRIAL_LOG' | 'TRIAL_STDOUT' | 'TRIAL_ERROR';

interface TrainingServiceMetadata {
readonly key: string;
Expand Down
7 changes: 5 additions & 2 deletions ts/nni_manager/training_service/local/localTrainingService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ class LocalTrainingService implements TrainingService {
let logPath: string;
if (logType === 'TRIAL_LOG') {
logPath = path.join(this.rootDir, 'trials', trialJobId, 'trial.log');
} else if (logType === 'TRIAL_STDOUT'){
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stdout');
} else if (logType === 'TRIAL_ERROR') {
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stderr');
} else {
Expand Down Expand Up @@ -412,15 +414,16 @@ class LocalTrainingService implements TrainingService {
private getScript(workingDirectory: string): string[] {
const script: string[] = [];
if (process.platform === 'win32') {
script.push(`$PSDefaultParameterValues = @{'Out-File:Encoding' = 'utf8'}`);
script.push(`cd $env:NNI_CODE_DIR`);
script.push(
`cmd.exe /c ${this.config.trialCommand} 2>&1 | Out-File "${path.join(workingDirectory, 'stderr')}" -encoding utf8`,
`cmd.exe /c ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`,
`$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
`$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
} else {
script.push(`cd $NNI_CODE_DIR`);
script.push(`eval ${this.config.trialCommand} 2>"${path.join(workingDirectory, 'stderr')}"`);
script.push(`eval ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`);
if (process.platform === 'darwin') {
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration
Expand Down