Add log information for gpu collector (microsoft#3506)

acured · Apr 7, 2021 · 876a366 · 876a366
1 parent aa42ddd
commit 876a366
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/ts/nni_manager/training_service/common/gpuData.ts b/ts/nni_manager/training_service/common/gpuData.ts
@@ -79,6 +79,7 @@ export function parseGpuIndices(gpuIndices?: string): Set<number> | undefined {
 export const GPU_INFO_COLLECTOR_FORMAT_WINDOWS: string =
     `
 $env:METRIC_OUTPUT_DIR="{0}"
-$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow
+$app = Start-Process "python" -ArgumentList "-m nni.tools.gpu_tool.gpu_metrics_collector" -passthru -NoNewWindow \
+-redirectStandardOutput {0}\\stdout -redirectStandardError {0}\\stderr
 Write $app.ID | Out-File {1} -NoNewline -encoding utf8
 `;
diff --git a/ts/nni_manager/training_service/common/util.ts b/ts/nni_manager/training_service/common/util.ts
@@ -236,7 +236,8 @@ export function getScriptName(fileNamePrefix: string): string {
 }
 
 export function getGpuMetricsCollectorBashScriptContent(scriptFolder: string): string {
-    return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector`;
+    return `echo $$ > ${scriptFolder}/pid ; METRIC_OUTPUT_DIR=${scriptFolder} python3 -m nni.tools.gpu_tool.gpu_metrics_collector \
+1>${scriptFolder}/stdout 2>${scriptFolder}/stderr`;
 }
 
 export function runGpuMetricsCollector(scriptFolder: string): void {