forked from microsoft/nni
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #72 from Microsoft/master
[Kubeflow Training Service] V1, merge from kubeflow branch to master …
- Loading branch information
Showing
28 changed files
with
1,116 additions
and
135 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
96 changes: 96 additions & 0 deletions
96
src/nni_manager/training_service/common/clusterJobRestServer.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| /** | ||
| * Copyright (c) Microsoft Corporation | ||
| * All rights reserved. | ||
| * | ||
| * MIT License | ||
| * | ||
| * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated | ||
| * documentation files (the "Software"), to deal in the Software without restriction, including without limitation | ||
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and | ||
| * to permit persons to whom the Software is furnished to do so, subject to the following conditions: | ||
| * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. | ||
| * | ||
| * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING | ||
| * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | ||
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, | ||
| * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | ||
| */ | ||
|
|
||
| 'use strict'; | ||
|
|
||
| import * as assert from 'assert'; | ||
| import { Request, Response, Router } from 'express'; | ||
| import * as bodyParser from 'body-parser'; | ||
| import * as component from '../../common/component'; | ||
| import { getBasePort, getExperimentId } from '../../common/experimentStartupInfo'; | ||
| import { RestServer } from '../../common/restServer' | ||
|
|
||
| /** | ||
| * Cluster Job Training service Rest server, provides rest API to support Cluster job metrics update | ||
| * | ||
| */ | ||
| @component.Singleton | ||
| export abstract class ClusterJobRestServer extends RestServer{ | ||
| private readonly API_ROOT_URL: string = '/api/v1/nni-pai'; | ||
|
|
||
| private readonly expId: string = getExperimentId(); | ||
|
|
||
| /** | ||
| * constructor to provide NNIRestServer's own rest property, e.g. port | ||
| */ | ||
| constructor() { | ||
| super(); | ||
| const basePort: number = getBasePort(); | ||
| assert(basePort && basePort > 1024); | ||
|
|
||
| this.port = basePort + 1; | ||
| } | ||
|
|
||
| public get clusterRestServerPort(): number { | ||
| if(!this.port) { | ||
| throw new Error('PAI Rest server port is undefined'); | ||
| } | ||
| return this.port; | ||
| } | ||
|
|
||
| /** | ||
| * NNIRestServer's own router registration | ||
| */ | ||
| protected registerRestHandler(): void { | ||
| this.app.use(bodyParser.json()); | ||
| this.app.use(this.API_ROOT_URL, this.createRestHandler()); | ||
| } | ||
|
|
||
| private createRestHandler() : Router { | ||
| const router: Router = Router(); | ||
|
|
||
| // tslint:disable-next-line:typedef | ||
| router.use((req: Request, res: Response, next) => { | ||
| this.log.info(`${req.method}: ${req.url}: body:\n${JSON.stringify(req.body, undefined, 4)}`); | ||
| res.setHeader('Content-Type', 'application/json'); | ||
| next(); | ||
| }); | ||
|
|
||
| router.post(`/update-metrics/${this.expId}/:trialId`, (req: Request, res: Response) => { | ||
| try { | ||
| this.log.info(`Get update-metrics request, trial job id is ${req.params.trialId}`); | ||
| this.log.info(`update-metrics body is ${JSON.stringify(req.body)}`); | ||
|
|
||
| this.handleTrialMetrics(req.body.jobId, req.body.metrics); | ||
|
|
||
| res.send(); | ||
| } | ||
| catch(err) { | ||
| this.log.error(`json parse metrics error: ${err}`); | ||
| res.status(500); | ||
| res.send(err.message); | ||
| } | ||
| }); | ||
|
|
||
| return router; | ||
| } | ||
|
|
||
| /** Abstract method to handle trial metrics data */ | ||
| protected abstract handleTrialMetrics(jobId : string, trialMetrics : any[]) : void; | ||
| } |
30 changes: 30 additions & 0 deletions
30
src/nni_manager/training_service/common/containerJobData.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /** | ||
| * Copyright (c) Microsoft Corporation | ||
| * All rights reserved. | ||
| * | ||
| * MIT License | ||
| * | ||
| * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated | ||
| * documentation files (the "Software"), to deal in the Software without restriction, including without limitation | ||
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and | ||
| * to permit persons to whom the Software is furnished to do so, subject to the following conditions: | ||
| * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. | ||
| * | ||
| * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING | ||
| * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | ||
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, | ||
| * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | ||
| */ | ||
|
|
||
| 'use strict'; | ||
|
|
||
| export const CONTAINER_INSTALL_NNI_SHELL_FORMAT: string = | ||
| `#!/bin/bash | ||
| if python3 -c 'import nni' > /dev/null 2>&1; then | ||
| # nni module is already installed, skip | ||
| return | ||
| else | ||
| # Install nni | ||
| python3 -m pip install --user --upgrade nni | ||
| fi`; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
src/nni_manager/training_service/kubeflow/kubeflowConfig.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| import { TrialConfig } from "../common/trialConfig"; | ||
|
|
||
| /** | ||
| * Copyright (c) Microsoft Corporation | ||
| * All rights reserved. | ||
| * | ||
| * MIT License | ||
| * | ||
| * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated | ||
| * documentation files (the "Software"), to deal in the Software without restriction, including without limitation | ||
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and | ||
| * to permit persons to whom the Software is furnished to do so, subject to the following conditions: | ||
| * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. | ||
| * | ||
| * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING | ||
| * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | ||
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, | ||
| * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
| * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | ||
| */ | ||
|
|
||
| 'use strict'; | ||
|
|
||
|
|
||
| /** operator types that kubeflow supported */ | ||
| export type KubeflowOperator = 'tf-operator' | 'pytorch-operator' | 'mxnet-operator' | 'caffe2-operator' | 'chainer-operator' | 'mpi-operator'; | ||
| export type KubeflowOperatorPlural = 'tfjobs' | 'pytorchjobs' | 'mxjobs' | 'caffe2jobs' | 'chainerjobs' | 'mpijobs'; | ||
|
|
||
| /** | ||
| * map from Kubeflow operator name to its plural name in K8S | ||
| */ | ||
| export const kubeflowOperatorMap : Map<KubeflowOperator, KubeflowOperatorPlural> = new Map<KubeflowOperator, KubeflowOperatorPlural>([ | ||
| ['tf-operator' , 'tfjobs'], | ||
| ['pytorch-operator', 'pytorchjobs'], | ||
| ['mxnet-operator', 'mxjobs'], | ||
| ['caffe2-operator', 'caffe2jobs'], | ||
| ['chainer-operator', 'chainerjobs'], | ||
| ['mpi-operator', 'mpijobs'] | ||
| ]); | ||
|
|
||
| /** | ||
| * Kuberflow cluster configuration | ||
| * | ||
| */ | ||
| export class KubeflowClusterConfig { | ||
| /** Name of Kubeflow operator, like tf-operator */ | ||
| public readonly operator: KubeflowOperator; | ||
| public readonly nfs: NFSConfig; | ||
| public readonly kubernetesServer: string; | ||
|
|
||
| /** | ||
| * Constructor | ||
| * @param userName User name of Kubeflow Cluster | ||
| * @param passWord password of Kubeflow Cluster | ||
| * @param host Host IP of Kubeflow Cluster | ||
| */ | ||
| constructor(operator: KubeflowOperator, nfs : NFSConfig, kubernetesServer : string) { | ||
| this.operator = operator; | ||
| this.nfs = nfs; | ||
| this.kubernetesServer = kubernetesServer; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * NFS configuration to store Kubeflow job related files | ||
| */ | ||
| export class NFSConfig { | ||
| /** IP Adress of NFS server */ | ||
| public readonly server : string; | ||
| /** exported NFS path on NFS server */ | ||
| public readonly path : string; | ||
|
|
||
| constructor(server : string, path : string) { | ||
| this.server = server; | ||
| this.path = path; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Trial job configuration for Kubeflow | ||
| */ | ||
| export class KubeflowTrialConfig extends TrialConfig { | ||
| public readonly cpuNum: number; | ||
| public readonly memoryMB: number; | ||
| public readonly image: string; | ||
|
|
||
| constructor(command : string, codeDir : string, gpuNum : number, cpuNum: number, memoryMB: number, image: string) { | ||
| super(command, codeDir, gpuNum); | ||
| this.cpuNum = cpuNum; | ||
| this.memoryMB = memoryMB; | ||
| this.image = image; | ||
| } | ||
| } |
Oops, something went wrong.