diff --git a/docs/en_US/TrainingService/RemoteMachineMode.rst b/docs/en_US/TrainingService/RemoteMachineMode.rst index aa2052bdb5..9a7a2d94ae 100644 --- a/docs/en_US/TrainingService/RemoteMachineMode.rst +++ b/docs/en_US/TrainingService/RemoteMachineMode.rst @@ -144,7 +144,7 @@ Files in ``codeDir`` will be uploaded to remote machines automatically. You can Configure python environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, commands and scripts will be executed in the default environment in remote machine. If there are multiple python virtual environments in your remote machine, and you want to run experiments in a specific environment, then use **preCommand** to specify a python environment on your remote machine. +By default, commands and scripts will be executed in the default environment in remote machine. If there are multiple python virtual environments in your remote machine, and you want to run experiments in a specific environment, then use **pythonPath** to specify a python environment on your remote machine. Use ``examples/trials/mnist-tfv2`` as the example. Below is content of ``examples/trials/mnist-tfv2/config_remote.yml``\ : diff --git a/docs/en_US/reference/experiment_config.rst b/docs/en_US/reference/experiment_config.rst index 7143376dd6..79a147c368 100644 --- a/docs/en_US/reference/experiment_config.rst +++ b/docs/en_US/reference/experiment_config.rst @@ -538,10 +538,16 @@ If `trialGpuNumber`_ is less than the length of this value, only a subset will b This will be used as ``CUDA_VISIBLE_DEVICES`` environment variable. -trialPrepareCommand +pythonPath ------------------- -Command(s) to run before launching each trial. +Specify a python environment, this path will insert at the front of PATH. Here are some examples: + - (linux) pythonPath: /opt/python3.7/bin + - (windows) pythonPath: C:/Python37 + +Notice: If you are working on anacondaļ¼Œthere are some difference. You have to add "../script" and "../Library/bin" to this and separated by ";" on windows, example as below: + - (linux anaconda) pythonPath: /home/yourname/anaconda3/envs/myenv/bin/ + - (windows anaconda) pythonPath: C:/Users/yourname/.conda/envs/myenv;C:/Users/yourname/.conda/envs/myenv/Scripts;C:/Users/yourname/.conda/envs/myenv/Library/bin type: ``Optional[str]`` diff --git a/docs/tools/md2rst.py b/docs/tools/md2rst.py index a99a6816e8..1136c75990 100644 --- a/docs/tools/md2rst.py +++ b/docs/tools/md2rst.py @@ -21,7 +21,7 @@ def single_line_process(line): line = re.sub(r'\\\* \*\*(.*?)\*\* \(\\\*\s*(.*?)\s*\*\\ \)', r'* \1 (\2)', line) line = re.sub(r'\<(.*)\.md(\>|#)', r'<\1.rst\2', line) line = re.sub(r'`\*\*(.*?)\*\* <#(.*?)>`__', r'`\1 <#\2>`__', line) - line = re.sub(r'\*\* (classArgs|stop|FLOPS.*?|pruned.*?|large.*?|path|preCommand|2D.*?|codeDirectory|ps|worker|Tuner|Assessor)\*\*', + line = re.sub(r'\*\* (classArgs|stop|FLOPS.*?|pruned.*?|large.*?|path|pythonPath|2D.*?|codeDirectory|ps|worker|Tuner|Assessor)\*\*', r' **\1**', line) line = line.replace('.. code-block:::: bash', '.. code-block:: bash') diff --git a/examples/trials/mnist-tfv2/config_remote.yml b/examples/trials/mnist-tfv2/config_remote.yml index 0c39914dbc..09fb8634fe 100644 --- a/examples/trials/mnist-tfv2/config_remote.yml +++ b/examples/trials/mnist-tfv2/config_remote.yml @@ -24,9 +24,9 @@ machineList: - ip: ${replace_to_your_remote_machine_ip} username: ${replace_to_your_remote_machine_username} sshKeyPath: ${replace_to_your_remote_machine_sshKeyPath} - # Pre-command will be executed before the remote machine executes other commands. - # Below is an example of specifying python environment. - # If you want to execute multiple commands, please use "&&" to connect them. - # preCommand: source ${replace_to_absolute_path_recommended_here}/bin/activate - # preCommand: source ${replace_to_conda_path}/bin/activate ${replace_to_conda_env_name} - preCommand: export PATH=${replace_to_python_environment_path_in_your_remote_machine}:$PATH + # Below are examples of specifying python environment. + # pythonPath: /opt/python3.7/bin + # pythonPath: C:/Python37 + # Below is an example of specifying python environment for windows anaconda user. Multiple paths separated by ';'. + # pythonPath: C:/Users/yourname/.conda/envs/myenv;C:/Users/yourname/.conda/envs/myenv/Scripts;C:/Users/yourname/.conda/envs/myenv/Library/bin + pythonPath: ${replace_to_python_environment_path_in_your_remote_machine} diff --git a/nni/experiment/config/convert.py b/nni/experiment/config/convert.py index c2f8fc95a5..1d83d3fa25 100644 --- a/nni/experiment/config/convert.py +++ b/nni/experiment/config/convert.py @@ -105,7 +105,7 @@ def _handle_training_service(ts, data): 'gpuIndices': _convert_gpu_indices(machine.get('gpuIndices')), 'maxTrialNumPerGpu': machine.get('maxTrialNumPerGpu'), 'useActiveGpu': machine.get('useActiveGpu'), - 'preCommand': machine.get('trialPrepareCommand') + 'pythonPath': machine.get('pythonPath') } machine_v1 = {k: v for k, v in machine_v1.items() if v is not None} data['machineList'].append(machine_v1) diff --git a/nni/experiment/config/remote.py b/nni/experiment/config/remote.py index 45303b0c57..16cf5d3089 100644 --- a/nni/experiment/config/remote.py +++ b/nni/experiment/config/remote.py @@ -23,7 +23,7 @@ class RemoteMachineConfig(ConfigBase): use_active_gpu: bool = False max_trial_number_per_gpu: int = 1 gpu_indices: Optional[Union[List[int], str]] = None - trial_prepare_command: Optional[str] = None + python_path: Optional[str] = None _canonical_rules = { 'ssh_key_file': util.canonical_path, diff --git a/ts/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts b/ts/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts index 3aee8ba020..3529948381 100644 --- a/ts/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts +++ b/ts/nni_manager/training_service/remote_machine/test/shellExecutor.test.ts @@ -88,7 +88,7 @@ describe('ShellExecutor test', () => { await executor.close(); }); - it('Test preCommand-1', async () => { + it('Test pythonPath-1', async () => { if (skip) { return; } @@ -99,16 +99,16 @@ describe('ShellExecutor test', () => { await executor.close(); }); - it('Test preCommand-2', async () => { + it('Test pythonPath-2', async () => { if (skip) { return; } const executor: ShellExecutor = new ShellExecutor(); - rmMeta.preCommand = isWindows ? "set TEST_PRE_COMMAND=test_pre_command" : "export TEST_PRE_COMMAND=test_pre_command"; + rmMeta.pythonPath = "test_python_path"; await executor.initialize(rmMeta); - const command = isWindows ? "python -c \"import os; print(os.environ.get(\'TEST_PRE_COMMAND\'))\"" : "python3 -c \"import os; print(os.environ.get(\'TEST_PRE_COMMAND\'))\""; + const command = isWindows ? "python -c \"import os; print(os.environ.get(\'PATH\'))\"" : "python3 -c \"import os; print(os.environ.get(\'PATH\'))\""; const result = (await executor.executeScript(command, false, false)).stdout.replace(/[\ +\r\n]/g, ""); - chai.expect(result).eq("test_pre_command"); + chai.expect(result).contain("test_python_path"); await executor.close(); }); });