diff --git a/configure b/configure index f23845558d6..b777b7e9db1 100755 --- a/configure +++ b/configure @@ -25,7 +25,9 @@ if [ "$1" == "" ]; then CFG_CMD_LINE_ARGS=$CONF_DEFAULT fi -python2.7 etc/configure.py $CFG_CMD_LINE_ARGS -if [ -f "bin/activate" ]; then - source bin/activate +CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +python2.7 $CONFIGURE_ROOT_DIR/etc/configure.py $CFG_CMD_LINE_ARGS +if [ -f "$CONFIGURE_ROOT_DIR/bin/activate" ]; then + source $CONFIGURE_ROOT_DIR/bin/activate fi diff --git a/configure.bat b/configure.bat index 15b7311110b..7208a8c45fd 100644 --- a/configure.bat +++ b/configure.bat @@ -15,6 +15,7 @@ set TPP_DIR_DEV=thirdparty/dev set CONF_DEFAULT="etc/conf/dev" @rem ################################# +set SCANCODE_ROOT_DIR=%~dp0 set SCANCODE_CLI_ARGS= @rem Collect/Slurp all command line arguments in a variable :collectarg @@ -53,12 +54,12 @@ if not exist "c:\python27\python.exe" ( exit /b 1 ) -call c:\python27\python.exe etc/configure.py %SCANCODE_CLI_ARGS% +call c:\python27\python.exe %SCANCODE_ROOT_DIR%etc\configure.py %SCANCODE_CLI_ARGS% if %errorlevel% neq 0 ( exit /b %errorlevel% ) -if exist bin\activate ( - bin\activate +if exist %SCANCODE_ROOT_DIR%bin\activate ( + %SCANCODE_ROOT_DIR%bin\activate ) goto EOS diff --git a/etc/configure.py b/etc/configure.py index 612427c565d..cf863fb5012 100644 --- a/etc/configure.py +++ b/etc/configure.py @@ -78,6 +78,7 @@ raise Exception('Unsupported OS/platform') platform_names = tuple() + # common file basenames for requirements and scripts base = ('base',) @@ -96,12 +97,12 @@ shell_scripts = ('win.bat',) -def call(cmd): +def call(cmd, root_dir): """ Run a `cmd` command (as a list of args) with all env vars.""" cmd = ' '.join(cmd) - if subprocess.Popen(cmd, shell=True, env=dict(os.environ)).wait() != 0: + if subprocess.Popen(cmd, shell=True, env=dict(os.environ), cwd=root_dir).wait() != 0: print() - print('Failed to execute command:\n%(cmd)s' % locals()) + print('Failed to execute command:\n%(cmd)s. Aborting...' % locals()) sys.exit(1) @@ -121,7 +122,7 @@ def find_pycache(root_dir): def clean(root_dir): """ - Remove cleanable directories and files. + Remove cleanable directories and files in root_dir. """ print('* Cleaning ...') cleanable = '''build bin lib Lib include Include Scripts local @@ -148,12 +149,14 @@ def clean(root_dir): pass -def build_pip_dirs_args(paths, option='--extra-search-dir='): +def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='): """ - Return an iterable of pip command line arguments for the `option` pip - command line option using a list of `paths` to directories. + Return an iterable of pip command line options for `option` of pip using a + list of `paths` to directories. """ for path in paths: + if not os.path.isabs(path): + path = os.path.join(root_dir, path) if os.path.exists(path): yield option + path @@ -161,7 +164,7 @@ def build_pip_dirs_args(paths, option='--extra-search-dir='): def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False): """ Create a virtualenv in `root_dir` using the `std_python` Python - executable. One of the tpp_dirs must contain a vendored virtualenv.py and + executable. One of the `tpp_dirs` must contain a vendored virtualenv.py and virtualenv dependencies such as setuptools and pip packages. @std_python: Path or name of the Python executable to use. @@ -178,19 +181,24 @@ def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False): # search virtualenv.py in the tpp_dirs. keep the first found venv_py = None for tpd in tpp_dirs: - venv = os.path.join(tpd, 'virtualenv.py') + venv = os.path.join(root_dir, tpd, 'virtualenv.py') if os.path.exists(venv): venv_py = venv break - # TODO: error out if venv_py not found + + # error out if venv_py not found + if not venv_py: + print("Configuration Error ... aborting.") + exit(1) + vcmd = [std_python, venv_py, '--never-download'] if quiet: vcmd += ['--quiet'] # third parties may be in more than one directory - vcmd.extend(build_pip_dirs_args(tpp_dirs)) + vcmd.extend(build_pip_dirs_args(tpp_dirs, root_dir)) # we create the virtualenv in the root_dir vcmd.append(root_dir) - call(vcmd) + call(vcmd, root_dir) def activate(root_dir): @@ -202,35 +210,42 @@ def activate(root_dir): def install_3pp(configs, root_dir, tpp_dirs, quiet=False): - """ Install requirements with pip.""" + """ + Install requirements from requirement files found in `configs` with pip, + using the vendored components in `tpp_dirs`. + """ print("* Installing components ...") - for req_file in get_conf_files(configs, requirements): + requirement_files = get_conf_files(configs, root_dir, requirements) + for req_file in requirement_files: pcmd = ['pip', 'install', '--no-allow-external', '--use-wheel', '--no-index'] if quiet: pcmd += ['--quiet'] - pcmd.extend(build_pip_dirs_args(tpp_dirs, '--find-links=')) + pip_dir_args = list(build_pip_dirs_args(tpp_dirs, root_dir, '--find-links=')) + pcmd.extend(pip_dir_args) req_loc = os.path.join(root_dir, req_file) pcmd.extend(['-r' , req_loc]) - call(pcmd) + call(pcmd, root_dir) def run_scripts(configs, root_dir, configured_python): - """ Run py_script and sh_script scripts.""" + """ + Run Python scripts and shell scripts found in `configs`. + """ print("* Configuring ...") # Run Python scripts for each configurations - for py_script in get_conf_files(configs, python_scripts): + for py_script in get_conf_files(configs, root_dir, python_scripts): cmd = [configured_python, os.path.join(root_dir, py_script)] - call(cmd) + call(cmd, root_dir) # Run sh_script scripts for each configurations - for sh_script in get_conf_files(configs, shell_scripts): + for sh_script in get_conf_files(configs, root_dir, shell_scripts): # we source the scripts on posix cmd = ['.'] if on_win: cmd = [] cmd = cmd + [os.path.join(root_dir, sh_script)] - call(cmd) + call(cmd, root_dir) def chmod_bin(directory): @@ -244,16 +259,18 @@ def chmod_bin(directory): os.chmod(os.path.join(path, f), rwx) -def get_conf_files(config_dir_paths, file_names=requirements): +def get_conf_files(config_dir_paths, root_dir, file_names=requirements): """ - Based on config_dir_paths return a list of collected path-prefixed file - paths matching names in a file_names tuple. Returned paths are posix - paths. + Return a list of collected path-prefixed file paths matching names in a + file_names tuple, based on config_dir_paths, root_dir and the types of + file_names requested. Returned paths are posix paths. @config_dir_paths: Each config_dir_path is a relative from the project root to a config dir. This script should always be called from the project root dir. + @root_dir: The project absolute root dir. + @file_names: get requirements, python or shell files based on list of supported file names provided as a tuple of supported file_names. @@ -267,9 +284,9 @@ def get_conf_files(config_dir_paths, file_names=requirements): """ # collect files for each requested dir path collected = [] - for config_dir_path in config_dir_paths: - if not os.path.exists(config_dir_path): + abs_config_dir_path = os.path.join(root_dir, config_dir_path) + if not os.path.exists(abs_config_dir_path): print('Configuration directory %(config_dir_path)s ' 'does not exists. Skipping.' % locals()) continue @@ -279,19 +296,18 @@ def get_conf_files(config_dir_paths, file_names=requirements): current = None for path in paths: if not current: - current = (path, os.path.abspath(path),) + current = (path, os.path.join(root_dir, path),) else: base_path, base_loc = current current = (os.path.join(base_path, path), os.path.join(base_loc, path),) - path, loc = current # we iterate on known filenames to ensure the defined precedence # is respected (posix over mac, linux), etc for n in file_names: for f in os.listdir(loc): if f == n: - f_loc = os.path.join(path, f) + f_loc = os.path.join(loc, f) if f_loc not in collected: collected.append(f_loc) @@ -312,7 +328,7 @@ def get_conf_files(config_dir_paths, file_names=requirements): bin_dir = os.path.join(root_dir, 'bin') standard_python = sys.executable - # create a CONFIGURE_QUIET env var to run quietly + # you must create a CONFIGURE_QUIET env var if you want to run quietly run_quiet = 'CONFIGURE_QUIET' in os.environ if on_win: @@ -323,30 +339,34 @@ def get_conf_files(config_dir_paths, file_names=requirements): os.makedirs(scripts_dir) if not os.path.exists(bin_dir): cmd = ('mklink /J %(bin_dir)s %(scripts_dir)s' % locals()).split() - call(cmd) + call(cmd, root_dir) else: configured_python = os.path.join(bin_dir, 'python') scripts_dir = bin_dir - # get requested configuration paths and install components and run scripts + # Get requested configuration paths to collect components and scripts later configs = [] for path in args[:]: - if os.path.exists(path): - configs.append(path) + if not os.path.isabs(path): + abs_path = os.path.join(root_dir, path) + if os.path.exists(abs_path): + configs.append(path) else: print() print('WARNING: Skipping missing Configuration directory:\n' ' %(path)s does not exist.' % locals()) print() - # one or more third-party directories may exist - # as environment variables prefixed with TPP_DIR + # Collect vendor directories from environment variables: one or more third- + # party directories may exist as environment variables prefixed with TPP_DIR thirdparty_dirs = [] for envvar, path in os.environ.items(): if not envvar.startswith('TPP_DIR'): continue - if os.path.exists(path): - thirdparty_dirs.append(path) + if not os.path.isabs(path): + abs_path = os.path.join(root_dir, path) + if os.path.exists(abs_path): + thirdparty_dirs.append(path) else: print() print('WARNING: Skipping missing Python thirdparty directory:\n' @@ -355,9 +375,9 @@ def get_conf_files(config_dir_paths, file_names=requirements): ' set %(envvar)s=%(path)s' % locals()) print() + # Finally execute our three steps: venv, install and scripts if not os.path.exists(configured_python): - create_virtualenv(standard_python, root_dir, thirdparty_dirs, - quiet=run_quiet) + create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=run_quiet) activate(root_dir) install_3pp(configs, root_dir, thirdparty_dirs, quiet=run_quiet) diff --git a/extractcode b/extractcode index b77f8ae99ec..97bf4ebc626 100755 --- a/extractcode +++ b/extractcode @@ -6,11 +6,10 @@ # A minimal shell wrapper to the CLI entry point SCANCODE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$SCANCODE_ROOT_DIR" SCANCODE_CONFIGURED_PYTHON=$SCANCODE_ROOT_DIR/bin/python if [ ! -f "$SCANCODE_CONFIGURED_PYTHON" ]; then - echo "* Configuring ScanCode ..." + echo "* Configuring ScanCode for first use..." CONFIGURE_QUIET=1 $SCANCODE_ROOT_DIR/configure etc/conf fi diff --git a/extractcode.bat b/extractcode.bat index 68f9e2879b6..3acb9a071ff 100644 --- a/extractcode.bat +++ b/extractcode.bat @@ -6,7 +6,6 @@ @rem A minimal shell wrapper to the CLI entry point set SCANCODE_ROOT_DIR=%~dp0 -cd %SCANCODE_ROOT_DIR% set SCANCODE_CMD_LINE_ARGS= set SCANCODE_CONFIGURED_PYTHON=%SCANCODE_ROOT_DIR%\bin\python.exe @@ -25,7 +24,7 @@ if not exist %SCANCODE_CONFIGURED_PYTHON% goto configure goto scancode :configure - echo * Configuring ScanCode ... + echo * Configuring ScanCode for first use... set CONFIGURE_QUIET=1 call %SCANCODE_ROOT_DIR%\configure etc/conf if %errorlevel% neq 0 ( diff --git a/scancode b/scancode index 60cd6e08896..64c069fa72d 100755 --- a/scancode +++ b/scancode @@ -6,11 +6,10 @@ # A minimal shell wrapper to the CLI entry point SCANCODE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -cd "$SCANCODE_ROOT_DIR" SCANCODE_CONFIGURED_PYTHON=$SCANCODE_ROOT_DIR/bin/python if [ ! -f "$SCANCODE_CONFIGURED_PYTHON" ]; then - echo "* Configuring ScanCode ..." + echo "* Configuring ScanCode for first use..." CONFIGURE_QUIET=1 $SCANCODE_ROOT_DIR/configure etc/conf fi diff --git a/scancode.bat b/scancode.bat index c64d32878c8..a9cec9083e0 100644 --- a/scancode.bat +++ b/scancode.bat @@ -6,8 +6,6 @@ @rem A minimal shell wrapper to the CLI entry point set SCANCODE_ROOT_DIR=%~dp0 -cd %SCANCODE_ROOT_DIR% - set SCANCODE_CMD_LINE_ARGS= set SCANCODE_CONFIGURED_PYTHON=%SCANCODE_ROOT_DIR%\bin\python.exe @@ -25,7 +23,7 @@ if not exist %SCANCODE_CONFIGURED_PYTHON% goto configure goto scancode :configure - echo * Configuring ScanCode ... + echo * Configuring ScanCode for first use... set CONFIGURE_QUIET=1 call %SCANCODE_ROOT_DIR%\configure etc/conf if %errorlevel% neq 0 (