Skip to content

Commit

Permalink
#55 Ensured that root scripts can be called from anywhere
Browse files Browse the repository at this point in the history
 * key fix is ensuring that scripts do not cd to the root directory
   and that root-relative path vs. absolute paths are used as needed
  • Loading branch information
pombredanne authored and jdaguil committed Nov 24, 2015
1 parent a33a5f5 commit 1377a78
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 56 deletions.
8 changes: 5 additions & 3 deletions configure
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@ if [ "$1" == "" ]; then
CFG_CMD_LINE_ARGS=$CONF_DEFAULT
fi

python2.7 etc/configure.py $CFG_CMD_LINE_ARGS
if [ -f "bin/activate" ]; then
source bin/activate
CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

python2.7 $CONFIGURE_ROOT_DIR/etc/configure.py $CFG_CMD_LINE_ARGS
if [ -f "$CONFIGURE_ROOT_DIR/bin/activate" ]; then
source $CONFIGURE_ROOT_DIR/bin/activate
fi
7 changes: 4 additions & 3 deletions configure.bat
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ set TPP_DIR_DEV=thirdparty/dev
set CONF_DEFAULT="etc/conf/dev"
@rem #################################

set SCANCODE_ROOT_DIR=%~dp0
set SCANCODE_CLI_ARGS=
@rem Collect/Slurp all command line arguments in a variable
:collectarg
Expand Down Expand Up @@ -53,12 +54,12 @@ if not exist "c:\python27\python.exe" (
exit /b 1
)

call c:\python27\python.exe etc/configure.py %SCANCODE_CLI_ARGS%
call c:\python27\python.exe %SCANCODE_ROOT_DIR%etc\configure.py %SCANCODE_CLI_ARGS%
if %errorlevel% neq 0 (
exit /b %errorlevel%
)
if exist bin\activate (
bin\activate
if exist %SCANCODE_ROOT_DIR%bin\activate (
%SCANCODE_ROOT_DIR%bin\activate
)
goto EOS

Expand Down
102 changes: 61 additions & 41 deletions etc/configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
raise Exception('Unsupported OS/platform')
platform_names = tuple()


# common file basenames for requirements and scripts
base = ('base',)

Expand All @@ -96,12 +97,12 @@
shell_scripts = ('win.bat',)


def call(cmd):
def call(cmd, root_dir):
""" Run a `cmd` command (as a list of args) with all env vars."""
cmd = ' '.join(cmd)
if subprocess.Popen(cmd, shell=True, env=dict(os.environ)).wait() != 0:
if subprocess.Popen(cmd, shell=True, env=dict(os.environ), cwd=root_dir).wait() != 0:
print()
print('Failed to execute command:\n%(cmd)s' % locals())
print('Failed to execute command:\n%(cmd)s. Aborting...' % locals())
sys.exit(1)


Expand All @@ -121,7 +122,7 @@ def find_pycache(root_dir):

def clean(root_dir):
"""
Remove cleanable directories and files.
Remove cleanable directories and files in root_dir.
"""
print('* Cleaning ...')
cleanable = '''build bin lib Lib include Include Scripts local
Expand All @@ -148,20 +149,22 @@ def clean(root_dir):
pass


def build_pip_dirs_args(paths, option='--extra-search-dir='):
def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='):
"""
Return an iterable of pip command line arguments for the `option` pip
command line option using a list of `paths` to directories.
Return an iterable of pip command line options for `option` of pip using a
list of `paths` to directories.
"""
for path in paths:
if not os.path.isabs(path):
path = os.path.join(root_dir, path)
if os.path.exists(path):
yield option + path


def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False):
"""
Create a virtualenv in `root_dir` using the `std_python` Python
executable. One of the tpp_dirs must contain a vendored virtualenv.py and
executable. One of the `tpp_dirs` must contain a vendored virtualenv.py and
virtualenv dependencies such as setuptools and pip packages.
@std_python: Path or name of the Python executable to use.
Expand All @@ -178,19 +181,24 @@ def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False):
# search virtualenv.py in the tpp_dirs. keep the first found
venv_py = None
for tpd in tpp_dirs:
venv = os.path.join(tpd, 'virtualenv.py')
venv = os.path.join(root_dir, tpd, 'virtualenv.py')
if os.path.exists(venv):
venv_py = venv
break
# TODO: error out if venv_py not found

# error out if venv_py not found
if not venv_py:
print("Configuration Error ... aborting.")
exit(1)

vcmd = [std_python, venv_py, '--never-download']
if quiet:
vcmd += ['--quiet']
# third parties may be in more than one directory
vcmd.extend(build_pip_dirs_args(tpp_dirs))
vcmd.extend(build_pip_dirs_args(tpp_dirs, root_dir))
# we create the virtualenv in the root_dir
vcmd.append(root_dir)
call(vcmd)
call(vcmd, root_dir)


def activate(root_dir):
Expand All @@ -202,35 +210,42 @@ def activate(root_dir):


def install_3pp(configs, root_dir, tpp_dirs, quiet=False):
""" Install requirements with pip."""
"""
Install requirements from requirement files found in `configs` with pip,
using the vendored components in `tpp_dirs`.
"""
print("* Installing components ...")
for req_file in get_conf_files(configs, requirements):
requirement_files = get_conf_files(configs, root_dir, requirements)
for req_file in requirement_files:
pcmd = ['pip', 'install', '--no-allow-external',
'--use-wheel', '--no-index']
if quiet:
pcmd += ['--quiet']
pcmd.extend(build_pip_dirs_args(tpp_dirs, '--find-links='))
pip_dir_args = list(build_pip_dirs_args(tpp_dirs, root_dir, '--find-links='))
pcmd.extend(pip_dir_args)
req_loc = os.path.join(root_dir, req_file)
pcmd.extend(['-r' , req_loc])
call(pcmd)
call(pcmd, root_dir)


def run_scripts(configs, root_dir, configured_python):
""" Run py_script and sh_script scripts."""
"""
Run Python scripts and shell scripts found in `configs`.
"""
print("* Configuring ...")
# Run Python scripts for each configurations
for py_script in get_conf_files(configs, python_scripts):
for py_script in get_conf_files(configs, root_dir, python_scripts):
cmd = [configured_python, os.path.join(root_dir, py_script)]
call(cmd)
call(cmd, root_dir)

# Run sh_script scripts for each configurations
for sh_script in get_conf_files(configs, shell_scripts):
for sh_script in get_conf_files(configs, root_dir, shell_scripts):
# we source the scripts on posix
cmd = ['.']
if on_win:
cmd = []
cmd = cmd + [os.path.join(root_dir, sh_script)]
call(cmd)
call(cmd, root_dir)


def chmod_bin(directory):
Expand All @@ -244,16 +259,18 @@ def chmod_bin(directory):
os.chmod(os.path.join(path, f), rwx)


def get_conf_files(config_dir_paths, file_names=requirements):
def get_conf_files(config_dir_paths, root_dir, file_names=requirements):
"""
Based on config_dir_paths return a list of collected path-prefixed file
paths matching names in a file_names tuple. Returned paths are posix
paths.
Return a list of collected path-prefixed file paths matching names in a
file_names tuple, based on config_dir_paths, root_dir and the types of
file_names requested. Returned paths are posix paths.
@config_dir_paths: Each config_dir_path is a relative from the project
root to a config dir. This script should always be called from the project
root dir.
@root_dir: The project absolute root dir.
@file_names: get requirements, python or shell files based on list of
supported file names provided as a tuple of supported file_names.
Expand All @@ -267,9 +284,9 @@ def get_conf_files(config_dir_paths, file_names=requirements):
"""
# collect files for each requested dir path
collected = []

for config_dir_path in config_dir_paths:
if not os.path.exists(config_dir_path):
abs_config_dir_path = os.path.join(root_dir, config_dir_path)
if not os.path.exists(abs_config_dir_path):
print('Configuration directory %(config_dir_path)s '
'does not exists. Skipping.' % locals())
continue
Expand All @@ -279,19 +296,18 @@ def get_conf_files(config_dir_paths, file_names=requirements):
current = None
for path in paths:
if not current:
current = (path, os.path.abspath(path),)
current = (path, os.path.join(root_dir, path),)
else:
base_path, base_loc = current
current = (os.path.join(base_path, path),
os.path.join(base_loc, path),)

path, loc = current
# we iterate on known filenames to ensure the defined precedence
# is respected (posix over mac, linux), etc
for n in file_names:
for f in os.listdir(loc):
if f == n:
f_loc = os.path.join(path, f)
f_loc = os.path.join(loc, f)
if f_loc not in collected:
collected.append(f_loc)

Expand All @@ -312,7 +328,7 @@ def get_conf_files(config_dir_paths, file_names=requirements):
bin_dir = os.path.join(root_dir, 'bin')
standard_python = sys.executable

# create a CONFIGURE_QUIET env var to run quietly
# you must create a CONFIGURE_QUIET env var if you want to run quietly
run_quiet = 'CONFIGURE_QUIET' in os.environ

if on_win:
Expand All @@ -323,30 +339,34 @@ def get_conf_files(config_dir_paths, file_names=requirements):
os.makedirs(scripts_dir)
if not os.path.exists(bin_dir):
cmd = ('mklink /J %(bin_dir)s %(scripts_dir)s' % locals()).split()
call(cmd)
call(cmd, root_dir)
else:
configured_python = os.path.join(bin_dir, 'python')
scripts_dir = bin_dir

# get requested configuration paths and install components and run scripts
# Get requested configuration paths to collect components and scripts later
configs = []
for path in args[:]:
if os.path.exists(path):
configs.append(path)
if not os.path.isabs(path):
abs_path = os.path.join(root_dir, path)
if os.path.exists(abs_path):
configs.append(path)
else:
print()
print('WARNING: Skipping missing Configuration directory:\n'
' %(path)s does not exist.' % locals())
print()

# one or more third-party directories may exist
# as environment variables prefixed with TPP_DIR
# Collect vendor directories from environment variables: one or more third-
# party directories may exist as environment variables prefixed with TPP_DIR
thirdparty_dirs = []
for envvar, path in os.environ.items():
if not envvar.startswith('TPP_DIR'):
continue
if os.path.exists(path):
thirdparty_dirs.append(path)
if not os.path.isabs(path):
abs_path = os.path.join(root_dir, path)
if os.path.exists(abs_path):
thirdparty_dirs.append(path)
else:
print()
print('WARNING: Skipping missing Python thirdparty directory:\n'
Expand All @@ -355,9 +375,9 @@ def get_conf_files(config_dir_paths, file_names=requirements):
' set %(envvar)s=%(path)s' % locals())
print()

# Finally execute our three steps: venv, install and scripts
if not os.path.exists(configured_python):
create_virtualenv(standard_python, root_dir, thirdparty_dirs,
quiet=run_quiet)
create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=run_quiet)
activate(root_dir)

install_3pp(configs, root_dir, thirdparty_dirs, quiet=run_quiet)
Expand Down
3 changes: 1 addition & 2 deletions extractcode
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@
# A minimal shell wrapper to the CLI entry point

SCANCODE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$SCANCODE_ROOT_DIR"

SCANCODE_CONFIGURED_PYTHON=$SCANCODE_ROOT_DIR/bin/python
if [ ! -f "$SCANCODE_CONFIGURED_PYTHON" ]; then
echo "* Configuring ScanCode ..."
echo "* Configuring ScanCode for first use..."
CONFIGURE_QUIET=1 $SCANCODE_ROOT_DIR/configure etc/conf
fi

Expand Down
3 changes: 1 addition & 2 deletions extractcode.bat
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
@rem A minimal shell wrapper to the CLI entry point

set SCANCODE_ROOT_DIR=%~dp0
cd %SCANCODE_ROOT_DIR%

set SCANCODE_CMD_LINE_ARGS=
set SCANCODE_CONFIGURED_PYTHON=%SCANCODE_ROOT_DIR%\bin\python.exe
Expand All @@ -25,7 +24,7 @@ if not exist %SCANCODE_CONFIGURED_PYTHON% goto configure
goto scancode

:configure
echo * Configuring ScanCode ...
echo * Configuring ScanCode for first use...
set CONFIGURE_QUIET=1
call %SCANCODE_ROOT_DIR%\configure etc/conf
if %errorlevel% neq 0 (
Expand Down
3 changes: 1 addition & 2 deletions scancode
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,10 @@
# A minimal shell wrapper to the CLI entry point

SCANCODE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$SCANCODE_ROOT_DIR"

SCANCODE_CONFIGURED_PYTHON=$SCANCODE_ROOT_DIR/bin/python
if [ ! -f "$SCANCODE_CONFIGURED_PYTHON" ]; then
echo "* Configuring ScanCode ..."
echo "* Configuring ScanCode for first use..."
CONFIGURE_QUIET=1 $SCANCODE_ROOT_DIR/configure etc/conf
fi

Expand Down
4 changes: 1 addition & 3 deletions scancode.bat
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
@rem A minimal shell wrapper to the CLI entry point

set SCANCODE_ROOT_DIR=%~dp0
cd %SCANCODE_ROOT_DIR%

set SCANCODE_CMD_LINE_ARGS=
set SCANCODE_CONFIGURED_PYTHON=%SCANCODE_ROOT_DIR%\bin\python.exe

Expand All @@ -25,7 +23,7 @@ if not exist %SCANCODE_CONFIGURED_PYTHON% goto configure
goto scancode

:configure
echo * Configuring ScanCode ...
echo * Configuring ScanCode for first use...
set CONFIGURE_QUIET=1
call %SCANCODE_ROOT_DIR%\configure etc/conf
if %errorlevel% neq 0 (
Expand Down

0 comments on commit 1377a78

Please sign in to comment.