Paver

AUTHORS

@@ -104,4 +104,6 @@ Bertrand Marron <bertrand.marron@ionis-group.com>
 Yihua Lou <supermouselyh@hotmail.com>
 Andy Armstrong <andya@edx.org>
 Matt Drayer <mattdrayer@edx.org>
+David Glance <david.glance@gmail.com>
+
 

pavelib/README.md

@@ -0,0 +1,85 @@
+Paver requires the pre-requisite of psutils
+
+run paver --help for a list of commands
+
+run individual commands using:
+
+paver <command_name>
+
+some commands take parameters
+
+
+Commands available:
+
+**install_prereqs** (installs ruby, node and python)
+
+Runs following commands:
+
+**install_ruby_prereqs**
+**install_node_prereqs**
+**install_python_prereqs**
+
+
+**build_docs** (Invoke sphinx 'make build' to generate docs.)
+    "--type=" "Type of docs to compile"
+    "--verbose" "Display verbose output"
+
+**show_docs** (Show docs in browser)
+    "--type=" "Type of docs to compile"
+
+**doc** (Invoke sphinx 'make build' to generate docs and then show in browser)
+    "--type=" "Type of docs to compile"
+    "--verbose" "Display verbose output"
+
+**pre_django**  (Installs requirements and cleans previous python compiled files)
+
+**compile_coffeescript** (Compiles Coffeescript files)
+    "--system=" "System to act on e.g. lms, cms"
+    "--env=" "Environment settings e.g. aws, dev"
+    "--watch" "Run with watch"
+    "--debug" "Run with debug"
+    "--clobber" "Remove compiled Coffeescript files"
+
+**compile_sass** (Compiles Sass files)
+    "--system=" "System to act on e.g. lms, cms"
+    "--env=" "Environment settings e.g. aws, dev"
+    "--watch" "Run with watch"
+    "--debug" "Run with debug"
+
+**compile_xmodule** (Compiles Xmodule)
+    "--system=" "System to act on e.g. lms, cms"
+    "--env=" "Environment settings e.g. aws, dev"
+    "--watch" "Run with watch"
+    "--debug" "Run with debug"
+
+**compile_assets** (Compiles Coffeescript, Sass, Xmodule and optionally runs collectstatic)
+    "--system=" "System to act on e.g. lms, cms"
+    "--env=" "Environment settings e.g. aws, dev"
+    "--watch" "Run with watch"
+    "--debug" "Run with debug"
+    "--collectstatic" "Runs collectstatic
+
+**lms** (runs lms)
+    "--env=" "Environment settings e.g. aws, dev"
+
+**cms** (runs cms)
+    "--env=" "Environment settings e.g. aws, dev"
+
+**run_server** (run a specific server)
+    "--system=" "System to act on e.g. lms, cms"
+    "--env=" "Environment settings e.g. aws, dev"
+
+**resetdb** (runs syncdb and then migrate)
+    "--env=" "Environment settings e.g. aws, dev"
+
+**check_settings** (checks settings files)
+    "--env=" "Environment settings e.g. aws, dev"
+
+**run_all_servers** (runs lms and cms)
+    "--env=" "Environment settings e.g. aws, dev"
+    "--worker_env=" "Environment settings for celery workers"
+    "--logfile=" "File to log output to"
+
+**run_celery** (runs celery for specified system)
+    "--system=" "System to act on e.g. lms, cms"
+    "--env=" "Environment settings e.g. aws, dev"

pavelib/__init__.py

@@ -0,0 +1 @@
+__author__ = 'dglance'

pavelib/assets.py

@@ -0,0 +1,292 @@
+from paver.easy import *
+from paver.setuputils import setup
+from pavelib import prereqs, proc_utils
+from proc_utils import write_stderr
+
+import json
+import glob
+import os
+import platform
+
+setup(
+    name="OpenEdX",
+    packages=['OpenEdX'],
+    version="1.0",
+    url="",
+    author="OpenEdX",
+    author_email=""
+)
+
+
+# Build Constants
+REPO_ROOT = path(__file__).abspath().dirname().dirname()  # /project_dir/edx-platform/
+PROJECT_ROOT = REPO_ROOT.dirname()      # /project_dir
+REPORT_DIR = PROJECT_ROOT / "reports"   # /project_dir/reports
+COMMON_ROOT = PROJECT_ROOT / "common"   # /project_dir/common
+COURSES_ROOT = PROJECT_ROOT / "data"    # /project_dir/data
+
+# Environment constants
+if 'SERVICE_VARIANT' in os.environ:
+    CONFIG_PREFIX = os.environ['SERVICE_VARIANT'] + '.'
+else:
+    CONFIG_PREFIX = ''
+
+ENV_FILE = os.path.join(PROJECT_ROOT, CONFIG_PREFIX + "env.json")
+
+env_data = None
+
+try:
+    with open(ENV_FILE) as env_file:
+        env_data = json.load(env_file)
+except IOError:
+    write_stderr("Warning: File env.json not found - some configuration requires this\n")
+
+USE_CUSTOM_THEME = False
+
+if env_data:
+    if 'FEATURES' in env_data and 'USE_CUSTOM_THEME' in env_data['FEATURES']:
+        USE_CUSTOM_THEME = env_data['FEATURES']['USE_CUSTOM_THEME']
+
+    if USE_CUSTOM_THEME:
+        THEME_NAME = env_data['THEME_NAME']
+        THEME_ROOT = PROJECT_ROOT / "themes" / THEME_NAME
+        THEME_SASS = THEME_ROOT / "static" / "sass"
+
+MINIMAL_DARWIN_NOFILE_LIMIT = 8000
+
+
+def xmodule_cmd(watch=False, debug=False):
+    xmodule = 'xmodule_assets common/static/xmodule'
+
+    if watch:
+        xmodule = ("watchmedo shell-command "
+                   " --patterns='*.js;*.coffee;*.sass;*.scss;*.css' "
+                   " --recursive --command=\'xmodule_assets common/static/xmodule\'"
+                   " --wait common/lib/xmodule"
+                   )
+
+    return xmodule
+
+
+def coffee_clean():
+    files = glob.glob('*/static/coffee/**/*.js')
+
+    for f in files:
+        os.remove(f)
+
+
+def coffee_cmd(watch=False):
+    flags = ["--compile"]
+
+    if watch:
+        flags.append("--watch")
+
+    if platform.system() == "Darwin":
+        precmd = "ulimit -n 8000;"
+    else:
+        precmd = ""
+
+    tpl = "{precmd} {coffee} {flags} {dir}"
+
+    return tpl.format(
+        precmd=precmd,
+        coffee="node_modules/.bin/coffee",
+        flags=" ".join(flags),
+        dir="."
+    )
+
+
+def sass_cmd(watch=False, debug=False):
+    load_paths = ["./common/static/sass"]
+    watch_paths = ["*/static"]
+
+    if USE_CUSTOM_THEME:
+        load_paths.append(THEME_SASS)
+        watch_paths.append(THEME_SASS)
+
+    load_paths = ' '.join(load_paths)
+    watch_paths = ' '.join(watch_paths)
+
+    debug_info = '--debug-info' if debug else '--style=compressed '
+    watch_or_update = '--watch' if watch else '--update'
+
+    cmd = ('sass {debug_info} --load-path={load_paths} {watch_or_update} -E utf-8 {watch_paths}'.format(
+           debug_info=debug_info, load_paths=load_paths, watch_or_update=watch_or_update, watch_paths=watch_paths)
+           )
+
+    return cmd
+
+
+ # This task takes arguments purely to pass them via dependencies to the preprocess task
+@task
+@cmdopts([
+    ("system=", "s", "System to act on"),
+    ("env=", "e", "Environment settings"),
+    ("watch", "w", "Run with watch"),
+    ("debug", "d", "Run with debug"),
+    ("clobber", "c", "Remove compiled Coffeescript files"),
+])
+def compile_coffeescript(options):
+    """
+    Runs coffeescript
+    """
+
+    system = getattr(options, 'system', 'lms')
+    env = getattr(options, 'env', 'dev')
+    run_watch = getattr(options, 'watch', False)
+    run_debug = getattr(options, 'debug', False)
+    clobber = getattr(options, 'clobber', False)
+
+    print ("Compile Coffeescript")
+
+    coffee_clean()
+
+    if clobber:
+        print("Coffeescript files removed")
+        return
+
+    try:
+        sh('python manage.py %s preprocess_assets --settings=%s --traceback ' % (system, env))
+    except:
+        write_stderr("asset preprocessing failed")
+        return
+
+    sh(coffee_cmd(False))
+
+    if run_watch:
+        proc_utils.run_process([coffee_cmd(run_watch)], True)
+
+
+@task
+@cmdopts([
+    ("system=", "s", "System to act on"),
+    ("env=", "e", "Environment settings"),
+    ("watch", "w", "Run with watch"),
+    ("debug", "d", "Run with debug"),
+])
+def compile_xmodule(options):
+    """
+    Runs xmodule_cmd
+    """
+
+    system = getattr(options, 'system', 'lms')
+    env = getattr(options, 'env', 'dev')
+    run_watch = getattr(options, 'watch', False)
+    run_debug = getattr(options, 'debug', False)
+
+    print ("Compile xmodule assets")
+
+    try:
+        sh('python manage.py %s preprocess_assets --settings=%s --traceback ' % (system, env))
+    except:
+        write_stderr("asset preprocessing failed")
+        return
+
+    sh(xmodule_cmd(False, run_debug))
+
+    if run_watch:
+        proc_utils.run_process([xmodule_cmd(run_watch, run_debug)], True)
+
+
+@task
+@cmdopts([
+    ("system=", "s", "System to act on"),
+    ("env=", "e", "Environment settings"),
+    ("watch", "w", "Run with watch"),
+    ("debug", "d", "Run with debug"),
+])
+def compile_sass(options):
+    """
+    Runs sass
+    """
+
+    system = getattr(options, 'system', 'lms')
+    env = getattr(options, 'env', 'dev')
+    run_watch = getattr(options, 'watch', False)
+    run_debug = getattr(options, 'debug', False)
+
+    print ("Compile sass")
+
+    try:
+        sh('python manage.py %s preprocess_assets --settings=%s --traceback ' % (system, env))
+    except:
+        write_stderr("asset preprocessing failed")
+        return
+
+    sh(sass_cmd(False, run_debug))
+
+    if run_watch:
+        proc_utils.run_process([sass_cmd(run_watch, run_debug)], True)
+
+
+@task
+@cmdopts([
+    ("system=", "s", "System to act on"),
+    ("env=", "e", "Environment settings"),
+])
+def collectstatic(options):
+    """
+    Runs collectstatic
+    """
+
+    system = getattr(options, 'system', 'lms')
+    env = getattr(options, 'env', 'dev')
+
+    print ("Run collectstatic")
+
+    try:
+        sh('python manage.py %s preprocess_assets --settings=%s --traceback ' % (system, env))
+    except:
+        write_stderr("asset preprocessing failed")
+        return
+
+    try:
+        sh('python manage.py %s collectstatic --traceback --settings=%s' % (system, env) + ' --noinput > /dev/null')
+    except:
+        pass
+
+
+# This task takes arguments purely to pass them via dependencies to the preprocess task
+@task
+@cmdopts([
+    ("system=", "s", "System to act on"),
+    ("env=", "e", "Environment settings"),
+    ("watch", "w", "Run with watch"),
+    ("debug", "d", "Run with debug"),
+    ("collectstatic", "c", "Collect Static"),
+])
+def compile_assets(options):
+    """
+    Runs coffeescript, sass and xmodule_cmd and then optionally collectstatic
+    """
+
+    system = getattr(options, 'system', 'lms')
+    env = getattr(options, 'env', 'dev')
+    run_watch = getattr(options, 'watch', False)
+    run_debug = getattr(options, 'debug', False)
+    collectstatic = getattr(options, 'collectstatic', False)
+
+    print ("Compile all assets")
+
+    try:
+        sh('python manage.py %s preprocess_assets --settings=%s --traceback ' % (system, env))
+    except:
+        write_stderr("asset preprocessing failed")
+        return
+
+    prereqs.install_prereqs()
+
+    coffee_clean()
+
+    if collectstatic:
+        print("collecting static")
+        sh('python manage.py {system} collectstatic --traceback --settings={env} --noinput > /dev/null'.format(system=system, env=env))
+
+    if run_watch:
+        proc_utils.run_process([coffee_cmd(run_watch),
+                                xmodule_cmd(run_watch, run_debug),
+                                sass_cmd(run_watch, run_debug)], True)
+    else:
+        sh(coffee_cmd(False))
+        sh(xmodule_cmd(False, run_debug))
+        sh(sass_cmd(False, run_debug))