From 3c76637077c3eca0d933c933f3a38f4313295d44 Mon Sep 17 00:00:00 2001 From: Jonathan Bohren Date: Tue, 15 Dec 2015 20:15:53 -0500 Subject: [PATCH] doc: Adding documentation for forthcoming changes in 0.4.0, updating local sphinx style, and reorganizing main index --- docs/advanced/executor_events.dia | 1185 ++++ docs/advanced/executor_events.svg | 475 ++ docs/advanced/executor_job_lifecycle.svg | 319 + docs/advanced/executor_job_resources.svg | 472 ++ docs/advanced/job_executor.rst | 196 + docs/build_types.rst | 92 + docs/catkin_build.dia | 1063 ++++ docs/catkin_tools_execution.dia | 5147 +++++++++++++++++ docs/conf.py | 7 +- docs/development/adding_build_types.rst | 75 + .../extending_the_catkin_command.rst | 3 +- docs/executor_job_lifecycle.dia | 612 ++ docs/executor_job_lifecycle.svg | 766 +++ docs/index.rst | 58 +- docs/installing.rst | 2 +- docs/verbs/catkin_build.rst | 622 +- .../{catkin_find.rst => catkin_locate.rst} | 4 +- 17 files changed, 10748 insertions(+), 350 deletions(-) create mode 100644 docs/advanced/executor_events.dia create mode 100644 docs/advanced/executor_events.svg create mode 100644 docs/advanced/executor_job_lifecycle.svg create mode 100644 docs/advanced/executor_job_resources.svg create mode 100644 docs/advanced/job_executor.rst create mode 100644 docs/build_types.rst create mode 100644 docs/catkin_build.dia create mode 100644 docs/catkin_tools_execution.dia create mode 100644 docs/development/adding_build_types.rst create mode 100644 docs/executor_job_lifecycle.dia create mode 100644 docs/executor_job_lifecycle.svg rename docs/verbs/{catkin_find.rst => catkin_locate.rst} (95%) diff --git a/docs/advanced/executor_events.dia b/docs/advanced/executor_events.dia new file mode 100644 index 00000000..0a500f04 --- /dev/null +++ b/docs/advanced/executor_events.dia @@ -0,0 +1,1185 @@ + + + + + + + + + + + + + #Letter# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Executor# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Event Queue# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Output Controller# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stdout# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stderr# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Command Stages# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stdout# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stderr# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #IO Protocol# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Function Stages# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #IO Logger# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/advanced/executor_events.svg b/docs/advanced/executor_events.svg new file mode 100644 index 00000000..0362d50d --- /dev/null +++ b/docs/advanced/executor_events.svg @@ -0,0 +1,475 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/advanced/executor_job_lifecycle.svg b/docs/advanced/executor_job_lifecycle.svg new file mode 100644 index 00000000..bd797875 --- /dev/null +++ b/docs/advanced/executor_job_lifecycle.svg @@ -0,0 +1,319 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/advanced/executor_job_resources.svg b/docs/advanced/executor_job_resources.svg new file mode 100644 index 00000000..8a3c7ccf --- /dev/null +++ b/docs/advanced/executor_job_resources.svg @@ -0,0 +1,472 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/advanced/job_executor.rst b/docs/advanced/job_executor.rst new file mode 100644 index 00000000..fe384177 --- /dev/null +++ b/docs/advanced/job_executor.rst @@ -0,0 +1,196 @@ +The Catkin Execution Engine +=========================== + +One of the core modules in ``catkin_tools`` is the **job executor**. The +executor performs jobs required to complete a task in a way that maximizes (or +achives a specific) resource utilization subject to job dependency constraints. +The executor is closely integrated with logging and job output capture. This +page details the design and implementation of the executor. + +Execution Model +^^^^^^^^^^^^^^^ + +The execution model is fairly simple. The executor executes a single **task** +for a given command (i.e. ``build``, ``clean``, etc.). A **task** is a set of +**jobs** which are related by an acyclic dependency graph. Each **job** is +given a unique identifier and is composed of a set of dependencies and a +sequence of executable **stages**, which are arbitrary functions or subprocess +calls which utilize one or more **workers** to be executed. The allocation of +workers is managed by the **job server**. Throughout execution, synchronization +with the user-facing interface and output formatting are mediated by a simple +**event queue**. + +The executor is single-threaded and uses an asynchronous loop to execute jobs +as futures. If a job contains blocking stages it can utilize a normal thread +pool for execution, but is still only guaranteed one worker by the main loop of +the executor. See the following section for more information on workers and the +job server. + +The input to the executor is a list of topologically-sorted jobs with no +circular dependencies and some parameters which control the jobserver behavior. +These behavior parameters are explained in detail in the following section. + +Each job is in one of the following lifecycle states at any time: + + - ``PENDING`` Not ready to be executed (dependencies not yet completed) + - ``QUEUED`` Ready to be executed once workers are available + - ``ACTIVE`` Being executed by one or more workers + - ``FINISHED`` Has been executed and either succeded or failed (terminal) + - ``ABANDONED`` Was not built because a prerequisite was not met (terminal) + +.. figure:: executor_job_lifecycle.svg + :scale: 50 % + :alt: Executor Job Lifecycle + + **Executor Job lifecycle** + +All jobs begin in the ``PENDING`` state, and any jobs with unsatisfiable +dependencies are immediately set to ``ABANDONED``, and any jobs without +dependencies are immediately set to ``QUEUED``. After the state initialization, +the executor processes jobs in a main loop until they are in one of the two +terminal states (``FINISHED`` or ``ABANDONED``). + +Each main loop iteration does the following: + + - While job server tokens are available, create futures for ``QUEUED`` jobs + and make them ``ACTIVE`` + - Report status of all jobs to the event queue + - Retrieve ``ACTIVE`` job futures which have completed and set them + ``FINISHED`` + - Check for any ``PENDING`` jobs which need to be ``ABANDONED`` due to failed + jobs + - Change all ``PENDING`` jobs whose dependencies are satisifed to ``QUEUED`` + +Once each job is in one of terminal states, the executor pushes a final status +event and returns. + + +Job Server Resource Model +^^^^^^^^^^^^^^^^^^^^^^^^^ + +As mentioned in the previous section, each task includes a set of jobs which +are activated by the **job server**. In order to start a queued job, at least +one worker needs to be available. Once a job is started, it is assigned a +single worker from the job server. These are considered **top-level jobs** +since they are managed directly by the catkin executor. The number of top-level +jobs can be configured for a given task. + +Additionally to top-level paralellism, some job stages are capable of running +in parallel, themselves. In such cases, the job server can interface directly +with the underlying stage's low-level job allocation. This enables multi-level +parallelism without allocating more than a fixed number of jobs. + + +.. figure:: executor_job_resources.svg + :scale: 50 % + :alt: Executor job resources + + **Executor Job Flow and Resource Utilization** -- In this snapshot of the job pipeline, the executor is executing four of six possible top-level jobs, each with three stages, and using sevel of eight total workers. Two jobs are executing subprocesses, which have side-channel communication with the job server. + +One such parallel-capable stage is the GNU Make build stage. In this case, the +job server implements a GNU Make job server interface, which involves reading +and writing tokens from file handles passed as build flags to the Make command. + +For top-level jobs, additional resources are monitored in addition to the +number of workers. Both system load and memory utilization checks can be +enabled to prevent overloading a system. + +Executor Job Failure Behavior +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The executor's behavior when a job fails can be modified with the following two +parameters: + + - ``continue_on_failure`` Continue executing jobs even if one job fails. If + this is set to ``false`` (the default), it will cause the executor to + abandon all pending and queued jobs and stop after the first failure. Note + that active jobs will still be allowed to complete before the executor + returns. + - ``continue_without_deps`` Continue executing jobs even if one + or more of their dependencies have failed. If this is set to ``false`` (the + default), it will cause the executor to abandon only the jobs which depend + on the failed job. If it is set to ``true``, then it will build dependent + jobs regardless. + + +Jobs and Job Stages +^^^^^^^^^^^^^^^^^^^ + +As mentioned above, a **job** is a set of dependencies and a sequence of **job +stages**. Jobs and stages are constructed before a given task starts executing, +and hold only specificaitons of what needs to be done to complete them. All +stages are given a label for user introspection, a logger interface, and can +either require or not require allocation of a worker from the job server. + +Stage execution is performed asynchronously by Python's ``asyncio`` module. +This means that exceptions thrown in job stages are handled directly by the +executor. It also means job stages can be interrupted easily through Python's +normal signal handling mechanism. + +Stages can either be **command stages** (subprocess commands) or **function +stages** (python functions). In either case, loggers used by stages support +segmentation of ``stdout`` and ``stderr`` from job stages for both real-time +introspection and logging. + + +Command Stages +~~~~~~~~~~~~~~~ + +In addition to the basic arguments mentioned above, command stages are +paramterized by the standard subprocess command arguments including the +following: + + - The command, itself, and its arguments, + - The working directory for the command, + - Any additional environment variables, + - Whether to use a shell interpreter + - Whether to emulate a TTY + - Whether to partition ``stdout`` and ``stderr`` + +When executed, command stages use ``asncio``'s asynchronous process executor +with a custom I/O protocol. + +Function Stages +~~~~~~~~~~~~~~~ + +In addition to the basic arguments mentioned above, function stages are +parameterized by a function handle and a set of function-specific Python +arguments and keyword arguments. When executed, they use the thread pool +mentioned above. + +Since the function stages aren't subprocesses, I/O isn't piped or redirected. +Instead, a custom I/O logger is passed to the function for output. Functions +used as function stages should use this logger to write to ``stdout`` and +``stderr`` instead of using normal system calls. + +Introspection via Executor Events +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Introspection into the different asynchronously-executed components of a task +is performed by a simple event queue. Events are created by the executor, +loggers, and stages, and they are consumed by an output controller. Events are +defined by an event identifier and a data payload, which is an arbitrary +dictionary. + +There are numerous events which correspond to changes in job states, but events are also used for transporting captured I/O from job stages. + +.. figure:: executor_events.svg + :scale: 50 % + :alt: Executor Event Pipeline + + **Executor Event Pipeline** -- Above, the executor writes events to the event queue, and the I/O loggers used by function and command stages write output events as well. All of these events are handled by the output controller, which writes to the real ``stdout`` and ``stderr``. + +The modeled events include the following: + + - ``JOB_STATUS`` A report of running job states, + - ``QUEUED_JOB`` A job has been queued to be executed, + - ``STARTED_JOB`` A job has started to be executed, + - ``FINISHED_JOB`` A job has finished executing (succeeded or failed), + - ``ABANDONED_JOB`` A job has been abandoned for some reason, + - ``STARTED_STAGE`` A job stage has started to be executed, + - ``FINISHED_STAGE`` A job stage has finished executing (succeeded or failed), + - ``STAGE_PROGRESS`` A job stage has executed partially, + - ``STDOUT`` A status message from a job, + - ``STDERR`` A warning or error message from a job, + - ``SUBPROCESS`` A subprocess has been created, + - ``MESSAGE`` Arbitrary string message diff --git a/docs/build_types.rst b/docs/build_types.rst new file mode 100644 index 00000000..51d55f77 --- /dev/null +++ b/docs/build_types.rst @@ -0,0 +1,92 @@ +Supported Build Types +===================== + +The current release of ``catkin_tools`` supports building two types of packages: + + - **Catkin** -- CMake packages that use the Catkin CMake macros + - **CMake** -- "Plain" CMake packages + +There is currently limited support for adding other build types. For information +on extending ``catkin_tools`` to be able to build other types of packages, see +:doc:`Adding New Build Types `. Below are +details on the stages involved in building a given package for each of +the currently-supported build types. + +Catkin +^^^^^^ + +Catkin packages are CMake packages which utilize the Catkin CMake macros for +finding packages and defining configuration files. + +Configuration Arguments +----------------------- + + - ``--cmake-args`` + - ``--make-args`` + - ``--catkin-make-args`` + +Build Stages +------------ + +============== ============ ================================================== + First Subsequent Description +============== ============ ================================================== + ``mkdir`` | Create package build space if it doesn't exist. +---------------------------- -------------------------------------------------- + ``envgen`` | Generate environment setup file for building. +---------------------------- -------------------------------------------------- + ``cmake`` ``check`` | Run CMake configure step **once** for the + | first build and the ``cmake_check_build_system`` + | target for subsequent builds unless the + | ``--force-cmake`` argument is given. +-------------- ------------ -------------------------------------------------- + ``preclean`` `optional` | Run the ``clean`` target before building. + | This is only done with the ``--pre-clean`` \ + option. +---------------------------- -------------------------------------------------- + ``make`` | Build the default target with GNU make. +---------------------------- -------------------------------------------------- + ``install`` `optional` | Run the ``install`` target after building. + | This is only done with the ``--install`` option. +============================ ================================================== + +CMake +^^^^^ + +Configuration Arguments +----------------------- + + - ``--cmake-args`` + - ``--make-args`` + +Build Stages +------------ + +============== ============ ================================================== + First Subsequent Description +============== ============ ================================================== + ``mkdir`` | Create package build space if it doesn't exist. +---------------------------- -------------------------------------------------- + ``envgen`` | Generate environment setup file for building. +---------------------------- -------------------------------------------------- + ``cmake`` ``check`` | Run CMake configure step **once** for the + | first build and the ``cmake_check_build_system`` + | target for subsequent builds unless the + | ``--force-cmake`` argument is given. +-------------- ------------ -------------------------------------------------- + ``preclean`` `optional` | Run the ``clean`` target before building. + | This is only done with the ``--pre-clean`` \ + option. +---------------------------- -------------------------------------------------- + ``make`` | Build the default target with GNU make. +---------------------------- -------------------------------------------------- + ``install`` | Run the ``install`` target after building, + | and install products to the **devel space**. + | If the ``--install`` option is given, + | products are installed to the \ + **install space** instead. +---------------------------- -------------------------------------------------- + ``setupgen`` | Generate a ``setup.sh`` file if necessary. +============================ ================================================== + + diff --git a/docs/catkin_build.dia b/docs/catkin_build.dia new file mode 100644 index 00000000..a5ae26f8 --- /dev/null +++ b/docs/catkin_build.dia @@ -0,0 +1,1063 @@ + + + + + + + + + + + + + #Letter# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #catkin build# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #CMake# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #CMake# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #...# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #CMake# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #GNU Make# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #GNU Make# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #GNU Make# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_a# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_b# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_c# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_d# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_x# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_y# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #pkg_z# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/catkin_tools_execution.dia b/docs/catkin_tools_execution.dia new file mode 100644 index 00000000..3b392861 --- /dev/null +++ b/docs/catkin_tools_execution.dia @@ -0,0 +1,5147 @@ + + + + + + + + + + + + + #Letter# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stage 1# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stage m# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #job 1# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #task# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #JobServer# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stage 1# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #stage m# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #job n# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #.# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #PENDING# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #QUEUED# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #ACTIVE# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #COMPLETED# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #ABANDONED# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Dependencies +Completed# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Worker +Available# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Success +or Failure# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Dependency Failed or +Peer Failed (CoF disabled)# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #...# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #...# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #...# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #...# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Pending Job Set# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Queued Job Queue# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Active Job Set (4/6 High-Level Jobs)# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Finished Job Set# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Abandoned Job Set# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Job Server (7/8 Low-Level Workers)# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Subprocess to +Job Server +Interfaces# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/conf.py b/docs/conf.py index 7fa36629..20a4795f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -104,9 +104,14 @@ # -- Options for HTML output ---------------------------------------------- +import sphinx_rtd_theme +html_theme = "sphinx_rtd_theme" +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + + # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +#html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/development/adding_build_types.rst b/docs/development/adding_build_types.rst new file mode 100644 index 00000000..5d0b8bcd --- /dev/null +++ b/docs/development/adding_build_types.rst @@ -0,0 +1,75 @@ +Adding New Build Types +====================== + +The current release of ``catkin_tools`` supports building two types of packages: + + - **Catkin** -- CMake packages that use the Catkin CMake macros + - **CMake** -- "Plain" CMake packages + +In order to fully support additional build types, numerous additions need to be +made to the command-line interfaces so that the necessary parameters can be passed +to the ``build`` verb. For partial support, however, all that's needded is to +add a build type identifier and a function for generating build jobs. + +The supported build typs are easily extendable using the ``setuptools`` +``entry_points`` interface without modifying the ``catkin_tools`` project, +itself. Regardless of what package the ``entry_point`` is defined in, it will +be defined in the ``setup.py`` of that package, and will take this form: + +.. code-block:: python + + from setuptools import setup + + setup( + ... + entry_points={ + ... + 'catkin_tools.jobs': [ + 'mybuild = my_package.some.module:description', + ], + }, + ) + +This entry in the ``setup.py`` places a file in the ``PYTHONPATH`` when either +the ``install`` or the ``develop`` verb is given to ``setup.py``. This file +relates the key (in this case ``mybuild``) to a module and attribute (in this +case ``my_package.some.module`` and ``description``). + +Then the ``catkin`` command will use the ``pkg_resources`` modules to retrieve +these mapping at run time. Any entry for the ``catkin_tools.jobs`` group must +point to a ``description`` attribute of a module, where the ``description`` +attribute is a ``dict``. The ``description`` ``dict`` should take this form: + +.. code-block:: python + + description = dict( + build_type='mybuild', + description="Builds a package with the 'mybuild' build type", + create_build_job=create_mybuild_build_job + ) + +This ``dict`` defines all the information that the ``catkin`` command needs to +create jobs for the ``mybuild`` build type. The ``build_type`` key takes a +string which is the build type identifier. The ``description`` key takes a +string which briefly describes the build type. The ``create_build_job`` key +takes a callable (function) factory which is called in order to create a +``Job`` to build a package of type ``mybuild``. + +The signature of the factory callable should be similar to the following: + +.. code-block:: python + + def create_mybuild_build_job(context, package, package_path, dependencies, **kwargs): + # Initialize empty list of build stages + stages = [] + + # Add stages required to build ``mybuild``-type packages, + # based on the configuration context. + # ... + + # Create and return new build Job + return Job( + jid=package.name, + deps=dependencies, + stages=stages) + diff --git a/docs/development/extending_the_catkin_command.rst b/docs/development/extending_the_catkin_command.rst index 921efc4a..ada43d31 100644 --- a/docs/development/extending_the_catkin_command.rst +++ b/docs/development/extending_the_catkin_command.rst @@ -1,8 +1,7 @@ Extending the ``catkin`` command ================================ -The ``catkin`` command is setup to be easily extended using the ``setuptools`` notion of ``entry_points``, see: http://guide.python-distribute.org/creation.html#entry-points -By using the ``entry_points`` extensions to the ``catkin`` command can be made within the ``catkin_tools`` package or an external package. +The ``catkin`` command is designed to be easily extendable using the ``setuptools`` ``entry_points`` interface without modifying the ``catkin_tools`` project, itself. Regardless of what package the ``entry_point`` is defined in, it will be defined in the ``setup.py`` of that package, and will take this form: .. code-block:: python diff --git a/docs/executor_job_lifecycle.dia b/docs/executor_job_lifecycle.dia new file mode 100644 index 00000000..7f523dd2 --- /dev/null +++ b/docs/executor_job_lifecycle.dia @@ -0,0 +1,612 @@ + + + + + + + + + + + + + #Letter# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #PENDING# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #QUEUED# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #ACTIVE# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #COMPLETED# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #ABANDONED# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Dependencies +Completed# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Worker +Available# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Success +or Failure# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #Dependency Failed or +Peer Failed (CoF disabled)# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/executor_job_lifecycle.svg b/docs/executor_job_lifecycle.svg new file mode 100644 index 00000000..a6ee6366 --- /dev/null +++ b/docs/executor_job_lifecycle.svg @@ -0,0 +1,766 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/index.rst b/docs/index.rst index 3699dca4..4e16ecf5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,6 +2,9 @@ Catkin Command Line Tools ========================= .. toctree:: + :name: tocmain + :caption: Main Overview + :glob: :hidden: :maxdepth: 2 @@ -10,18 +13,46 @@ Catkin Command Line Tools mechanics config_summary cheat_sheet + build_types Troubleshooting - verbs/catkin_build - verbs/catkin_clean - verbs/catkin_config - verbs/catkin_create - verbs/catkin_init - verbs/catkin_list - verbs/catkin_locate - verbs/catkin_profile - Advanced: Verb Aliasing - Advanced: Contributing Verbs -.. TODO: Advanced: Workspace Chaining + +.. toctree:: + :name: tocverbs + :caption: Verb Details + :glob: + :hidden: + :maxdepth: 2 + + verbs/* + +.. toctree:: + :name: tocadv + :caption: Advanced Usage + :glob: + :hidden: + :maxdepth: 2 + + Verb Aliasing +.. Workspace Chaining + +.. toctree:: + :name: tocdes + :caption: Design + :glob: + :hidden: + :maxdepth: 2 + + Execution Engine + +.. toctree:: + :name: toccontrib + :caption: Contributing + :glob: + :hidden: + :maxdepth: 2 + + development/adding_build_types + Adding New Verbs This Python package provides command line tools for working with the catkin meta-buildsystem and catkin workspaces. @@ -34,6 +65,10 @@ This Python package provides command line tools for working with the catkin meta The ``catkin`` command ^^^^^^^^^^^^^^^^^^^^^^ +.. .. raw:: html +.. +.. + The ``catkin`` Command-Line Interface (CLI) tool is the single point of entry for most of the functionality provided by this package. All invocations of the ``catkin`` CLI tool take this form: @@ -65,6 +100,7 @@ Each of the following verbs is built-in to the ``catkin`` command and has its ow - :doc:`create -- Create structrures like Catkin packages ` - :doc:`init -- Initialize a catkin workspace ` - :doc:`list -- Find and list information about catkin packages in a workspace ` +- :doc:`locate -- Get important workspace directory paths ` - :doc:`profile -- Manage different named configuration profiles ` Extending the ``catkin`` command diff --git a/docs/installing.rst b/docs/installing.rst index d3ccb803..2114f549 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -50,7 +50,7 @@ Then install with the ``setup.py`` file: .. code-block:: bash $ python setup.py install - + Note: Depending on your environment/machine, you may need to use ``sudo`` with this command. Developing diff --git a/docs/verbs/catkin_build.rst b/docs/verbs/catkin_build.rst index c188b53c..1e276e6f 100644 --- a/docs/verbs/catkin_build.rst +++ b/docs/verbs/catkin_build.rst @@ -1,10 +1,10 @@ ``catkin build`` -- Build Packages ================================== -The ``build`` verb for the ``catkin`` command is used to build one or more packages in a catkin workspace. -Like most ``catkin`` verbs, the ``catkin build`` verb is context-aware. This means that it can be executed from within any directory contained by an *initialized* workspace. +The ``build`` verb is used to build one or more packages in a catkin workspace. +Like most verbs, ``build`` is context-aware. This means that it can be executed from within any directory contained by an *initialized* workspace. -If a workspace is not yet initialized, ``catkin build`` can initialize it, but only if it is called from the workspace root and the default workspace structure is desired. +If a workspace is not yet initialized, ``build`` can initialize it, but only if it is called from the workspace root and the default workspace structure is desired. Workspaces can also be built from arbitrary working directories if the user specifies the path to the workspace with the ``--workspace`` option. @@ -26,26 +26,12 @@ Workspaces can also be built from arbitrary working directories if the user spec Basic Usage ^^^^^^^^^^^ -Consider a Catkin workspace with a **source space** populated with the -following Catkin packages which have yet to be built: - -.. code-block:: bash - - $ pwd - /tmp/path/to/my_catkin_ws - - $ ls ./* - ./src: - catkin console_bridge genlisp genpy - message_runtime ros_comm roscpp_core std_msgs - common_msgs gencpp genmsg message_generation - ros ros_tutorials rospack Previewing The Build -------------------- -Before actually building anything in the workspace, it is useful to preview which -packages ``catkin build`` will build, and in what order. This can be done with the +Before actually building anything in the workspace, it is useful to preview +which packages will be built and in what order. This can be done with the ``--dry-run`` option: .. code-block:: bash @@ -123,118 +109,11 @@ above are: * **metapackage** -- A package which contains no build products, but just groups other packages together for distribution -Building Specific Packages --------------------------- - -In addition to the usage above, the ``--dry-run`` option will show what the -behavior of ``catkin build`` will be with various other options. -For example, the following will happen when you specify a single package to -build: - -.. code-block:: bash - - $ catkin build roscpp_tutorials --dry-run - .... - Found '36' packages in 0.1 seconds. - Packages to be built: - - catkin (catkin) - - genmsg (catkin) - - gencpp (catkin) - - genlisp (catkin) - - genpy (catkin) - - console_bridge (cmake) - - cpp_common (catkin) - - message_generation (catkin) - - message_runtime (catkin) - - rosbuild (catkin) - - roscpp_traits (catkin) - - roslang (catkin) - - rospack (catkin) - - roslib (catkin) - - rostime (catkin) - - roscpp_serialization (catkin) - - rosunit (catkin) - - rosconsole (catkin) - - std_msgs (catkin) - - rosgraph_msgs (catkin) - - xmlrpcpp (catkin) - - roscpp (catkin) - - roscpp_tutorials (catkin) - Total packages: 23 - -As shown above, only 23 packages (``roscpp_tutorials`` and its dependencies), -of the total 36 packages would be built. - -Skipping Packages ------------------ - -Suppose you built every package up to ``roscpp_tutorials``, but that package -had a build error. -After fixing the error, you could run the same build command again, but the -``build`` verb provides an option to save time in this situation. -If re-started from the beginning, none of the products of the dependencies of -``roscpp_tutorials`` would be re-built, but it would still take some time for -the underlying byuildsystem to verify that for each package. - -Those checks could be skipped, however, by jumping directly to a given package. -You could use the ``--start-with`` option to continue the build where you left -off after fixing the problem. (The following example uses the ``--dry-run`` -option again to preview the behavior): - -.. code-block:: bash - - $ catkin build roscpp_tutorials --start-with roscpp_tutorials --dry-run - .... - Found '36' packages in 0.0 seconds. - Packages to be built: - (skip) catkin (catkin) - (skip) genmsg (catkin) - (skip) gencpp (catkin) - (skip) genlisp (catkin) - (skip) genpy (catkin) - (skip) console_bridge (cmake) - (skip) cpp_common (catkin) - (skip) message_generation (catkin) - (skip) message_runtime (catkin) - (skip) rosbuild (catkin) - (skip) roscpp_traits (catkin) - (skip) roslang (catkin) - (skip) rospack (catkin) - (skip) roslib (catkin) - (skip) rostime (catkin) - (skip) roscpp_serialization (catkin) - (skip) rosunit (catkin) - (skip) rosconsole (catkin) - (skip) std_msgs (catkin) - (skip) rosgraph_msgs (catkin) - (skip) xmlrpcpp (catkin) - (skip) roscpp (catkin) - ------ roscpp_tutorials (catkin) - Total packages: 23 - -However, you should be careful when using the ``--start-with`` option, as -``catkin build`` will assume that all dependencies leading up to that package -have already been successfully built. - -If you're only interested in building a *single* package in a workspace, you -can also use the ``--no-deps`` option along with a package name. This will -skip all of the package's dependencies, build the given package, and then exit. - -.. code-block:: bash - - $ catkin build roscpp_tutorials --no-deps roscpp_tutorials --dry-run - .... - Found '36' packages in 0.0 seconds. - Packages to be built: - - roscpp_tutorials (catkin) - Total packages: 1 - -Build Products --------------- +Building a Workspace +-------------------- -At this point the workspace has not been modified, but once we tell the -``build`` verb to actually build the workspace then directories for a **build -space** and a **devel space** will be created: +When no packages are given as arguments, ``catkin build`` builds the entire workspace. +It automatically creates directories for a **build space** and a **devel space**: .. code-block:: bash @@ -273,13 +152,7 @@ space** and a **devel space** will be created: [build] Finished. [build] Runtime: 3 minutes and 54.6 seconds -Since no packages were given as arguments, ``catkin build`` built all of -the packages in the workspace. - -As shown above, after the build finishes, we now have a **build space** with a -folder containing the intermediate build products for each package and a -**devel space** with an FHS layout into which all the final build products have -been written. +After the build finishes, the **build space** contains directories containing the intermediate build products for each package, and the **devel space** contains an FHS layout into which all the final build products are written. .. code-block:: bash @@ -314,82 +187,40 @@ been written. ``catkin_make_isolated`` which would have an isolated FHS directory for each package in the **devel space**. -Context-Aware Building -^^^^^^^^^^^^^^^^^^^^^^ - -In addition to building all packages or specified packages with various dependency requirements, -``catkin build`` can also determine the package containing the current working directory. This -is equivalent to specifying the name of the package on the command line, and is -done by passing the ``--this`` option to ``catkin build`` like the following: - -.. code-block:: bash - - $ cd /tmp/path/to/my_catkin_ws/src/roscpp_tutorials - $ catkin build --this --dry-run - .... - Found '36' packages in 0.0 seconds. - Packages to be built: - - roscpp_tutorials (catkin) - Total packages: 1 - -Controlling the Number of Build Jobs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default ``catkin build`` on a computer with ``N`` cores will build up to -``N`` packages in parallel and will distribute ``N`` ``make`` jobs among them -using an internal jobserver. If your platform doesn't support jobserver -scheduling, ``catkin build`` will pass ``-jN -lN`` to ``make`` for each package. - -You can control the maximum number of packages allowed to build in parallel by -using the ``-p`` or ``--parallel-packages`` option and you can change the -number of ``make`` jobs available with the ``-j`` or ``--jobs`` option. - -By default, these jobs options aren't passed to the underlying ``make`` -command. To disable the jobserver, you can use the ``--no-jobserver`` option, and -you can pass flags directly to ``make`` with the ``--make-args`` option. - -.. note:: - - Jobs flags (``-jN`` and/or ``-lN``) can be passed directly to ``make`` by - giving them to ``catkin build``, but other ``make`` arguments need to be - passed to the ``--make-args`` option. - -Controlling Command-Line Output -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Status Line ----------- -While running ``catkin build`` with default options, you would have seen the -"live" status lines similar to the following: +When running ``catkin build`` with default options, it displays a "live" status line similar to the following: .. code-block:: none - [build - 5.9] [genmsg - 1.3] [message_runtime - 0.7] ... [4/4 Active | 3/36 Completed] - -This status line stays at the bottom of the screen and displays the continuously-updated progress -of the entire build as well as the active build jobs which are still running. It is composed -of the following information: - - * **Total Build Time** -- The first block on the left, indicates the total - elapsed build time in seconds thus far. Above, ``[build - 5.9]`` means that - the build has been running for a total of ``5.9`` seconds. - * **Active Job Status** -- The next blocks show the currently active jobs with as - name of the package being built and the elapsed time for that job, in - seconds. The above block like ``[genmsg - 1.3]`` means that the ``genmsg`` - package is currently being built, and it has been building for ``1.3`` - seconds. - * **Active and Completed Counts** -- The final block, justified to the right, - is the number of packages being actively built out of the total allowed - parallel jobs (specified with the ``-p`` options) as well as the number of - completed packages out of the total. Above, the block ``[4/4 Active | 3/36 - Completed]`` means that there are four out of four jobs active and three of - the total 36 packages to be built have been completed. - -This status line can be disabled by passing the ``--no-status`` option to ``catkin build``. - -Package Build Messages ----------------------- + [build - 20.2] [18/34 complete] [4/4 jobs] [1 queued] [xmlrpcpp:make (66%) - 4.9] ... + +The status line stays at the bottom of the screen and displays the continuously-updated progress of the entire build as well as the active build jobs which are still running. It is composed of the following information: + + * ``[build - ]`` -- The first block on the left indicates the total elapsed + build time ```` in seconds thus far. + * ``[/ complete]`` -- The second block from the left indicates the + build progress in terms of the number of completed packages, ```` out of + the total number of packages to be built ````. + * ``[/ jobs]`` -- The third block from the left indicates the number of + active total low-level jobs ```` out of the total number of low-level + workers ````. + * ``[ queued]`` -- The fourth block from the left indicates the number of + jobs ```` whose dependencies have already been satisfied and are ready to + be built. + * ``[ failed]`` -- The fifth block from the left indicates the number of + jobs ```` which have failed. This block only appears once one or more + jobs has failed. + * ``[: (

%) - ]`` -- The remaining blocks show details on + the active jobs. These include the percent complete, ``

``, of the stage, + if available, as well as the time elapsed building the package, ````. + +When necessary, the status line can be disabled by passing the ``--no-status`` option to ``catkin build``. +This is sometimes required when running ``catkin build`` from within a program that doesn't support the ASCII escape sequences required to reset and re-write the status line. + +Console Messages +---------------- Normally, unless an error occurs, the output from each package's build proces is collected but not printed to the console. All that is printed is a pair of @@ -398,91 +229,254 @@ like the following for the ``genmsg`` package: .. code-block:: none - Starting ==> genmsg - Finished <== genmsg [ 2.4 seconds ] + ... + Starting >>> genmsg + ... + Finished <<< genmsg [ 0.1 seconds ] + ... -However, if you would like to see more of the messages from the underlying -buildsystem, you can invoke the ``-v`` or ``--verbose`` option. -This will print the normal message when a package build starts and finished as -well as the output of each build command in a block, once it finishes: +Error messages are printed whenever a build job writes to ``stderr``. +In such cases, the ``build`` verb will automatically print the captured ``stderr`` buffer under a ``Warnings`` header once the job has completed, similarly to below: .. code-block:: none - Starting ==> catkin + ... + ____________________________________________________________________________ + Warnings << rospack:make /path/to/my_catkin_ws/build/_logs/rospack/build.make.000.log + In file included from /usr/include/python2.7/Python.h:8:0, + from /path/to/my_catkin_ws/src/rospack/src/rospack.cpp:71: + /usr/include/python2.7/pyconfig.h:1161:0: warning: "_POSIX_C_SOURCE" redefined [enabled by default] + /usr/include/features.h:164:0: note: this is the location of the previous definition + /usr/include/python2.7/pyconfig.h:1183:0: warning: "_XOPEN_SOURCE" redefined [enabled by default] + /usr/include/features.h:166:0: note: this is the location of the previous definition + ............................................................................ + Finished <<< rospack [ 11.7 seconds ] + ... + +Note that the first line displays the path to the interleaved log file, which persists until the build space is cleaned. +Additionally, if a package fails, the output to ``stderr`` is printed under the ``Errors`` header. + +.. code-block:: none + + ____________________________________________________________________________ + Errors << catkin_pkg_make_err:make /home/jbohren/ws/catkin_tools_test_ws/build/_logs/catkin_pkg_make_err/build.make.062.log + /home/jbohren/ws/catkin_tools_test_ws/src/catkin_pkg_make_err/main.cpp: In function ‘int main(int, char**)’: + /home/jbohren/ws/catkin_tools_test_ws/src/catkin_pkg_make_err/main.cpp:3:3: error: expected ‘,’ or ‘;’ before ‘return’ + make[2]: *** [CMakeFiles/main.dir/main.cpp.o] Error 1 + make[1]: *** [CMakeFiles/main.dir/all] Error 2 + make: *** [all] Error 2 + ............................................................................ + Failed << catkin_pkg_make_err:make [ Exited with code 2 ] + +All of the messages from the underlying jobs can be shown when using the +``-v`` or ``--verbose`` option. This will print the normal messages when a +build job starts and finishes as well as the interleaved output to ``stdout`` +and ``stderr`` from each build command in a block. - [catkin]: ==> '/path/to/my_catkin_ws/build/catkin/build_env.sh /usr/local/bin/cmake /path/to/my_catkin_ws/src/catkin -DCATKIN_DEVEL_PREFIX=/path/to/my_catkin_ws/devel/catkin -DCMAKE_INSTALL_PREFIX=/path/to/my_catkin_ws/install' in '/path/to/my_catkin_ws/build/catkin' - -- The C compiler identification is Clang 5.0.0 - -- The CXX compiler identification is Clang 5.0.0 - -- Check for working C compiler: /usr/bin/cc - -- Check for working C compiler: /usr/bin/cc -- works +.. code-block:: none + + Starting >>> genmsg + Starting >> genmsg:mkdir + Finished << genmsg:mkdir + Starting >> genmsg:envgen + Finished << genmsg:envgen + Starting >> genmsg:cmake + Subprocess > genmsg:cmake `cd /path/to/my_catkin_ws/build/genmsg && /path/to/my_catkin_ws/build/genmsg/build_env.sh /usr/bin/cmake /path/to/my_catkin_ws/src/genmsg --no-warn-unused-cli -DCATKIN_DEVEL_PREFIX=/path/to/my_catkin_ws/devel -DCMAKE_INSTALL_PREFIX=/path/to/my_catkin_ws/install` + Output << genmsg:cmake /path/to/my_catkin_ws/build/_logs/genmsg/build.cmake.000.log + Not searching for unused variables given on the command line. + Re-run cmake no build system arguments + -- The C compiler identification is GNU + -- The CXX compiler identification is GNU + -- Check for working C compiler: /usr/bin/gcc + -- Check for working C compiler: /usr/bin/gcc -- works -- Detecting C compiler ABI info -- Detecting C compiler ABI info - done -- Check for working CXX compiler: /usr/bin/c++ -- Check for working CXX compiler: /usr/bin/c++ -- works -- Detecting CXX compiler ABI info -- Detecting CXX compiler ABI info - done - -- Using CATKIN_DEVEL_PREFIX: /path/to/my_catkin_ws/devel/catkin - -- Using CMAKE_PREFIX_PATH: /path/to/my_catkin_ws/install - -- This workspace overlays: /path/to/my_catkin_ws/install - -- Found PythonInterp: /usr/bin/python (found version "2.7.5") + -- Using CATKIN_DEVEL_PREFIX: /path/to/my_catkin_ws/devel + -- Using CMAKE_PREFIX_PATH: /path/to/my_catkin_ws/smach/devel;/opt/ros/hydro + -- This workspace overlays: /path/to/my_catkin_ws/smach/devel;/opt/ros/hydro + -- Found PythonInterp: /usr/bin/python (found version "2.7.3") -- Using PYTHON_EXECUTABLE: /usr/bin/python -- Python version: 2.7 - -- Using default Python package layout - -- Found PY_em: /Library/Python/2.7/site-packages/em.pyc + -- Using Debian Python package layout -- Using CATKIN_ENABLE_TESTING: ON -- Call enable_testing() - -- Using CATKIN_TEST_RESULTS_DIR: /path/to/my_catkin_ws/build/catkin/test_results - -- Found gtest: gtests will be built - -- catkin 0.5.86 + -- Using CATKIN_TEST_RESULTS_DIR: /path/to/my_catkin_ws/build/genmsg/test_results + -- Looking for include files CMAKE_HAVE_PTHREAD_H + -- Looking for include files CMAKE_HAVE_PTHREAD_H - found + -- Looking for pthread_create in pthreads + -- Looking for pthread_create in pthreads - not found + -- Looking for pthread_create in pthread + -- Looking for pthread_create in pthread - found + -- Found Threads: TRUE + -- Found gtest sources under '/usr/src/gtest': gtests will be built + -- catkin 0.5.90 -- Configuring done -- Generating done - -- Build files have been written to: /path/to/my_catkin_ws/build/catkin - [catkin]: <== '/path/to/my_catkin_ws/build/catkin/build_env.sh /usr/local/bin/cmake /path/to/my_catkin_ws/src/catkin -DCATKIN_DEVEL_PREFIX=/path/to/my_catkin_ws/devel/catkin -DCMAKE_INSTALL_PREFIX=/path/to/my_catkin_ws/install' finished with return code '0' + -- Build files have been written to: /path/to/my_catkin_ws/build/genmsg + Finished << genmsg:cmake + Starting >> genmsg:make + Subprocess > genmsg:make `cd /path/to/my_catkin_ws/build/genmsg && /usr/bin/make --jobserver-fds=3,4 -j` + Output << genmsg:make /path/to/my_catkin_ws/build/_logs/genmsg/build.make.000.log + /usr/bin/cmake -H/path/to/my_catkin_ws/src/genmsg -B/path/to/my_catkin_ws/build/genmsg --check-build-system CMakeFiles/Makefile.cmake 0 + /usr/bin/cmake -E cmake_progress_start /path/to/my_catkin_ws/build/genmsg/CMakeFiles /path/to/my_catkin_ws/build/genmsg/CMakeFiles/progress.marks + /usr/bin/make -f CMakeFiles/Makefile2 all + make[1]: Entering directory `/path/to/my_catkin_ws/build/genmsg' + make[1]: Nothing to be done for `all'. + make[1]: Leaving directory `/path/to/my_catkin_ws/build/genmsg' + /usr/bin/cmake -E cmake_progress_start /path/to/my_catkin_ws/build/genmsg/CMakeFiles 0 + Finished << genmsg:make + Finished <<< genmsg [ 2.0 seconds ] + +Build Summary +------------- + +At the end of each build, a brief build summary is printed to guarantee that +anomalies aren't missed. This summary displays the total runtime, the number +of successful jobs, the number of jobs which produced warnings, and the +number of jobs which weren't attempted due to failed dependencies. - [catkin]: ==> '/path/to/my_catkin_ws/build/catkin/build_env.sh /usr/bin/make -j4 -l4' in '/path/to/my_catkin_ws/build/catkin' - [catkin]: <== '/path/to/my_catkin_ws/build/catkin/build_env.sh /usr/bin/make -j4 -l4' finished with return code '0' +.. code-block:: none - [catkin]: ==> '/path/to/my_catkin_ws/build/catkin/build_env.sh /usr/bin/make install' in '/path/to/my_catkin_ws/build/catkin' - Install the project... - -- Install configuration: "" - ... truncated for brevity - [catkin]: <== '/path/to/my_catkin_ws/build/catkin/build_env.sh /usr/bin/make install' finished with return code '0' + [build] Runtime: 1.9 seconds total. + [build] Summary: 4 of 7 jobs completed. + [build] Warnings: None. + [build] Abandoned: 1 jobs were abandoned. + [build] Failed: 2 jobs failed. - Finished <== catkin [ 3.4 seconds ] +Building Subsets of Packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Consider a Catkin workspace with a **source space** populated with the following Catkin packages which have yet to be built: -.. note:: +.. code-block:: bash - The printing of these command outputs maybe be interleaved with commands - from other package builds if more than one package is being built at the - same time. + $ pwd + /tmp/path/to/my_catkin_ws -If you want to see the output from commands streaming to the screen, then you -can use the ``-i`` or ``--interleave`` option. This option will cause the -output from commands to be pushed to the screen immediately, instead of -buffering until the command finishes. This ends up being pretty confusing, so -when interleaved output is used ``catkin build`` prefixes each line with -``[]:`` like this: + $ ls ./* + ./src: + catkin console_bridge genlisp genpy + message_runtime ros_comm roscpp_core std_msgs + common_msgs gencpp genmsg message_generation + ros ros_tutorials rospack -.. code-block:: none - [roscpp_traits]: ==> '/Users/william/my_catkin_ws/build/roscpp_traits/build_env.sh /usr/bin/make cmake_check_build_system' in '/Users/william/my_catkin_ws/build/roscpp_traits' - [ros_tutorials]: -- The CXX compiler identification is Clang 5.0.0 - [ros_tutorials]: -- Check for working C compiler: /usr/bin/cc - [roscpp_traits]: ==> '/Users/william/my_catkin_ws/build/roscpp_traits/build_env.sh /usr/bin/make -j4 -l4' in '/Users/william/my_catkin_ws/build/roscpp_traits' - [rosbuild]: ==> '/Users/william/my_catkin_ws/build/rosbuild/build_env.sh /usr/bin/make -j4 -l4' in '/Users/william/my_catkin_ws/build/rosbuild' - [rosclean]: -- The C compiler identification is Clang 5.0.0 - [ros_tutorials]: -- Check for working C compiler: /usr/bin/cc -- works - [ros_tutorials]: -- Detecting C compiler ABI info - [rosclean]: -- The CXX compiler identification is Clang 5.0.0 - [rosclean]: -- Check for working C compiler: /usr/bin/cc +Building Specific Packages +-------------------------- -.. note:: +In addition to the usage above, the ``--dry-run`` option will show what the behavior of ``catkin build`` will be with various other options. +For example, the following will happen when you specify a single package to build: + +.. code-block:: bash + + $ catkin build roscpp_tutorials --dry-run + .... + Found '36' packages in 0.1 seconds. + Packages to be built: + - catkin (catkin) + - genmsg (catkin) + - gencpp (catkin) + - genlisp (catkin) + - genpy (catkin) + - console_bridge (cmake) + - cpp_common (catkin) + - message_generation (catkin) + - message_runtime (catkin) + - rosbuild (catkin) + - roscpp_traits (catkin) + - roslang (catkin) + - rospack (catkin) + - roslib (catkin) + - rostime (catkin) + - roscpp_serialization (catkin) + - rosunit (catkin) + - rosconsole (catkin) + - std_msgs (catkin) + - rosgraph_msgs (catkin) + - xmlrpcpp (catkin) + - roscpp (catkin) + - roscpp_tutorials (catkin) + Total packages: 23 + +As shown above, only 23 packages (``roscpp_tutorials`` and its dependencies), +of the total 36 packages would be built. + +Context-Aware Building +---------------------- + +In addition to building all packages or specified packages with various dependency requirements, ``catkin build`` can also determine the package containing the current working directory. +This is equivalent to specifying the name of the package on the command line, and is done by passing the ``--this`` option to ``catkin build`` like the following: + +.. code-block:: bash + + $ cd /tmp/path/to/my_catkin_ws/src/roscpp_tutorials + $ catkin build --this --dry-run + .... + Found '36' packages in 0.0 seconds. + Packages to be built: + - roscpp_tutorials (catkin) + Total packages: 1 + +Skipping Packages +----------------- + +Suppose you built every package up to ``roscpp_tutorials``, but that package had a build error. +After fixing the error, you could run the same build command again, but the ``build`` verb provides an option to save time in this situation. +If re-started from the beginning, none of the products of the dependencies of ``roscpp_tutorials`` would be re-built, but it would still take some time for the underlying byuildsystem to verify that for each package. + +Those checks could be skipped, however, by jumping directly to a given package. +You could use the ``--start-with`` option to continue the build where you left off after fixing the problem. (The following example uses the ``--dry-run`` option again to preview the behavior): + +.. code-block:: bash + + $ catkin build roscpp_tutorials --start-with roscpp_tutorials --dry-run + .... + Found '36' packages in 0.0 seconds. + Packages to be built: + (skip) catkin (catkin) + (skip) genmsg (catkin) + (skip) gencpp (catkin) + (skip) genlisp (catkin) + (skip) genpy (catkin) + (skip) console_bridge (cmake) + (skip) cpp_common (catkin) + (skip) message_generation (catkin) + (skip) message_runtime (catkin) + (skip) rosbuild (catkin) + (skip) roscpp_traits (catkin) + (skip) roslang (catkin) + (skip) rospack (catkin) + (skip) roslib (catkin) + (skip) rostime (catkin) + (skip) roscpp_serialization (catkin) + (skip) rosunit (catkin) + (skip) rosconsole (catkin) + (skip) std_msgs (catkin) + (skip) rosgraph_msgs (catkin) + (skip) xmlrpcpp (catkin) + (skip) roscpp (catkin) + ------ roscpp_tutorials (catkin) + Total packages: 23 - When you use ``-p 1`` and ``-v`` at the same time, ``-i`` is implicitly added. +However, you should be careful when using the ``--start-with`` option, as ``catkin build`` will assume that all dependencies leading up to that package have already been successfully built. +If you're only interested in building a *single* package in a workspace, you can also use the ``--no-deps`` option along with a package name. +This will skip all of the package's dependencies, build the given package, and then exit. + +.. code-block:: bash + + $ catkin build roscpp_tutorials --no-deps roscpp_tutorials --dry-run + .... + Found '36' packages in 0.0 seconds. + Packages to be built: + - roscpp_tutorials (catkin) + Total packages: 1 -Running Tests Built in a Workspace -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Building and Running Tests +^^^^^^^^^^^^^^^^^^^^^^^^^^ Running tests for a given package typically is done by invoking a special ``make`` target like ``test`` or ``run_tests``. catkin packages all define the ``run_tests`` target which aggregates all types of tests and runs them together. @@ -533,94 +527,56 @@ So if you want to check for failing tests, use the ``catkin_test_results`` comma The result code will be non-zero unless all tests passed. -Building With Warnings -^^^^^^^^^^^^^^^^^^^^^^ - -It can sometimes be useful to compile with additional warnings enabled across your whole catkin workspace. -To achieve this, use a command similar to this: - -.. code-block:: bash - - $ catkin build -v --cmake-args -DCMAKE_C_FLAGS="-Wall -W -Wno-unused-parameter" - -This command passes the ``-DCMAKE_C_FLAGS=...`` arugment to all invocations of ``cmake``. - -Debugging Build Errors -^^^^^^^^^^^^^^^^^^^^^^ - -As mentioned above, by default the output from each build is optimistically -hidden to give a clean overview of the workspace build, but when there is a -problem with a build a few things happen. -First, the package with a failure prints the failing command's output to the -screen between some enclosing lines: - -.. code-block:: none +Advanced Options +^^^^^^^^^^^^^^^^ - [rospack]: ==> '/path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make -j4 -l4' in '/path/to/my_catkin_ws/build/rospack' - [ 66%] Built target rospack - make[1]: *** [CMakeFiles/rosstackexe.dir/all] Interrupt: 2 - make[1]: *** [CMakeFiles/rospackexe.dir/all] Interrupt: 2 - make: *** [all] Interrupt: 2 - [rospack]: <== '/path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make -j4 -l4' failed with return code '-2' +Temporarily Changing Build Flags +-------------------------------- -And the status line is updated to reflect that that package has run into an -issue by placing a ``!`` in front of it: +While the build configuratoin flags are set and stored in the build context, +it's possible to temporarily override or augment them when using the ``build`` +verb. -.. code-block:: none - - [build - 1.7] [!cpp_common] [!rospack] [genlisp - 0.3] [1/1 Active | 10/23 Completed] +.. code-block:: bash -Then the ``catkin build`` command waits for the rest of the currently still building packages to finish -(without starting new package builds) and then summarizes the errors for you: + $ catkin build --cmake-args -DCMAKE_C_FLAGS="-Wall -W -Wno-unused-parameter" -.. code-block:: none +Building With Warnings +---------------------- - [build] There were '2' errors: +It can sometimes be useful to compile with additional warnings enabled across your whole catkin workspace. +To achieve this, use a command similar to this: - Failed to build package 'cpp_common' because the following command: +.. code-block:: bash - # Command run in directory: /path/to/my_catkin_ws/build/cpp_common - /path/to/my_catkin_ws/build/cpp_common/build_env.sh /usr/bin/make -j4 -l4 + $ catkin build -v --cmake-args -DCMAKE_C_FLAGS="-Wall -W -Wno-unused-parameter" - Exited with return code: -2 +This command passes the ``-DCMAKE_C_FLAGS=...`` arugment to all invocations of ``cmake``. - Failed to build package 'rospack' because the following command: - # Command run in directory: /path/to/my_catkin_ws/build/rospack - /path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make -j4 -l4 +Configuring Build Jobs +---------------------- - Exited with return code: -2 - Build summary: - Successful catkin - Successful genmsg - ... - Failed cpp_common - Failed rospack - Not built roscpp_serialization - Not built roscpp - ... +By default ``catkin build`` on a computer with ``N`` cores will build up to +``N`` packages in parallel and will distribute ``N`` ``make`` jobs among them +using an internal jobserver. If your platform doesn't support jobserver +scheduling, ``catkin build`` will pass ``-jN -lN`` to ``make`` for each package. -Packages marked as `Not built` were requested, but not yet built because catkin stopped due to failed packages. +You can control the maximum number of packages allowed to build in parallel by +using the ``-p`` or ``--parallel-packages`` option and you can change the +number of ``make`` jobs available with the ``-j`` or ``--jobs`` option. -To try to build as many requested packages as possible (instead of stopping after the first package failed), -you can pass the ``--continue-on-failure`` option. Then the ``catkin build`` command will then continue building packages whose dependencies built successfully +By default, these jobs options aren't passed to the underlying ``make`` +command. To disable the jobserver, you can use the ``--no-jobserver`` option, and +you can pass flags directly to ``make`` with the ``--make-args`` option. -If you don't want to scroll back up to find the error amongst the other output, -you can ``cat`` the whole build log out of the ``build_logs`` folder in the -**build space**: +.. note:: -.. code-block:: bash + Jobs flags (``-jN`` and/or ``-lN``) can be passed directly to ``make`` by + giving them to ``catkin build``, but other ``make`` arguments need to be + passed to the ``--make-args`` option. - $ cat build/build_logs/rospack.log - [rospack]: ==> '/path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make cmake_check_build_system' in '/path/to/my_catkin_ws/build/rospack' - [rospack]: <== '/path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make cmake_check_build_system' finished with return code '0' - [rospack]: ==> '/path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make -j4 -l4' in '/path/to/my_catkin_ws/build/rospack' - [ 66%] Built target rospack - make[1]: *** [CMakeFiles/rosstackexe.dir/all] Interrupt: 2 - make[1]: *** [CMakeFiles/rospackexe.dir/all] Interrupt: 2 - make: *** [all] Interrupt: 2 - [rospack]: <== '/path/to/my_catkin_ws/build/rospack/build_env.sh /usr/bin/make -j4 -l4' failed with return code '-2' Full Command-Line Interface ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/verbs/catkin_find.rst b/docs/verbs/catkin_locate.rst similarity index 95% rename from docs/verbs/catkin_find.rst rename to docs/verbs/catkin_locate.rst index dc6a0c09..d954bcd9 100644 --- a/docs/verbs/catkin_find.rst +++ b/docs/verbs/catkin_locate.rst @@ -1,5 +1,5 @@ -``catkin locate`` -- Find Workspace Locations -=========================================== +``catkin locate`` -- Locate Directories +======================================= The ``locate`` verb can be used to locate important locations in the workspace such as the active ``source``, ``build``, ``devel``, and ``install`` spaces, and package