Skip to content

Commit

Permalink
[lit] Implement DEFINE and REDEFINE directives
Browse files Browse the repository at this point in the history
These directives define per-test lit substitutions.  The concept was
discussed at
<https://discourse.llvm.org/t/iterating-lit-run-lines/62596/10>.

For example, the following directives can be inserted into a test file
to define `%{cflags}` and `%{fcflags}` substitutions with empty
initial values, which serve as the parameters of another newly defined
`%{check}` substitution:

```
// DEFINE: %{cflags} =
// DEFINE: %{fcflags} =

// DEFINE: %{check} = %clang_cc1 %{cflags} -emit-llvm -o - %s | \
// DEFINE:            FileCheck %{fcflags} %s
```

The following directives then redefine the parameters before each use
of `%{check}`:

```
// REDEFINE: %{cflags} = -foo
// REDEFINE: %{fcflags} = -check-prefix=FOO
// RUN: %{check}

// REDEFINE: %{cflags} = -bar
// REDEFINE: %{fcflags} = -check-prefix=BAR
// RUN: %{check}
```

Of course, `%{check}` would typically be more elaborate, increasing
the benefit of the reuse.

One issue is that the strings `DEFINE:` and `REDEFINE:` already appear
in 5 tests.  This patch adjusts those tests not to use those strings.
Our prediction is that, in the vast majority of cases, if a test
author mistakenly uses one of those strings for another purpose, the
text appearing after the string will not happen to have the syntax
required for these directives.  Thus, the test author will discover
the mistake immediately when lit reports the syntax error.

This patch also expands the documentation on existing lit substitution
behavior.

Reviewed By: jhenderson, MaskRay, awarzynski

Differential Revision: https://reviews.llvm.org/D132513
  • Loading branch information
jdenny-ornl committed Sep 21, 2022
1 parent bc97751 commit 28412d1
Show file tree
Hide file tree
Showing 67 changed files with 1,537 additions and 85 deletions.
60 changes: 30 additions & 30 deletions clang/test/CodeGen/attr-noundef.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,27 @@ struct Trivial {
};
Trivial ret_trivial() { return {}; }
void pass_trivial(Trivial e) {}
// CHECK-INTEL: [[DEFINE:define( dso_local)?]] i32 @{{.*}}ret_trivial
// CHECK-AARCH: [[DEFINE:define( dso_local)?]] i32 @{{.*}}ret_trivial
// CHECK-INTEL: [[DEFINE]] void @{{.*}}pass_trivial{{.*}}(i32 %
// CHECK-AARCH: [[DEFINE]] void @{{.*}}pass_trivial{{.*}}(i64 %
// CHECK-INTEL: [[DEF:define( dso_local)?]] i32 @{{.*}}ret_trivial
// CHECK-AARCH: [[DEF:define( dso_local)?]] i32 @{{.*}}ret_trivial
// CHECK-INTEL: [[DEF]] void @{{.*}}pass_trivial{{.*}}(i32 %
// CHECK-AARCH: [[DEF]] void @{{.*}}pass_trivial{{.*}}(i64 %

struct NoCopy {
int a;
NoCopy(NoCopy &) = delete;
};
NoCopy ret_nocopy() { return {}; }
void pass_nocopy(NoCopy e) {}
// CHECK: [[DEFINE]] void @{{.*}}ret_nocopy{{.*}}(%"struct.check_structs::NoCopy"* noalias sret({{[^)]+}}) align 4 %
// CHECK: [[DEFINE]] void @{{.*}}pass_nocopy{{.*}}(%"struct.check_structs::NoCopy"* noundef %
// CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(%"struct.check_structs::NoCopy"* noalias sret({{[^)]+}}) align 4 %
// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(%"struct.check_structs::NoCopy"* noundef %

struct Huge {
int a[1024];
};
Huge ret_huge() { return {}; }
void pass_huge(Huge h) {}
// CHECK: [[DEFINE]] void @{{.*}}ret_huge{{.*}}(%"struct.check_structs::Huge"* noalias sret({{[^)]+}}) align 4 %
// CHECK: [[DEFINE]] void @{{.*}}pass_huge{{.*}}(%"struct.check_structs::Huge"* noundef
// CHECK: [[DEF]] void @{{.*}}ret_huge{{.*}}(%"struct.check_structs::Huge"* noalias sret({{[^)]+}}) align 4 %
// CHECK: [[DEF]] void @{{.*}}pass_huge{{.*}}(%"struct.check_structs::Huge"* noundef
} // namespace check_structs

//************ Passing unions by value
Expand All @@ -47,19 +47,19 @@ union Trivial {
};
Trivial ret_trivial() { return {}; }
void pass_trivial(Trivial e) {}
// CHECK-INTEL: [[DEFINE]] i32 @{{.*}}ret_trivial
// CHECK-AARCH: [[DEFINE]] i32 @{{.*}}ret_trivial
// CHECK-INTEL: [[DEFINE]] void @{{.*}}pass_trivial{{.*}}(i32 %
// CHECK-AARCH: [[DEFINE]] void @{{.*}}pass_trivial{{.*}}(i64 %
// CHECK-INTEL: [[DEF]] i32 @{{.*}}ret_trivial
// CHECK-AARCH: [[DEF]] i32 @{{.*}}ret_trivial
// CHECK-INTEL: [[DEF]] void @{{.*}}pass_trivial{{.*}}(i32 %
// CHECK-AARCH: [[DEF]] void @{{.*}}pass_trivial{{.*}}(i64 %

union NoCopy {
int a;
NoCopy(NoCopy &) = delete;
};
NoCopy ret_nocopy() { return {}; }
void pass_nocopy(NoCopy e) {}
// CHECK: [[DEFINE]] void @{{.*}}ret_nocopy{{.*}}(%"union.check_unions::NoCopy"* noalias sret({{[^)]+}}) align 4 %
// CHECK: [[DEFINE]] void @{{.*}}pass_nocopy{{.*}}(%"union.check_unions::NoCopy"* noundef %
// CHECK: [[DEF]] void @{{.*}}ret_nocopy{{.*}}(%"union.check_unions::NoCopy"* noalias sret({{[^)]+}}) align 4 %
// CHECK: [[DEF]] void @{{.*}}pass_nocopy{{.*}}(%"union.check_unions::NoCopy"* noundef %
} // namespace check_unions

//************ Passing `this` pointers
Expand Down Expand Up @@ -100,9 +100,9 @@ i32x3 ret_vec() {
void pass_vec(i32x3 v) {
}

// CHECK: [[DEFINE]] noundef <3 x i32> @{{.*}}ret_vec{{.*}}()
// CHECK-INTEL: [[DEFINE]] void @{{.*}}pass_vec{{.*}}(<3 x i32> noundef %
// CHECK-AARCH: [[DEFINE]] void @{{.*}}pass_vec{{.*}}(<4 x i32> %
// CHECK: [[DEF]] noundef <3 x i32> @{{.*}}ret_vec{{.*}}()
// CHECK-INTEL: [[DEF]] void @{{.*}}pass_vec{{.*}}(<3 x i32> noundef %
// CHECK-AARCH: [[DEF]] void @{{.*}}pass_vec{{.*}}(<4 x i32> %
} // namespace check_vecs

//************ Passing exotic types
Expand Down Expand Up @@ -145,23 +145,23 @@ void pass_large_BitInt(_BitInt(127) e) {
}

// Pointers to arrays/functions are always noundef
// CHECK: [[DEFINE]] noundef [32 x i32]* @{{.*}}ret_arrptr{{.*}}()
// CHECK: [[DEFINE]] noundef i32 (i32)* @{{.*}}ret_fnptr{{.*}}()
// CHECK: [[DEF]] noundef [32 x i32]* @{{.*}}ret_arrptr{{.*}}()
// CHECK: [[DEF]] noundef i32 (i32)* @{{.*}}ret_fnptr{{.*}}()

// Pointers to members are never noundef
// CHECK: [[DEFINE]] i64 @{{.*}}ret_mdptr{{.*}}()
// CHECK-INTEL: [[DEFINE]] { i64, i64 } @{{.*}}ret_mfptr{{.*}}()
// CHECK-AARCH: [[DEFINE]] [2 x i64] @{{.*}}ret_mfptr{{.*}}()
// CHECK: [[DEF]] i64 @{{.*}}ret_mdptr{{.*}}()
// CHECK-INTEL: [[DEF]] { i64, i64 } @{{.*}}ret_mfptr{{.*}}()
// CHECK-AARCH: [[DEF]] [2 x i64] @{{.*}}ret_mfptr{{.*}}()

// nullptr_t is never noundef
// CHECK: [[DEFINE]] i8* @{{.*}}ret_npt{{.*}}()
// CHECK: [[DEFINE]] void @{{.*}}pass_npt{{.*}}(i8* %
// CHECK: [[DEF]] i8* @{{.*}}ret_npt{{.*}}()
// CHECK: [[DEF]] void @{{.*}}pass_npt{{.*}}(i8* %

// TODO: for now, ExtInt is only noundef if it is sign/zero-extended
// CHECK-INTEL: [[DEFINE]] noundef signext i3 @{{.*}}ret_BitInt{{.*}}()
// CHECK-AARCH: [[DEFINE]] i3 @{{.*}}ret_BitInt{{.*}}()
// CHECK-INTEL: [[DEFINE]] void @{{.*}}pass_BitInt{{.*}}(i3 noundef signext %
// CHECK-AARCH: [[DEFINE]] void @{{.*}}pass_BitInt{{.*}}(i3 %
// CHECK-INTEL: [[DEFINE]] void @{{.*}}pass_large_BitInt{{.*}}(i64 %{{.*}}, i64 %
// CHECK-AARCH: [[DEFINE]] void @{{.*}}pass_large_BitInt{{.*}}(i127 %
// CHECK-INTEL: [[DEF]] noundef signext i3 @{{.*}}ret_BitInt{{.*}}()
// CHECK-AARCH: [[DEF]] i3 @{{.*}}ret_BitInt{{.*}}()
// CHECK-INTEL: [[DEF]] void @{{.*}}pass_BitInt{{.*}}(i3 noundef signext %
// CHECK-AARCH: [[DEF]] void @{{.*}}pass_BitInt{{.*}}(i3 %
// CHECK-INTEL: [[DEF]] void @{{.*}}pass_large_BitInt{{.*}}(i64 %{{.*}}, i64 %
// CHECK-AARCH: [[DEF]] void @{{.*}}pass_large_BitInt{{.*}}(i127 %
} // namespace check_exotic
4 changes: 2 additions & 2 deletions clang/test/CodeGen/indirect-noundef.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ int (*indirect_callee_int_ptr)(int);
// CHECK: @indirect_callee_union_ptr = [[GLOBAL]] i32 (i32)*
union u1 (*indirect_callee_union_ptr)(union u1);

// CHECK: [[DEFINE:define( dso_local)?]] noundef i32 @{{.*}}indirect_callee_int{{.*}}(i32 noundef %
// CHECK: [[DEF:define( dso_local)?]] noundef i32 @{{.*}}indirect_callee_int{{.*}}(i32 noundef %
int indirect_callee_int(int a) { return a; }
// CHECK: [[DEFINE]] i32 @{{.*}}indirect_callee_union{{.*}}(i32 %
// CHECK: [[DEF]] i32 @{{.*}}indirect_callee_union{{.*}}(i32 %
union u1 indirect_callee_union(union u1 a) {
return a;
}
Expand Down
12 changes: 6 additions & 6 deletions clang/test/Preprocessor/init.c
Original file line number Diff line number Diff line change
Expand Up @@ -1396,13 +1396,13 @@
// SPARC64-OBSD:#define __UINTMAX_C_SUFFIX__ ULL
// SPARC64-OBSD:#define __UINTMAX_TYPE__ long long unsigned int
//
// RUN: %clang_cc1 -E -dM -ffreestanding -triple=x86_64-pc-kfreebsd-gnu < /dev/null | FileCheck -match-full-lines -check-prefix KFREEBSD-DEFINE %s
// KFREEBSD-DEFINE:#define __FreeBSD_kernel__ 1
// KFREEBSD-DEFINE:#define __GLIBC__ 1
// RUN: %clang_cc1 -E -dM -ffreestanding -triple=x86_64-pc-kfreebsd-gnu < /dev/null | FileCheck -match-full-lines -check-prefix KFREEBSD-DEF %s
// KFREEBSD-DEF:#define __FreeBSD_kernel__ 1
// KFREEBSD-DEF:#define __GLIBC__ 1
//
// RUN: %clang_cc1 -E -dM -ffreestanding -triple=i686-pc-kfreebsd-gnu < /dev/null | FileCheck -match-full-lines -check-prefix KFREEBSDI686-DEFINE %s
// KFREEBSDI686-DEFINE:#define __FreeBSD_kernel__ 1
// KFREEBSDI686-DEFINE:#define __GLIBC__ 1
// RUN: %clang_cc1 -E -dM -ffreestanding -triple=i686-pc-kfreebsd-gnu < /dev/null | FileCheck -match-full-lines -check-prefix KFREEBSDI686-DEF %s
// KFREEBSDI686-DEF:#define __FreeBSD_kernel__ 1
// KFREEBSDI686-DEF:#define __GLIBC__ 1
//
// RUN: %clang_cc1 -x c++ -triple i686-pc-linux-gnu -fobjc-runtime=gcc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GNUSOURCE %s
// RUN: %clang_cc1 -x c++ -triple sparc-rtems-elf -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix GNUSOURCE %s
Expand Down
8 changes: 0 additions & 8 deletions llvm/docs/CommandGuide/lit.rst
Original file line number Diff line number Diff line change
Expand Up @@ -562,14 +562,6 @@ Other substitutions are provided that are variations on this base set and
further substitution patterns can be defined by each test module. See the
modules :ref:`local-configuration-files`.

By default, substitutions are expanded exactly once, so that if e.g. a
substitution ``%build`` is defined in top of another substitution ``%cxx``,
``%build`` will expand to ``%cxx`` textually, not to what ``%cxx`` expands to.
However, if the ``recursiveExpansionLimit`` property of the ``TestingConfig``
is set to a non-negative integer, substitutions will be expanded recursively
until that limit is reached. It is an error if the limit is reached and
expanding substitutions again would yield a different result.

More detailed information on substitutions can be found in the
:doc:`../TestingGuide`.

Expand Down
208 changes: 201 additions & 7 deletions llvm/docs/TestingGuide.rst
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,15 @@ RUN lines:
``%else %{<else branch>%}`` is optional and treated like ``%else %{%}``
if not present.

``%(line)``, ``%(line+<number>)``, ``%(line-<number>)``

The number of the line where this substitution is used, with an
optional integer offset. These expand only if they appear
immediately in ``RUN:``, ``DEFINE:``, and ``REDEFINE:`` directives.
Occurrences in substitutions defined elsewhere are never expanded.
For example, this can be used in tests with multiple RUN lines,
which reference the test file's line numbers.

**LLVM-specific substitutions:**

``%shlibext``
Expand All @@ -633,12 +642,6 @@ RUN lines:

Example: ``.exe`` (Windows), empty on Linux.

``%(line)``, ``%(line+<number>)``, ``%(line-<number>)``
The number of the line where this substitution is used, with an optional
integer offset. This can be used in tests with multiple RUN lines, which
reference test file's line numbers.


**Clang-specific substitutions:**

``%clang``
Expand Down Expand Up @@ -670,8 +673,199 @@ RUN lines:
output affects test results. It's usually easy to tell: just look for
redirection or piping of the ``FileCheck`` call's stdout or stderr.

To add more substitutions, look at ``test/lit.cfg`` or ``lit.local.cfg``.
.. _Test-specific substitutions:

**Test-specific substitutions:**

Additional substitutions can be defined as follows:

- Lit configuration files (e.g., ``lit.cfg`` or ``lit.local.cfg``) can define
substitutions for all tests in a test directory. They do so by extending the
substitution list, ``config.substitutions``. Each item in the list is a tuple
consisting of a pattern and its replacement, which lit applies using python's
``re.sub`` function.
- To define substitutions within a single test file, lit supports the
``DEFINE:`` and ``REDEFINE:`` directives, described in detail below. So that
they have no effect on other test files, these directives modify a copy of the
substitution list that is produced by lit configuration files.

For example, the following directives can be inserted into a test file to define
``%{cflags}`` and ``%{fcflags}`` substitutions with empty initial values, which
serve as the parameters of another newly defined ``%{check}`` substitution:

.. code-block:: llvm
; DEFINE: %{cflags} =
; DEFINE: %{fcflags} =
; DEFINE: %{check} = \
; DEFINE: %clang_cc1 -verify -fopenmp -fopenmp-version=51 %{cflags} \
; DEFINE: -emit-llvm -o - %s | \
; DEFINE: FileCheck %{fcflags} %s
Alternatively, the above substitutions can be defined in a lit configuration
file to be shared with other test files. Either way, the test file can then
specify directives like the following to redefine the parameter substitutions as
desired before each use of ``%{check}`` in a ``RUN:`` line:

.. code-block:: llvm
; REDEFINE: %{cflags} = -triple x86_64-apple-darwin10.6.0 -fopenmp-simd
; REDEFINE: %{fcflags} = -check-prefix=SIMD
; RUN: %{check}
; REDEFINE: %{cflags} = -triple x86_64-unknown-linux-gnu -fopenmp-simd
; REDEFINE: %{fcflags} = -check-prefix=SIMD
; RUN: %{check}
; REDEFINE: %{cflags} = -triple x86_64-apple-darwin10.6.0
; REDEFINE: %{fcflags} = -check-prefix=NO-SIMD
; RUN: %{check}
; REDEFINE: %{cflags} = -triple x86_64-unknown-linux-gnu
; REDEFINE: %{fcflags} = -check-prefix=NO-SIMD
; RUN: %{check}
Besides providing initial values, the initial ``DEFINE:`` directives for the
parameter substitutions in the above example serve a second purpose: they
establish the substitution order so that both ``%{check}`` and its parameters
expand as desired. There's a simple way to remember the required definition
order in a test file: define a substitution before any substitution that might
refer to it.

In general, substitution expansion behaves as follows:

- Upon arriving at each ``RUN:`` line, lit expands all substitutions in that
``RUN:`` line using their current values from the substitution list. No
substitution expansion is performed immediately at ``DEFINE:`` and
``REDEFINE:`` directives except ``%(line)``, ``%(line+<number>)``, and
``%(line-<number>)``.
- When expanding substitutions in a ``RUN:`` line, lit makes only one pass
through the substitution list by default. In this case, a substitution must
have been inserted earlier in the substitution list than any substitution
appearing in its value in order for the latter to expand. (For greater
flexibility, you can enable multiple passes through the substitution list by
setting `recursiveExpansionLimit`_ in a lit configuration file.)
- While lit configuration files can insert anywhere in the substitution list,
the insertion behavior of the ``DEFINE:`` and ``REDEFINE:`` directives is
specified below and is designed specifically for the use case presented in the
example above.
- Defining a substitution in terms of itself, whether directly or via other
substitutions, should be avoided. It usually produces an infinitely recursive
definition that cannot be fully expanded. It does *not* define the
substitution in terms of its previous value, even when using ``REDEFINE:``.

The relationship between the ``DEFINE:`` and ``REDEFINE:`` directive is
analogous to the relationship between a variable declaration and variable
assignment in many programming languages:

- ``DEFINE: %{name} = value``

This directive assigns the specified value to a new substitution whose
pattern is ``%{name}``, or it reports an error if there is already a
substitution whose pattern contains ``%{name}`` because that could produce
confusing expansions (e.g., a lit configuration file might define a
substitution with the pattern ``%{name}\[0\]``). The new substitution is
inserted at the start of the substitution list so that it will expand first.
Thus, its value can contain any substitution previously defined, whether in
the same test file or in a lit configuration file, and both will expand.

- ``REDEFINE: %{name} = value``

This directive assigns the specified value to an existing substitution whose
pattern is ``%{name}``, or it reports an error if there are no substitutions
with that pattern or if there are multiple substitutions whose patterns
contain ``%{name}``. The substitution's current position in the substitution
list does not change so that expansion order relative to other existing
substitutions is preserved.

The following properties apply to both the ``DEFINE:`` and ``REDEFINE:``
directives:

- **Substitution name**: In the directive, whitespace immediately before or
after ``%{name}`` is optional and discarded. ``%{name}`` must start with
``%{``, it must end with ``}``, and the rest must start with a letter or
underscore and contain only alphanumeric characters, hyphens, underscores, and
colons. This syntax has a few advantages:

- It is impossible for ``%{name}`` to contain sequences that are special in
python's ``re.sub`` patterns. Otherwise, attempting to specify
``%{name}`` as a substitution pattern in a lit configuration file could
produce confusing expansions.
- The braces help avoid the possibility that another substitution's pattern
will match part of ``%{name}`` or vice-versa, producing confusing
expansions. However, the patterns of substitutions defined by lit
configuration files and by lit itself are not restricted to this form, so
overlaps are still theoretically possible.

- **Substitution value**: The value includes all text from the first
non-whitespace character after ``=`` to the last non-whitespace character. If
there is no non-whitespace character after ``=``, the value is the empty
string. Escape sequences that can appear in python ``re.sub`` replacement
strings are treated as plain text in the value.
- **Line continuations**: If the last non-whitespace character on the line after
``:`` is ``\``, then the next directive must use the same directive keyword
(e.g., ``DEFINE:``) , and it is an error if there is no additional directive.
That directive serves as a continuation. That is, before following the rules
above to parse the text after ``:`` in either directive, lit joins that text
together to form a single directive, replaces the ``\`` with a single space,
and removes any other whitespace that is now adjacent to that space. A
continuation can be continued in the same manner. A continuation containing
only whitespace after its ``:`` is an error.

.. _recursiveExpansionLimit:

**recursiveExpansionLimit:**

As described in the previous section, when expanding substitutions in a ``RUN:``
line, lit makes only one pass through the substitution list by default. Thus,
if substitutions are not defined in the proper order, some will remain in the
``RUN:`` line unexpanded. For example, the following directives refer to
``%{inner}`` within ``%{outer}`` but do not define ``%{inner}`` until after
``%{outer}``:

.. code-block:: llvm
; By default, this definition order does not enable full expansion.
; DEFINE: %{outer} = %{inner}
; DEFINE: %{inner} = expanded
; RUN: echo '%{outer}'
``DEFINE:`` inserts substitutions at the start of the substitution list, so
``%{inner}`` expands first but has no effect because the original ``RUN:`` line
does not contain ``%{inner}``. Next, ``%{outer}`` expands, and the output of
the ``echo`` command becomes:

.. code-block:: shell
%{inner}
Of course, one way to fix this simple case is to reverse the definitions of
``%{outer}`` and ``%{inner}``. However, if a test has a complex set of
substitutions that can all reference each other, there might not exist a
sufficient substitution order.

To address such use cases, lit configuration files support
``config.recursiveExpansionLimit``, which can be set to a non-negative integer
to specify the maximum number of passes through the substitution list. Thus, in
the above example, setting the limit to 2 would cause lit to make a second pass
that expands ``%{inner}`` in the ``RUN:`` line, and the output from the ``echo``
command when then be:

.. code-block:: shell
expanded
To improve performance, lit will stop making passes when it notices the ``RUN:``
line has stopped changing. In the above example, setting the limit higher than
2 is thus harmless.

To facilitate debugging, after reaching the limit, lit will make one extra pass
and report an error if the ``RUN:`` line changes again. In the above example,
setting the limit to 1 will thus cause lit to report an error instead of
producing incorrect output.

Options
-------
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/tools/llvm-cvtres/help.test
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
; HELP_TEST: OVERVIEW: Resource Converter
; HELP_TEST-DAG: USAGE: llvm-cvtres [options] file...
; HELP_TEST-DAG: OPTIONS:
; HELP_TEST-NEXT: /DEFINE:symbol Not implemented
; HELP_TEST-NEXT: /{{DEFINE}}:symbol Not implemented
; HELP_TEST-NEXT: /FOLDDUPS: Not implemented
; HELP_TEST-NEXT: /HELP Display available options
; HELP_TEST-NEXT: /MACHINE:{ARM|ARM64|EBC|IA64|X64|X86}
Expand Down
Loading

0 comments on commit 28412d1

Please sign in to comment.