diff --git a/build/pkgs/sagemath_objects/spkg-install b/build/pkgs/sagemath_objects/spkg-install index 6cc85e07e55..472e7f0d4d6 100755 --- a/build/pkgs/sagemath_objects/spkg-install +++ b/build/pkgs/sagemath_objects/spkg-install @@ -21,9 +21,28 @@ export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS" DIST_DIR="$(mktemp -d)" python3 -m build --outdir "$DIST_DIR"/dist . || sdh_die "Failure building sdist and wheel" -wheel=$(cd "$DIST_DIR" && sdh_store_wheel . && echo $wheel) +wheel=$(cd "$DIST_DIR" && sdh_store_wheel . >&2 && echo $wheel) ls -l "$wheel" if [ "$SAGE_CHECK" != no ]; then - tox -r -v -e sagepython-sagewheels-nopypi-norequirements --installpkg $wheel + export TOX_PARALLEL_NO_SPINNER=1 + echo Running "tox -r -p auto -v --installpkg $wheel" + tox -r -p auto -v --installpkg $wheel + status=$? + case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in + 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";; + 0:*:*) echo "Passed the test suite";; + *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;; + *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;; + *:yes:0) echo "New failures, not in baseline known-test-failures*.json";; + *:yes:*) echo "Failures testing the package";; + esac + # Show summaries of failures (suppress lines ending with '[failed in baseline]') + for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do + if [ -r "$f" ]; then + echo "$f" + grep '^sage -t.*#[^]]*$' "$f" + fi + done + exit $status fi diff --git a/pkgs/sagemath-categories/known-test-failures.json b/pkgs/sagemath-categories/known-test-failures.json new file mode 100644 index 00000000000..ddae185a7d9 --- /dev/null +++ b/pkgs/sagemath-categories/known-test-failures.json @@ -0,0 +1,1421 @@ +{ + "sage.arith.numerical_approx": { + "ntests": 1 + }, + "sage.arith.power": { + "failed": true, + "ntests": 16 + }, + "sage.categories.action": { + "failed": true, + "ntests": 78 + }, + "sage.categories.additive_groups": { + "ntests": 9 + }, + "sage.categories.additive_magmas": { + "failed": true, + "ntests": 134 + }, + "sage.categories.additive_monoids": { + "failed": true, + "ntests": 16 + }, + "sage.categories.additive_semigroups": { + "failed": true, + "ntests": 28 + }, + "sage.categories.affine_weyl_groups": { + "ntests": 14 + }, + "sage.categories.algebra_ideals": { + "failed": true, + "ntests": 8 + }, + "sage.categories.algebra_modules": { + "failed": true, + "ntests": 9 + }, + "sage.categories.algebras": { + "failed": true, + "ntests": 20 + }, + "sage.categories.algebras_with_basis": { + "failed": true, + "ntests": 46 + }, + "sage.categories.aperiodic_semigroups": { + "ntests": 1 + }, + "sage.categories.associative_algebras": { + "failed": true, + "ntests": 5 + }, + "sage.categories.bialgebras": { + "ntests": 3 + }, + "sage.categories.bialgebras_with_basis": { + "failed": true, + "ntests": 31 + }, + "sage.categories.bimodules": { + "failed": true, + "ntests": 15 + }, + "sage.categories.cartesian_product": { + "failed": true, + "ntests": 42 + }, + "sage.categories.category": { + "failed": true, + "ntests": 433 + }, + "sage.categories.category_cy_helper": { + "failed": true, + "ntests": 27 + }, + "sage.categories.category_singleton": { + "failed": true, + "ntests": 59 + }, + "sage.categories.category_types": { + "failed": true, + "ntests": 74 + }, + "sage.categories.category_with_axiom": { + "failed": true, + "ntests": 328 + }, + "sage.categories.chain_complexes": { + "failed": true, + "ntests": 20 + }, + "sage.categories.coalgebras": { + "ntests": 6 + }, + "sage.categories.coalgebras_with_basis": { + "failed": true, + "ntests": 23 + }, + "sage.categories.coercion_methods": { + "failed": true, + "ntests": 6 + }, + "sage.categories.commutative_additive_groups": { + "failed": true, + "ntests": 17 + }, + "sage.categories.commutative_additive_monoids": { + "ntests": 5 + }, + "sage.categories.commutative_additive_semigroups": { + "failed": true, + "ntests": 6 + }, + "sage.categories.commutative_algebra_ideals": { + "failed": true, + "ntests": 9 + }, + "sage.categories.commutative_algebras": { + "failed": true, + "ntests": 11 + }, + "sage.categories.commutative_ring_ideals": { + "failed": true, + "ntests": 7 + }, + "sage.categories.commutative_rings": { + "failed": true, + "ntests": 39 + }, + "sage.categories.complete_discrete_valuation": { + "failed": true, + "ntests": 30 + }, + "sage.categories.complex_reflection_groups": { + "failed": true, + "ntests": 16 + }, + "sage.categories.complex_reflection_or_generalized_coxeter_groups": { + "ntests": 67 + }, + "sage.categories.covariant_functorial_construction": { + "failed": true, + "ntests": 65 + }, + "sage.categories.coxeter_group_algebras": { + "ntests": 2 + }, + "sage.categories.coxeter_groups": { + "failed": true, + "ntests": 362 + }, + "sage.categories.cw_complexes": { + "failed": true, + "ntests": 36 + }, + "sage.categories.discrete_valuation": { + "failed": true, + "ntests": 23 + }, + "sage.categories.distributive_magmas_and_additive_magmas": { + "failed": true, + "ntests": 13 + }, + "sage.categories.division_rings": { + "ntests": 11 + }, + "sage.categories.domains": { + "failed": true, + "ntests": 7 + }, + "sage.categories.drinfeld_modules": { + "ntests": 2 + }, + "sage.categories.dual": { + "failed": true, + "ntests": 1 + }, + "sage.categories.enumerated_sets": { + "failed": true, + "ntests": 132 + }, + "sage.categories.euclidean_domains": { + "failed": true, + "ntests": 22 + }, + "sage.categories.examples.algebras_with_basis": { + "ntests": 8 + }, + "sage.categories.examples.commutative_additive_monoids": { + "failed": true, + "ntests": 14 + }, + "sage.categories.examples.commutative_additive_semigroups": { + "failed": true, + "ntests": 28 + }, + "sage.categories.examples.cw_complexes": { + "failed": true, + "ntests": 33 + }, + "sage.categories.examples.facade_sets": { + "failed": true, + "ntests": 21 + }, + "sage.categories.examples.filtered_algebras_with_basis": { + "failed": true, + "ntests": 25 + }, + "sage.categories.examples.filtered_modules_with_basis": { + "ntests": 12 + }, + "sage.categories.examples.finite_dimensional_lie_algebras_with_basis": { + "ntests": 1 + }, + "sage.categories.examples.finite_enumerated_sets": { + "failed": true, + "ntests": 29 + }, + "sage.categories.examples.finite_monoids": { + "failed": true, + "ntests": 15 + }, + "sage.categories.examples.finite_semigroups": { + "failed": true, + "ntests": 27 + }, + "sage.categories.examples.finite_weyl_groups": { + "failed": true, + "ntests": 25 + }, + "sage.categories.examples.graded_modules_with_basis": { + "ntests": 14 + }, + "sage.categories.examples.graphs": { + "failed": true, + "ntests": 24 + }, + "sage.categories.examples.infinite_enumerated_sets": { + "failed": true, + "ntests": 35 + }, + "sage.categories.examples.lie_algebras": { + "ntests": 25 + }, + "sage.categories.examples.magmas": { + "failed": true, + "ntests": 20 + }, + "sage.categories.examples.manifolds": { + "failed": true, + "ntests": 15 + }, + "sage.categories.examples.monoids": { + "failed": true, + "ntests": 16 + }, + "sage.categories.examples.posets": { + "failed": true, + "ntests": 29 + }, + "sage.categories.examples.semigroups": { + "failed": true, + "ntests": 83 + }, + "sage.categories.examples.semigroups_cython": { + "failed": true, + "ntests": 47 + }, + "sage.categories.examples.sets_with_grading": { + "failed": true, + "ntests": 14 + }, + "sage.categories.facade_sets": { + "failed": true, + "ntests": 27 + }, + "sage.categories.fields": { + "failed": true, + "ntests": 109 + }, + "sage.categories.filtered_algebras": { + "failed": true, + "ntests": 5 + }, + "sage.categories.filtered_algebras_with_basis": { + "failed": true, + "ntests": 39 + }, + "sage.categories.filtered_hopf_algebras_with_basis": { + "failed": true, + "ntests": 12 + }, + "sage.categories.filtered_modules": { + "failed": true, + "ntests": 18 + }, + "sage.categories.filtered_modules_with_basis": { + "failed": true, + "ntests": 65 + }, + "sage.categories.finite_complex_reflection_groups": { + "failed": true, + "ntests": 178 + }, + "sage.categories.finite_coxeter_groups": { + "ntests": 6 + }, + "sage.categories.finite_dimensional_algebras_with_basis": { + "failed": true, + "ntests": 87 + }, + "sage.categories.finite_dimensional_bialgebras_with_basis": { + "failed": true, + "ntests": 4 + }, + "sage.categories.finite_dimensional_coalgebras_with_basis": { + "failed": true, + "ntests": 4 + }, + "sage.categories.finite_dimensional_graded_lie_algebras_with_basis": { + "failed": true, + "ntests": 22 + }, + "sage.categories.finite_dimensional_hopf_algebras_with_basis": { + "failed": true, + "ntests": 3 + }, + "sage.categories.finite_dimensional_lie_algebras_with_basis": { + "failed": true, + "ntests": 34 + }, + "sage.categories.finite_dimensional_modules_with_basis": { + "failed": true, + "ntests": 55 + }, + "sage.categories.finite_dimensional_nilpotent_lie_algebras_with_basis": { + "failed": true, + "ntests": 19 + }, + "sage.categories.finite_dimensional_semisimple_algebras_with_basis": { + "failed": true, + "ntests": 12 + }, + "sage.categories.finite_enumerated_sets": { + "failed": true, + "ntests": 123 + }, + "sage.categories.finite_fields": { + "failed": true, + "ntests": 14 + }, + "sage.categories.finite_lattice_posets": { + "ntests": 31 + }, + "sage.categories.finite_monoids": { + "failed": true, + "ntests": 28 + }, + "sage.categories.finite_permutation_groups": { + "ntests": 2 + }, + "sage.categories.finite_semigroups": { + "failed": true, + "ntests": 14 + }, + "sage.categories.finite_sets": { + "failed": true, + "ntests": 14 + }, + "sage.categories.finite_weyl_groups": { + "failed": true, + "ntests": 6 + }, + "sage.categories.finitely_generated_lambda_bracket_algebras": { + "ntests": 11 + }, + "sage.categories.finitely_generated_lie_conformal_algebras": { + "ntests": 10 + }, + "sage.categories.finitely_generated_magmas": { + "failed": true, + "ntests": 6 + }, + "sage.categories.finitely_generated_semigroups": { + "failed": true, + "ntests": 27 + }, + "sage.categories.function_fields": { + "failed": true, + "ntests": 11 + }, + "sage.categories.functor": { + "failed": true, + "ntests": 128 + }, + "sage.categories.g_sets": { + "ntests": 7 + }, + "sage.categories.gcd_domains": { + "ntests": 5 + }, + "sage.categories.generalized_coxeter_groups": { + "ntests": 12 + }, + "sage.categories.graded_algebras": { + "failed": true, + "ntests": 8 + }, + "sage.categories.graded_algebras_with_basis": { + "failed": true, + "ntests": 8 + }, + "sage.categories.graded_bialgebras": { + "failed": true, + "ntests": 3 + }, + "sage.categories.graded_bialgebras_with_basis": { + "failed": true, + "ntests": 3 + }, + "sage.categories.graded_coalgebras": { + "failed": true, + "ntests": 6 + }, + "sage.categories.graded_coalgebras_with_basis": { + "failed": true, + "ntests": 6 + }, + "sage.categories.graded_hopf_algebras": { + "failed": true, + "ntests": 3 + }, + "sage.categories.graded_hopf_algebras_with_basis": { + "failed": true, + "ntests": 12 + }, + "sage.categories.graded_lie_algebras": { + "failed": true, + "ntests": 12 + }, + "sage.categories.graded_lie_algebras_with_basis": { + "failed": true, + "ntests": 5 + }, + "sage.categories.graded_lie_conformal_algebras": { + "failed": true, + "ntests": 5 + }, + "sage.categories.graded_modules": { + "failed": true, + "ntests": 16 + }, + "sage.categories.graded_modules_with_basis": { + "failed": true, + "ntests": 25 + }, + "sage.categories.graphs": { + "failed": true, + "ntests": 25 + }, + "sage.categories.group_algebras": { + "failed": true, + "ntests": 27 + }, + "sage.categories.groups": { + "ntests": 45 + }, + "sage.categories.h_trivial_semigroups": { + "ntests": 4 + }, + "sage.categories.hecke_modules": { + "failed": true, + "ntests": 16 + }, + "sage.categories.homset": { + "failed": true, + "ntests": 197 + }, + "sage.categories.homsets": { + "failed": true, + "ntests": 56 + }, + "sage.categories.hopf_algebras": { + "failed": true, + "ntests": 15 + }, + "sage.categories.hopf_algebras_with_basis": { + "failed": true, + "ntests": 30 + }, + "sage.categories.infinite_enumerated_sets": { + "failed": true, + "ntests": 13 + }, + "sage.categories.inner_product_spaces": { + "failed": true, + "ntests": 9 + }, + "sage.categories.integral_domains": { + "failed": true, + "ntests": 22 + }, + "sage.categories.isomorphic_objects": { + "ntests": 2 + }, + "sage.categories.j_trivial_semigroups": { + "ntests": 1 + }, + "sage.categories.kac_moody_algebras": { + "failed": true, + "ntests": 9 + }, + "sage.categories.l_trivial_semigroups": { + "ntests": 5 + }, + "sage.categories.lambda_bracket_algebras": { + "failed": true, + "ntests": 16 + }, + "sage.categories.lambda_bracket_algebras_with_basis": { + "ntests": 5 + }, + "sage.categories.lattice_posets": { + "ntests": 10 + }, + "sage.categories.left_modules": { + "failed": true, + "ntests": 4 + }, + "sage.categories.lie_algebras": { + "failed": true, + "ntests": 125 + }, + "sage.categories.lie_algebras_with_basis": { + "ntests": 19 + }, + "sage.categories.lie_conformal_algebras": { + "failed": true, + "ntests": 25 + }, + "sage.categories.lie_conformal_algebras_with_basis": { + "ntests": 17 + }, + "sage.categories.lie_groups": { + "failed": true, + "ntests": 9 + }, + "sage.categories.loop_crystals": { + "ntests": 1 + }, + "sage.categories.magmas": { + "failed": true, + "ntests": 147 + }, + "sage.categories.magmas_and_additive_magmas": { + "ntests": 21 + }, + "sage.categories.magmatic_algebras": { + "failed": true, + "ntests": 27 + }, + "sage.categories.manifolds": { + "failed": true, + "ntests": 52 + }, + "sage.categories.map": { + "failed": true, + "ntests": 353 + }, + "sage.categories.matrix_algebras": { + "failed": true, + "ntests": 3 + }, + "sage.categories.metric_spaces": { + "failed": true, + "ntests": 46 + }, + "sage.categories.modular_abelian_varieties": { + "failed": true, + "ntests": 8 + }, + "sage.categories.modules": { + "failed": true, + "ntests": 120 + }, + "sage.categories.modules_with_basis": { + "failed": true, + "ntests": 221 + }, + "sage.categories.monoid_algebras": { + "failed": true, + "ntests": 4 + }, + "sage.categories.monoids": { + "failed": true, + "ntests": 81 + }, + "sage.categories.morphism": { + "failed": true, + "ntests": 99 + }, + "sage.categories.number_fields": { + "failed": true, + "ntests": 41 + }, + "sage.categories.objects": { + "failed": true, + "ntests": 12 + }, + "sage.categories.partially_ordered_monoids": { + "ntests": 4 + }, + "sage.categories.permutation_groups": { + "ntests": 6 + }, + "sage.categories.pointed_sets": { + "ntests": 3 + }, + "sage.categories.polyhedra": { + "failed": true, + "ntests": 4 + }, + "sage.categories.poor_man_map": { + "failed": true, + "ntests": 59 + }, + "sage.categories.posets": { + "ntests": 2 + }, + "sage.categories.primer": { + "failed": true, + "ntests": 162 + }, + "sage.categories.principal_ideal_domains": { + "failed": true, + "ntests": 11 + }, + "sage.categories.pushout": { + "failed": true, + "ntests": 624 + }, + "sage.categories.quantum_group_representations": { + "failed": true, + "ntests": 13 + }, + "sage.categories.quotient_fields": { + "failed": true, + "ntests": 81 + }, + "sage.categories.quotients": { + "ntests": 2 + }, + "sage.categories.r_trivial_semigroups": { + "ntests": 3 + }, + "sage.categories.realizations": { + "failed": true, + "ntests": 21 + }, + "sage.categories.right_modules": { + "failed": true, + "ntests": 4 + }, + "sage.categories.ring_ideals": { + "failed": true, + "ntests": 9 + }, + "sage.categories.rings": { + "failed": true, + "ntests": 141 + }, + "sage.categories.rngs": { + "ntests": 6 + }, + "sage.categories.schemes": { + "failed": true, + "ntests": 23 + }, + "sage.categories.semigroups": { + "failed": true, + "ntests": 112 + }, + "sage.categories.semirings": { + "ntests": 6 + }, + "sage.categories.semisimple_algebras": { + "failed": true, + "ntests": 15 + }, + "sage.categories.sets_cat": { + "failed": true, + "ntests": 412 + }, + "sage.categories.sets_with_grading": { + "failed": true, + "ntests": 23 + }, + "sage.categories.sets_with_partial_maps": { + "ntests": 4 + }, + "sage.categories.shephard_groups": { + "ntests": 5 + }, + "sage.categories.signed_tensor": { + "failed": true, + "ntests": 10 + }, + "sage.categories.simplicial_complexes": { + "ntests": 17 + }, + "sage.categories.simplicial_sets": { + "failed": true, + "ntests": 57 + }, + "sage.categories.subobjects": { + "ntests": 2 + }, + "sage.categories.super_algebras": { + "failed": true, + "ntests": 8 + }, + "sage.categories.super_algebras_with_basis": { + "failed": true, + "ntests": 10 + }, + "sage.categories.super_hopf_algebras_with_basis": { + "failed": true, + "ntests": 10 + }, + "sage.categories.super_lie_conformal_algebras": { + "failed": true, + "ntests": 20 + }, + "sage.categories.super_modules": { + "failed": true, + "ntests": 18 + }, + "sage.categories.super_modules_with_basis": { + "failed": true, + "ntests": 10 + }, + "sage.categories.supercommutative_algebras": { + "failed": true, + "ntests": 8 + }, + "sage.categories.tensor": { + "failed": true, + "ntests": 9 + }, + "sage.categories.topological_spaces": { + "ntests": 27 + }, + "sage.categories.triangular_kac_moody_algebras": { + "failed": true, + "ntests": 15 + }, + "sage.categories.tutorial": { + "failed": true, + "ntests": 4 + }, + "sage.categories.unique_factorization_domains": { + "failed": true, + "ntests": 39 + }, + "sage.categories.unital_algebras": { + "failed": true, + "ntests": 39 + }, + "sage.categories.vector_spaces": { + "failed": true, + "ntests": 44 + }, + "sage.categories.with_realizations": { + "failed": true, + "ntests": 24 + }, + "sage.cpython.atexit": { + "ntests": 19 + }, + "sage.cpython.cython_metaclass": { + "ntests": 4 + }, + "sage.cpython.debug": { + "failed": true, + "ntests": 14 + }, + "sage.cpython.dict_del_by_value": { + "failed": true, + "ntests": 21 + }, + "sage.cpython.getattr": { + "failed": true, + "ntests": 65 + }, + "sage.cpython.string": { + "ntests": 1 + }, + "sage.cpython.string.pxd": { + "ntests": 8 + }, + "sage.cpython.type": { + "ntests": 7 + }, + "sage.cpython.wrapperdescr": { + "failed": true, + "ntests": 0 + }, + "sage.doctest.control": { + "failed": true, + "ntests": 0 + }, + "sage.doctest.external": { + "ntests": 42 + }, + "sage.doctest.fixtures": { + "failed": true, + "ntests": 59 + }, + "sage.doctest.forker": { + "failed": true, + "ntests": 433 + }, + "sage.doctest.parsing": { + "failed": true, + "ntests": 321 + }, + "sage.doctest.reporting": { + "failed": true, + "ntests": 124 + }, + "sage.doctest.sources": { + "failed": true, + "ntests": 378 + }, + "sage.doctest.test": { + "failed": true, + "ntests": 23 + }, + "sage.doctest.util": { + "failed": true, + "ntests": 141 + }, + "sage.env": { + "failed": true, + "ntests": 41 + }, + "sage.features": { + "ntests": 145 + }, + "sage.features.all": { + "ntests": 14 + }, + "sage.features.bliss": { + "ntests": 8 + }, + "sage.features.cddlib": { + "ntests": 4 + }, + "sage.features.csdp": { + "ntests": 6 + }, + "sage.features.cython": { + "ntests": 3 + }, + "sage.features.databases": { + "failed": true, + "ntests": 26 + }, + "sage.features.dvipng": { + "ntests": 4 + }, + "sage.features.ffmpeg": { + "ntests": 4 + }, + "sage.features.four_ti_2": { + "ntests": 6 + }, + "sage.features.gap": { + "ntests": 6 + }, + "sage.features.gfan": { + "ntests": 2 + }, + "sage.features.graph_generators": { + "ntests": 18 + }, + "sage.features.graphviz": { + "ntests": 16 + }, + "sage.features.igraph": { + "ntests": 4 + }, + "sage.features.imagemagick": { + "ntests": 10 + }, + "sage.features.interfaces": { + "failed": true, + "ntests": 33 + }, + "sage.features.internet": { + "ntests": 5 + }, + "sage.features.join_feature": { + "ntests": 25 + }, + "sage.features.kenzo": { + "ntests": 6 + }, + "sage.features.latex": { + "ntests": 30 + }, + "sage.features.latte": { + "ntests": 8 + }, + "sage.features.lrs": { + "ntests": 16 + }, + "sage.features.mcqd": { + "ntests": 4 + }, + "sage.features.meataxe": { + "ntests": 4 + }, + "sage.features.mip_backends": { + "ntests": 7 + }, + "sage.features.msolve": { + "ntests": 6 + }, + "sage.features.nauty": { + "ntests": 8 + }, + "sage.features.normaliz": { + "ntests": 4 + }, + "sage.features.palp": { + "ntests": 4 + }, + "sage.features.pandoc": { + "ntests": 4 + }, + "sage.features.pdf2svg": { + "ntests": 4 + }, + "sage.features.phitigra": { + "ntests": 4 + }, + "sage.features.pkg_systems": { + "ntests": 25 + }, + "sage.features.polymake": { + "ntests": 4 + }, + "sage.features.poppler": { + "ntests": 4 + }, + "sage.features.rubiks": { + "ntests": 28 + }, + "sage.features.sagemath": { + "failed": true, + "ntests": 147 + }, + "sage.features.singular": { + "ntests": 4 + }, + "sage.features.sphinx": { + "ntests": 4 + }, + "sage.features.tdlib": { + "ntests": 2 + }, + "sage.misc.abstract_method": { + "failed": true, + "ntests": 33 + }, + "sage.misc.banner": { + "failed": true, + "ntests": 12 + }, + "sage.misc.bindable_class": { + "ntests": 47 + }, + "sage.misc.c3_controlled": { + "failed": true, + "ntests": 150 + }, + "sage.misc.cachefunc": { + "failed": true, + "ntests": 692 + }, + "sage.misc.call": { + "ntests": 28 + }, + "sage.misc.classcall_metaclass": { + "ntests": 78 + }, + "sage.misc.constant_function": { + "ntests": 21 + }, + "sage.misc.decorators": { + "failed": true, + "ntests": 126 + }, + "sage.misc.fast_methods": { + "failed": true, + "ntests": 80 + }, + "sage.misc.flatten": { + "failed": true, + "ntests": 15 + }, + "sage.misc.function_mangling": { + "ntests": 32 + }, + "sage.misc.inherit_comparison": { + "ntests": 2 + }, + "sage.misc.instancedoc": { + "ntests": 67 + }, + "sage.misc.lazy_attribute": { + "failed": true, + "ntests": 100 + }, + "sage.misc.lazy_format": { + "failed": true, + "ntests": 23 + }, + "sage.misc.lazy_import": { + "failed": true, + "ntests": 279 + }, + "sage.misc.lazy_import_cache": { + "failed": true, + "ntests": 8 + }, + "sage.misc.lazy_string": { + "failed": true, + "ntests": 131 + }, + "sage.misc.misc": { + "failed": true, + "ntests": 176 + }, + "sage.misc.misc_c": { + "failed": true, + "ntests": 121 + }, + "sage.misc.namespace_package": { + "ntests": 7 + }, + "sage.misc.nested_class": { + "ntests": 67 + }, + "sage.misc.package": { + "ntests": 36 + }, + "sage.misc.package_dir": { + "failed": true, + "ntests": 29 + }, + "sage.misc.persist": { + "failed": true, + "ntests": 137 + }, + "sage.misc.prandom": { + "failed": true, + "ntests": 74 + }, + "sage.misc.repr": { + "failed": true, + "ntests": 31 + }, + "sage.misc.sage_eval": { + "failed": true, + "ntests": 31 + }, + "sage.misc.sage_input": { + "failed": true, + "ntests": 732 + }, + "sage.misc.sage_unittest": { + "failed": true, + "ntests": 88 + }, + "sage.misc.sagedoc": { + "failed": true, + "ntests": 103 + }, + "sage.misc.sageinspect": { + "failed": true, + "ntests": 329 + }, + "sage.misc.superseded": { + "failed": true, + "ntests": 59 + }, + "sage.misc.temporary_file": { + "failed": true, + "ntests": 80 + }, + "sage.misc.timing": { + "failed": true, + "ntests": 35 + }, + "sage.misc.unknown": { + "ntests": 22 + }, + "sage.misc.verbose": { + "ntests": 22 + }, + "sage.misc.viewer": { + "failed": true, + "ntests": 49 + }, + "sage.misc.weak_dict": { + "failed": true, + "ntests": 251 + }, + "sage.repl.attach": { + "failed": true, + "ntests": 128 + }, + "sage.repl.configuration": { + "ntests": 22 + }, + "sage.repl.display.fancy_repr": { + "failed": true, + "ntests": 27 + }, + "sage.repl.display.formatter": { + "failed": true, + "ntests": 58 + }, + "sage.repl.display.jsmol_iframe": { + "ntests": 25 + }, + "sage.repl.display.pretty_print": { + "failed": true, + "ntests": 21 + }, + "sage.repl.display.util": { + "ntests": 7 + }, + "sage.repl.image": { + "failed": true, + "ntests": 42 + }, + "sage.repl.inputhook": { + "ntests": 7 + }, + "sage.repl.interface_magic": { + "failed": true, + "ntests": 20 + }, + "sage.repl.interpreter": { + "failed": true, + "ntests": 118 + }, + "sage.repl.ipython_extension": { + "failed": true, + "ntests": 80 + }, + "sage.repl.ipython_kernel.install": { + "failed": true, + "ntests": 40 + }, + "sage.repl.ipython_kernel.interact": { + "failed": true, + "ntests": 42 + }, + "sage.repl.ipython_kernel.kernel": { + "ntests": 12 + }, + "sage.repl.ipython_kernel.widgets": { + "failed": true, + "ntests": 98 + }, + "sage.repl.ipython_kernel.widgets_sagenb": { + "failed": true, + "ntests": 77 + }, + "sage.repl.ipython_tests": { + "failed": true, + "ntests": 26 + }, + "sage.repl.load": { + "failed": true, + "ntests": 42 + }, + "sage.repl.preparse": { + "failed": true, + "ntests": 349 + }, + "sage.repl.rich_output.backend_base": { + "failed": true, + "ntests": 96 + }, + "sage.repl.rich_output.backend_doctest": { + "ntests": 58 + }, + "sage.repl.rich_output.backend_emacs": { + "ntests": 15 + }, + "sage.repl.rich_output.backend_ipython": { + "failed": true, + "ntests": 78 + }, + "sage.repl.rich_output.buffer": { + "failed": true, + "ntests": 49 + }, + "sage.repl.rich_output.display_manager": { + "failed": true, + "ntests": 88 + }, + "sage.repl.rich_output.output_basic": { + "ntests": 47 + }, + "sage.repl.rich_output.output_browser": { + "ntests": 12 + }, + "sage.repl.rich_output.output_graphics": { + "ntests": 38 + }, + "sage.repl.rich_output.output_graphics3d": { + "ntests": 46 + }, + "sage.repl.rich_output.output_video": { + "ntests": 25 + }, + "sage.repl.rich_output.preferences": { + "ntests": 68 + }, + "sage.repl.rich_output.pretty_print": { + "failed": true, + "ntests": 41 + }, + "sage.repl.rich_output.test_backend": { + "failed": true, + "ntests": 37 + }, + "sage.rings.ideal": { + "failed": true, + "ntests": 336 + }, + "sage.rings.integer_fake.pxd": { + "ntests": 1 + }, + "sage.rings.ring": { + "failed": true, + "ntests": 337 + }, + "sage.sets.pythonclass": { + "failed": true, + "ntests": 55 + }, + "sage.structure.category_object": { + "failed": true, + "ntests": 140 + }, + "sage.structure.coerce": { + "failed": true, + "ntests": 314 + }, + "sage.structure.coerce_actions": { + "failed": true, + "ntests": 128 + }, + "sage.structure.coerce_dict": { + "failed": true, + "ntests": 289 + }, + "sage.structure.coerce_maps": { + "failed": true, + "ntests": 90 + }, + "sage.structure.debug_options": { + "ntests": 5 + }, + "sage.structure.dynamic_class": { + "failed": true, + "ntests": 83 + }, + "sage.structure.element": { + "failed": true, + "ntests": 562 + }, + "sage.structure.element.pxd": { + "failed": true, + "ntests": 23 + }, + "sage.structure.element_wrapper": { + "failed": true, + "ntests": 160 + }, + "sage.structure.factorization": { + "failed": true, + "ntests": 203 + }, + "sage.structure.factorization_integer": { + "failed": true, + "ntests": 6 + }, + "sage.structure.factory": { + "failed": true, + "ntests": 99 + }, + "sage.structure.formal_sum": { + "failed": true, + "ntests": 67 + }, + "sage.structure.global_options": { + "ntests": 145 + }, + "sage.structure.graphics_file": { + "ntests": 8 + }, + "sage.structure.indexed_generators": { + "failed": true, + "ntests": 90 + }, + "sage.structure.list_clone": { + "failed": true, + "ntests": 380 + }, + "sage.structure.list_clone_demo": { + "ntests": 43 + }, + "sage.structure.list_clone_timings": { + "ntests": 18 + }, + "sage.structure.list_clone_timings_cy": { + "ntests": 12 + }, + "sage.structure.mutability": { + "failed": true, + "ntests": 68 + }, + "sage.structure.nonexact": { + "failed": true, + "ntests": 11 + }, + "sage.structure.parent": { + "failed": true, + "ntests": 300 + }, + "sage.structure.parent_gens": { + "failed": true, + "ntests": 24 + }, + "sage.structure.parent_old": { + "failed": true, + "ntests": 5 + }, + "sage.structure.proof.all": { + "ntests": 31 + }, + "sage.structure.proof.proof": { + "ntests": 50 + }, + "sage.structure.richcmp": { + "failed": true, + "ntests": 56 + }, + "sage.structure.richcmp.pxd": { + "ntests": 24 + }, + "sage.structure.sage_object": { + "failed": true, + "ntests": 84 + }, + "sage.structure.sequence": { + "failed": true, + "ntests": 183 + }, + "sage.structure.set_factories": { + "failed": true, + "ntests": 225 + }, + "sage.structure.set_factories_example": { + "failed": true, + "ntests": 81 + }, + "sage.structure.support_view": { + "ntests": 38 + }, + "sage.structure.test_factory": { + "failed": true, + "ntests": 6 + }, + "sage.structure.unique_representation": { + "failed": true, + "ntests": 226 + }, + "sage.typeset.ascii_art": { + "failed": true, + "ntests": 24 + }, + "sage.typeset.character_art": { + "failed": true, + "ntests": 108 + }, + "sage.typeset.character_art_factory": { + "failed": true, + "ntests": 53 + }, + "sage.typeset.symbols": { + "failed": true, + "ntests": 28 + }, + "sage.typeset.unicode_art": { + "failed": true, + "ntests": 18 + }, + "sage.typeset.unicode_characters": { + "ntests": 27 + } +} \ No newline at end of file diff --git a/pkgs/sagemath-categories/tox.ini b/pkgs/sagemath-categories/tox.ini index c7dbc3fb5dd..5469be55b3c 100644 --- a/pkgs/sagemath-categories/tox.ini +++ b/pkgs/sagemath-categories/tox.ini @@ -60,7 +60,7 @@ commands = {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()' bash -c 'cd $(python -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") \ - && sage-runtests -p --initial --environment=sage.all__sagemath_categories --optional=sage sage/structure || echo "(lots of doctest failures are expected)"' + && sage-runtests -p --initial --environment=sage.all__sagemath_categories --probe all --baseline-stats-path={toxinidir}/known-test-failures.json --optional=sage --installed' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version diff --git a/pkgs/sagemath-environment/tox.ini b/pkgs/sagemath-environment/tox.ini index 65bcdc902e0..ff2eef44f66 100644 --- a/pkgs/sagemath-environment/tox.ini +++ b/pkgs/sagemath-environment/tox.ini @@ -8,7 +8,7 @@ # [tox] envlist = - sagepython-norequirements + sagepython-sagewheels-nopypi-norequirements requires = # Auto-provision a modern tox. @@ -85,10 +85,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python package_env = .pkg-sagepython @@ -96,3 +92,8 @@ package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi diff --git a/pkgs/sagemath-objects/tox.ini b/pkgs/sagemath-objects/tox.ini index e5bf1e6dac7..0ca92ea9570 100644 --- a/pkgs/sagemath-objects/tox.ini +++ b/pkgs/sagemath-objects/tox.ini @@ -8,7 +8,7 @@ # [tox] envlist = - sagepython-norequirements + sagepython-sagewheels-nopypi-norequirements requires = # Auto-provision a modern tox. @@ -89,10 +89,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python package_env = .pkg-sagepython @@ -100,3 +96,8 @@ package_env = .pkg-sagepython [testenv:sagepython-norequirements] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi diff --git a/pkgs/sagemath-repl/known-test-failures.json b/pkgs/sagemath-repl/known-test-failures.json new file mode 100644 index 00000000000..6703c0a50cf --- /dev/null +++ b/pkgs/sagemath-repl/known-test-failures.json @@ -0,0 +1,165 @@ +{ + "sage.doctest.control": { + "failed": true, + "ntests": 0 + }, + "sage.doctest.external": { + "ntests": 41 + }, + "sage.doctest.fixtures": { + "failed": true, + "ntests": 59 + }, + "sage.doctest.forker": { + "failed": true, + "ntests": 432 + }, + "sage.doctest.parsing": { + "failed": true, + "ntests": 316 + }, + "sage.doctest.reporting": { + "failed": true, + "ntests": 124 + }, + "sage.doctest.sources": { + "failed": true, + "ntests": 376 + }, + "sage.doctest.test": { + "ntests": 23 + }, + "sage.doctest.util": { + "failed": true, + "ntests": 141 + }, + "sage.misc.sage_eval": { + "failed": true, + "ntests": 28 + }, + "sage.misc.sage_input": { + "failed": true, + "ntests": 721 + }, + "sage.repl.attach": { + "ntests": 128 + }, + "sage.repl.configuration": { + "ntests": 22 + }, + "sage.repl.display.fancy_repr": { + "failed": true, + "ntests": 24 + }, + "sage.repl.display.formatter": { + "failed": true, + "ntests": 47 + }, + "sage.repl.display.jsmol_iframe": { + "ntests": 24 + }, + "sage.repl.display.pretty_print": { + "failed": true, + "ntests": 21 + }, + "sage.repl.display.util": { + "ntests": 6 + }, + "sage.repl.image": { + "failed": true, + "ntests": 42 + }, + "sage.repl.inputhook": { + "ntests": 7 + }, + "sage.repl.interface_magic": { + "failed": true, + "ntests": 11 + }, + "sage.repl.interpreter": { + "failed": true, + "ntests": 109 + }, + "sage.repl.ipython_extension": { + "failed": true, + "ntests": 72 + }, + "sage.repl.ipython_kernel.install": { + "failed": true, + "ntests": 40 + }, + "sage.repl.ipython_kernel.interact": { + "failed": true, + "ntests": 35 + }, + "sage.repl.ipython_kernel.kernel": { + "ntests": 12 + }, + "sage.repl.ipython_kernel.widgets": { + "failed": true, + "ntests": 91 + }, + "sage.repl.ipython_kernel.widgets_sagenb": { + "failed": true, + "ntests": 77 + }, + "sage.repl.ipython_tests": { + "failed": true, + "ntests": 20 + }, + "sage.repl.load": { + "failed": true, + "ntests": 41 + }, + "sage.repl.preparse": { + "failed": true, + "ntests": 343 + }, + "sage.repl.rich_output.backend_base": { + "failed": true, + "ntests": 96 + }, + "sage.repl.rich_output.backend_doctest": { + "ntests": 50 + }, + "sage.repl.rich_output.backend_emacs": { + "ntests": 15 + }, + "sage.repl.rich_output.backend_ipython": { + "failed": true, + "ntests": 75 + }, + "sage.repl.rich_output.buffer": { + "ntests": 49 + }, + "sage.repl.rich_output.display_manager": { + "failed": true, + "ntests": 85 + }, + "sage.repl.rich_output.output_basic": { + "ntests": 46 + }, + "sage.repl.rich_output.output_browser": { + "ntests": 12 + }, + "sage.repl.rich_output.output_graphics": { + "ntests": 38 + }, + "sage.repl.rich_output.output_graphics3d": { + "ntests": 46 + }, + "sage.repl.rich_output.output_video": { + "ntests": 25 + }, + "sage.repl.rich_output.preferences": { + "ntests": 68 + }, + "sage.repl.rich_output.pretty_print": { + "failed": true, + "ntests": 22 + }, + "sage.repl.rich_output.test_backend": { + "failed": true, + "ntests": 37 + } +} \ No newline at end of file diff --git a/pkgs/sagemath-repl/tox.ini b/pkgs/sagemath-repl/tox.ini index 1637788be39..6944df002f5 100644 --- a/pkgs/sagemath-repl/tox.ini +++ b/pkgs/sagemath-repl/tox.ini @@ -56,7 +56,7 @@ commands = # Beware of the treacherous non-src layout. "./sage/" shadows the installed sage package. {envpython} -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.repl.all; import sage.doctest.all' - bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py || echo "(lots of doctest failures are expected)"' + bash -c 'cd $({envpython} -c "import sys; \"\" in sys.path and sys.path.remove(\"\"); from sage.env import SAGE_LIB; print(SAGE_LIB)") && sage-runtests -p --environment=sage.all__sagemath_repl --baseline-stats-path={toxinidir}/known-test-failures.json --initial --optional=sage sage/repl sage/doctest sage/misc/sage_input.py sage/misc/sage_eval.py' [testenv:.tox] # Allow access to PyPI for auto-provisioning a suitable tox version diff --git a/pkgs/sagemath-standard/tox.ini b/pkgs/sagemath-standard/tox.ini index 0430bcda5cc..c14f97f8bbf 100644 --- a/pkgs/sagemath-standard/tox.ini +++ b/pkgs/sagemath-standard/tox.ini @@ -16,23 +16,20 @@ # [tox] envlist = - # - # SUPPORTED ENVIRONMENTS: - # - # Build dependencies according to requirements.txt (all versions fixed). - # Use ONLY the wheels built and stored by the Sage distribution (no PyPI): - # - # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi)' - # - sagepython-sagewheels-nopypi, - # # Build and test without using the concrete dependencies specified by requirements.txt, # using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only: # Still use ONLY the wheels built and stored by the Sage distribution (no PyPI). # # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)' # - sagepython-sagewheels-nopypi-norequirements, + sagepython-sagewheels-nopypi-norequirements + # + # OTHER SUPPORTED ENVIRONMENTS: + # + # Build dependencies according to requirements.txt (all versions fixed). + # Use ONLY the wheels built and stored by the Sage distribution (no PyPI): + # + # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi)' # # EXPERIMENTAL ENVIRONMENTS: # @@ -43,12 +40,10 @@ envlist = # # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels)' # - sagepython-sagewheels, - # # Likewise, but using pipenv using Pipfile-dist (= SAGE_ROOT/Pipfile). # This also fixes the concrete dependencies (at least for some packages). # - sagepython-sagewheels-pipenv-dist, + # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-pipenv-dist)' # # Build using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only. # Use the wheels built and stored by the Sage distribution, @@ -57,11 +52,12 @@ envlist = # Because the version ranges will allow for packages to come in from PyPI (in source or wheel form), # this is likely to fail because we do not have control over the configuration of these packages. # - sagepython-sagewheels-norequirements, + # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-norequirements)' # # Likewise, but using pipenv # - sagepython-sagewheels-pipenv + # ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-pipenv)' + # requires = # Auto-provision a modern tox. @@ -162,10 +158,6 @@ package_env = .pkg-sagepython basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython-sagewheels-nopypi -[testenv:sagepython-sagewheels-nopypi-norequirements] -basepython = {env:SAGE_VENV}/bin/python3 -package_env = .pkg-sagepython-sagewheels-nopypi - [testenv:sagepython-sagewheels] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython @@ -181,3 +173,8 @@ package_env = .pkg-sagepython [testenv:sagepython-sagewheels-pipenv] basepython = {env:SAGE_VENV}/bin/python3 package_env = .pkg-sagepython + + +[testenv:sagepython-sagewheels-nopypi-norequirements] +basepython = {env:SAGE_VENV}/bin/python3 +package_env = .pkg-sagepython-sagewheels-nopypi diff --git a/src/bin/sage-fixdoctests b/src/bin/sage-fixdoctests index 50387881ad3..49de702cd51 100755 --- a/src/bin/sage-fixdoctests +++ b/src/bin/sage-fixdoctests @@ -31,13 +31,14 @@ AUTHORS:: # **************************************************************************** import itertools +import json import os import re import shlex import subprocess import sys -from argparse import ArgumentParser, FileType +from argparse import ArgumentParser from pathlib import Path from sage.doctest.control import DocTestDefaults, DocTestController @@ -45,13 +46,16 @@ from sage.doctest.parsing import parse_file_optional_tags, parse_optional_tags, from sage.env import SAGE_ROOT from sage.features import PythonModule from sage.features.all import all_features, module_feature, name_feature +from sage.misc.cachefunc import cached_function from sage.misc.temporary_file import tmp_filename parser = ArgumentParser(description="Given an input file with doctests, this creates a modified file that passes the doctests (modulo any raised exceptions). By default, the input file is modified. You can also name an output file.") parser.add_argument('-l', '--long', dest='long', action="store_true", default=False, help="include tests tagged '# long time'") -parser.add_argument("--distribution", type=str, default='', - help="distribution package to test, e.g., 'sagemath-graphs', 'sagemath-combinat[modules]'; sets defaults for --venv and --environment") +parser.add_argument("--distribution", type=str, default=[], action='append', + help="distribution package to test, e.g., 'sagemath-graphs', 'sagemath-combinat[modules]'; sets defaults for --venv and --environment. This option can be repeated to test several distributions") +parser.add_argument("--fixed-point", default=False, action="store_true", + help="whether to repeat until stable") parser.add_argument("--venv", type=str, default='', help="directory name of a venv where 'sage -t' is to be run") parser.add_argument("--environment", type=str, default='', @@ -63,31 +67,37 @@ parser.add_argument("--full-tracebacks", default=False, action="store_true", parser.add_argument("--only-tags", default=False, action="store_true", help="only add '# optional/needs' tags where needed, ignore other failures") parser.add_argument("--probe", metavar="FEATURES", type=str, default='', - help="check whether '# optional/needs' tags are still needed, remove these") + help="check whether '# optional/needs' tags are still needed, remove those not needed") parser.add_argument("--keep-both", default=False, action="store_true", help="do not replace test results; duplicate the test instead, showing both results, and mark both copies '# optional'") parser.add_argument("--overwrite", default=False, action="store_true", help="never interpret a second filename as OUTPUT; overwrite the source files") parser.add_argument("--no-overwrite", default=False, action="store_true", help="never interpret a second filename as OUTPUT; output goes to files named INPUT.fixed") +parser.add_argument("--update-known-test-failures", default=False, action="store_true", + help="update the file pkgs/DISTRIBUTION/known-test-failures.json") +parser.add_argument("--verbose", default=False, action="store_true", + help="show details of all changes; implies --no-diff") +parser.add_argument("--no-diff", default=False, action="store_true", + help="don't show the 'git diff' of the modified files") parser.add_argument("filename", nargs='*', help="input filenames; or (deprecated) INPUT_FILENAME OUTPUT_FILENAME if exactly two filenames are given and neither --overwrite nor --no-overwrite is present", type=str) -args = parser.parse_args() - - runtest_default_environment = "sage.repl.ipython_kernel.all_jupyter" -def default_venv_environment_from_distribution(): - if args.distribution: - # shortcuts / variants - args.distribution = args.distribution.replace('_', '-') - if not (args.distribution.startswith('sagemath-') - or args.distribution.startswith('sage-')): - args.distribution = f'sagemath-{args.distribution}' - # extras - m = re.fullmatch(r'([^[]*)(\[([^]]*)\])?', args.distribution) - plain_distribution, extras = m.group(1), m.group(3) +def plain_distribution_and_extras(distribution): + # shortcuts / variants + distribution = distribution.replace('_', '-') + if not (distribution.startswith('sagemath-') + or distribution.startswith('sage-')): + distribution = f'sagemath-{distribution}' + # extras + m = re.fullmatch(r'([^[]*)(\[([^]]*)\])?', distribution) + return m.group(1), m.group(3) + +def default_venv_environment_from_distribution(distribution): + if distribution: + plain_distribution, extras = plain_distribution_and_extras(distribution) tox_env_name = 'sagepython-sagewheels-nopypi-norequirements' if extras: tox_env_name += '-' + extras.replace(',', '-') @@ -98,56 +108,83 @@ def default_venv_environment_from_distribution(): default_environment = runtest_default_environment return default_venv, default_environment -default_venv, default_environment = default_venv_environment_from_distribution() -if not args.venv: - args.venv = default_venv -if not args.environment: - args.environment = default_environment +@cached_function +def venv_explainer(distribution, venv=None, environment=None): + venv_explainers = [] + default_venv, default_environment = default_venv_environment_from_distribution(distribution) + if venv: + if m := re.search(f'pkgs/(sage[^/]*)/[.]tox/((sagepython|sagewheels|nopypi|norequirements)-*)*([^/]*)$', + venv): + distribution, extras = m.group(1), m.group(4) + if extras: + distribution += '[' + extras.replace('-', ',') + ']' + default_venv_given_distribution, default_environment_given_distribution = default_venv_environment_from_distribution(distribution) -if args.distribution or args.venv != default_venv or args.environment != default_environment: - args.keep_both = args.full_tracebacks = True + if (Path(venv).resolve() == Path(default_venv_given_distribution).resolve() + or not environment or environment == default_environment_given_distribution): + venv_explainers.append(f'--distribution {shlex.quote(distribution)}') + default_venv, default_environment = default_venv_given_distribution, default_environment_given_distribution -venv_explainers = [] + if venv and Path(venv).resolve() != Path(default_venv).resolve(): + venv_explainers.append(f'--venv {shlex.quote(venv)}') + if environment and environment != default_environment: + venv_explainers.append(f'--environment {environment}') -if args.venv: - if m := re.search(f'pkgs/(sage[^/]*)/[.]tox/((sagepython|sagewheels|nopypi|norequirements)-*)*([^/]*)$', - args.venv): - args.distribution, extras = m.group(1), m.group(4) - if extras: - args.distribution += '[' + extras.replace('-', ',') + ']' - default_venv_given_distribution, default_environment_given_distribution = default_venv_environment_from_distribution() + if venv_explainers: + return ' (with ' + ' '.join(venv_explainers) + ')' + return '' - if (Path(args.venv).resolve() == Path(default_venv_given_distribution).resolve() - or args.environment == default_environment_given_distribution): - venv_explainers.append(f'--distribution {shlex.quote(args.distribution)}') - default_venv, default_environment = default_venv_given_distribution, default_environment_given_distribution -if Path(args.venv).resolve() != Path(default_venv).resolve(): - venv_explainers.append(f'--venv {shlex.quote(args.venv)}') -if args.environment != default_environment: - venv_explainers.append(f'--environment {args.environment}') +sep = "**********************************************************************\n" -if venv_explainers: - venv_explainer = ' (with ' + ' '.join(venv_explainers) + ')' -else: - venv_explainer = '' +def process_block(block, src_in_lines, file_optional_tags, venv_explainer=''): + if args.verbose: + print(sep + block.rstrip()) -def process_block(block, src_in_lines, file_optional_tags): # Extract the line, what was expected, and was got. if not (m := re.match('File "([^"]*)", line ([0-9]+), in ', block)): return + + def print_line(num): + if args.verbose and (src := src_in_lines[num]): + if src: + for line in src.split('\n'): + line = line.strip() + if line.startswith("sage: ") or line.startswith("....: "): + line = line[6:] + print(f" {line}") # indent to match the displayed "Example" in the sage-runtest message + + def update_line(num, new, message=None): + src_in_lines[num] = new + if args.verbose and message: + print(f"sage-fixdoctests: {message}") + print_line(num) + + def append_to_line(num, new, message=None): + update_line(num, src_in_lines[num] + new, message=message) + + def prepend_to_line(num, new, message=None): + update_line(num, new + src_in_lines[num], message=message) + + def update_line_optional_tags(num, *args, message=None, **kwds): + update_line(num, + update_optional_tags(src_in_lines[num], *args, **kwds), + message=message) + filename = m.group(1) first_line_num = line_num = int(m.group(2)) # 1-based line number of the first line of the example if m := re.search(r"using.*block-scoped tag.*'(sage: .*)'.*to avoid repeating the tag", block): indent = (len(src_in_lines[first_line_num - 1]) - len(src_in_lines[first_line_num - 1].lstrip())) - src_in_lines[line_num - 2] += '\n' + ' ' * indent + m.group(1) + append_to_line(line_num - 2, '\n' + ' ' * indent + m.group(1), + message="Adding this block-scoped tag") + print_line(first_line_num - 1) if m := re.search(r"updating.*block-scoped tag.*'sage: (.*)'.*to avoid repeating the tag", block): - src_in_lines[first_line_num - 1] = update_optional_tags(src_in_lines[first_line_num - 1], - tags=parse_optional_tags('# ' + m.group(1))) + update_line_optional_tags(first_line_num - 1, tags=parse_optional_tags('# ' + m.group(1)), + message="Adding this tag to the existing block-scoped tag") if m := re.search(r"referenced here was set only in doctest marked '# (optional|needs)[-: ]*([^;']*)", block): optional = m.group(2).split() @@ -155,13 +192,13 @@ def process_block(block, src_in_lines, file_optional_tags): # This happens due to a virtual doctest in src/sage/repl/user_globals.py return optional = set(optional) - set(file_optional_tags) - src_in_lines[first_line_num - 1] = update_optional_tags(src_in_lines[first_line_num - 1], - add_tags=optional) + update_line_optional_tags(first_line_num - 1, add_tags=optional, + message=f"Adding the tag(s) {optional}") if m := re.search(r"tag '# (optional|needs)[-: ]([^;']*)' may no longer be needed", block): optional = m.group(2).split() - src_in_lines[first_line_num - 1] = update_optional_tags(src_in_lines[first_line_num - 1], - remove_tags=optional) + update_line_optional_tags(first_line_num - 1, remove_tags=optional, + message=f"Removing the tag(s) {optional}") if m2 := re.search('(Expected:|Expected nothing|Exception raised:)\n', block): m1 = re.search('Failed example:\n', block) @@ -189,17 +226,21 @@ def process_block(block, src_in_lines, file_optional_tags): return # Error testing. - if m := re.search(r"(?:ModuleNotFoundError: No module named|ImportError: cannot import name '([^']*)' from) '([^']*)'", block): + if m := re.search(r"(?:ModuleNotFoundError: No module named|ImportError: cannot import name '(.*?)' from) '(.*?)'|AttributeError: module '(.*)?' has no attribute '(.*?)'", block): if m.group(1): # "ImportError: cannot import name 'function_field_polymod' from 'sage.rings.function_field' (unknown location)" module = m.group(2) + '.' + m.group(1) - else: + elif m.group(2): + # "ModuleNotFoundError: No module named ..." module = m.group(2) + else: + # AttributeError: module 'sage.rings' has no attribute 'qqbar' + module = m.group(3) + '.' + m.group(4) asked_why = re.search('#.*(why|explain)', src_in_lines[first_line_num - 1]) optional = module_feature(module) if optional and optional.name not in file_optional_tags: - src_in_lines[first_line_num - 1] = update_optional_tags(src_in_lines[first_line_num - 1], - add_tags=[optional.name]) + update_line_optional_tags(first_line_num - 1, add_tags=[optional.name], + message=f"Module '{module}' may be provided by feature '{optional.name}'; adding this tag") if not asked_why: # When no explanation has been demanded, # we just mark the doctest with the feature @@ -225,23 +266,30 @@ def process_block(block, src_in_lines, file_optional_tags): if (last_frame >= 0 and (index_NameError := got.rfind("NameError:")) >= 0 and got[last_frame:].startswith('File "= 2 and (feature := name_feature(name)) and feature.name != 'sage.all': + # Don't mark use of 'x' '# needs sage.symbolic'; that's almost always wrong + # Likewise for variables like 'R', 'r' + add_tags = [feature.name] # FIXME: This feature may actually already be present in line, block, or file. Move this lookup code into the doctester and issue more specific instructions + elif args.only_tags: + if args.verbose: + print("sage-fixdoctests: No feature providing this global is known; no action because of --only-tags") return - if feature := name_feature(name): - add_tags = [feature.name] else: - if args.only_tags: - return - add_tags = [f"NameError: '{name}'{venv_explainer}"] + add_tags = [f"NameError ('{name}', {venv_explainer.lstrip().lstrip('(')}"] else: if args.only_tags: + if args.verbose: + print("sage-fixdoctests: No feature providing this global is known; no action because of --only-tags") return add_tags = [f"NameError{venv_explainer}"] - src_in_lines[first_line_num - 1] = update_optional_tags(src_in_lines[first_line_num - 1], - add_tags=add_tags) + update_line_optional_tags(first_line_num - 1, add_tags=add_tags, + message=f"Adding tag {add_tags}") return got = got.splitlines() else: @@ -255,6 +303,8 @@ def process_block(block, src_in_lines, file_optional_tags): got = got.splitlines() # got can't be the empty string if args.only_tags: + if args.verbose: + print("sage-fixdoctests: No action because of --only-tags") return expected = expected.splitlines() @@ -263,11 +313,11 @@ def process_block(block, src_in_lines, file_optional_tags): test_lines = ([update_optional_tags(src_in_lines[first_line_num - 1], add_tags=[f'GOT{venv_explainer}'])] + src_in_lines[first_line_num : line_num]) - src_in_lines[first_line_num - 1] = update_optional_tags(src_in_lines[first_line_num - 1], - add_tags=['EXPECTED']) + update_line_optional_tags(first_line_num - 1, add_tags=['EXPECTED'], + message="Marking the doctest with idempotent tag EXPECTED, creating another copy with idempotent tag GOT") indent = (len(src_in_lines[line_num - 1]) - len(src_in_lines[line_num - 1].lstrip())) line_num += len(expected) # skip to the last line of the expected output - src_in_lines[line_num - 1] += '\n'.join([''] + test_lines) # 2nd copy of the test + append_to_line(line_num - 1, '\n'.join([''] + test_lines)) # 2nd copy of the test # now line_num is the last line of the 2nd copy of the test expected = [] @@ -275,7 +325,9 @@ def process_block(block, src_in_lines, file_optional_tags): # and match indentation with line number line_num-1 if not expected: indent = (len(src_in_lines[first_line_num - 1]) - len(src_in_lines[first_line_num - 1].lstrip())) - src_in_lines[line_num - 1] += '\n' + '\n'.join('%s%s' % (' ' * indent, line.lstrip()) for line in got) + append_to_line(line_num - 1, + '\n' + '\n'.join('%s%s' % (' ' * indent, line.lstrip()) for line in got), + message="Adding the new output") return # Guess how much extra indenting ``got`` needs to match with the indentation @@ -297,17 +349,18 @@ def process_block(block, src_in_lines, file_optional_tags): '<' * 40, '\n'.join(got), '<' * 40)) return - # If we got something when we expected nothing then we delete the line from the + # If we got nothing when we expected something then we delete the line from the # output, otherwise, add all of what we `got` onto the end of src_in_lines[line_num] if got == ['']: - src_in_lines[line_num] = None + update_line(line_num, None, + message="Expected something, got nothing; deleting the old output") else: - src_in_lines[line_num] = '\n'.join((' ' * indent + got[i]) - for i in range(len(got))) + update_line(line_num, '\n'.join((' ' * indent + got[i]) for i in range(len(got))), + message="Replacing the old expected output with the new output") # Mark any remaining `expected` lines as ``None`` so as to preserve the line numbering for i in range(1, len(expected)): - src_in_lines[line_num + i] = None + update_line(line_num + i, None) # set input and output files @@ -322,31 +375,36 @@ def output_filename(filename): return filename + ".fixed" return filename -# Test the doctester, putting the output of the test into sage's temporary directory -if args.no_test: - doc_out = '' -else: - executable = f'{os.path.relpath(args.venv)}/bin/sage' if args.venv else 'sage' - environment_args = f'--environment {args.environment} ' if args.environment != runtest_default_environment else '' + +tested_doctesters = set() +venv_files = {} # distribution -> files that are not yet known to be fixed points in venv; we add and remove items +venv_ignored_files = {} # distribution -> files that should be ignored; we only add items +unprocessed_files = set() + + +class BadDistribution(Exception): + pass + + +def doctest_blocks(args, input_filenames, distribution=None, venv=None, environment=None): + executable = f'{os.path.relpath(venv)}/bin/sage' if venv else 'sage' + environment_args = f'--environment {environment} ' if environment and environment != runtest_default_environment else '' long_args = f'--long ' if args.long else '' probe_args = f'--probe {shlex.quote(args.probe)} ' if args.probe else '' - lib_args = f'--if-installed ' if args.venv else '' + lib_args = f'--if-installed ' if venv else '' doc_file = tmp_filename() - if args.venv or environment_args: - input = os.path.join(os.path.relpath(SAGE_ROOT), 'src', 'sage', 'version.py') - cmdline = f'{shlex.quote(executable)} -t {environment_args}{long_args}{probe_args}{lib_args}{shlex.quote(input)}' - print(f'Running "{cmdline}"') - if status := os.waitstatus_to_exitcode(os.system(f'{cmdline} > {shlex.quote(doc_file)}')): - print(f'Doctester exited with error status {status}') - sys.exit(status) + if venv or environment_args: + # Test the doctester, putting the output of the test into sage's temporary directory + input = os.path.join(os.path.relpath(SAGE_ROOT), 'src', 'sage', 'version.py') + cmdline = f'{shlex.quote(executable)} -t {environment_args}{long_args}{probe_args}'.rstrip() + if cmdline not in tested_doctesters: + if args.verbose: + print(f'sage-fixdoctests: Checking whether the doctester "{cmdline}" works') + cmdline += f' {shlex.quote(input)}' + if status := os.waitstatus_to_exitcode(os.system(f'{cmdline} > {shlex.quote(doc_file)}')): + raise BadDistribution(f"Doctester exited with error status {status}") + tested_doctesters.add(cmdline) # Run the doctester, putting the output of the test into sage's temporary directory - if len(args.filename) == 2 and not args.overwrite and not args.no_overwrite: - print("sage-fixdoctests: When passing two filenames, the second one is taken as an output filename; " - "this is deprecated. To pass two input filenames, use the option --overwrite.") - - input_filenames = [args.filename[0]] - else: - input_filenames = args.filename input_args = " ".join(shlex.quote(f) for f in input_filenames) cmdline = f'{shlex.quote(executable)} -t -p {environment_args}{long_args}{probe_args}{lib_args}{input_args}' print(f'Running "{cmdline}"') @@ -355,20 +413,22 @@ else: with open(doc_file, 'r') as doc: doc_out = doc.read() - # echo control messages - for m in re.finditer('^Skipping .*', doc_out, re.MULTILINE): + # Remove skipped files, echo control messages + for m in re.finditer(r"^Skipping '(.*?)'.*$", doc_out, re.MULTILINE): print('sage-runtests: ' + m.group(0)) + if distribution is not None: + venv_files[distribution].discard(m.group(1)) + venv_ignored_files[distribution].add(m.group(1)) -sep = "**********************************************************************\n" -doctests = doc_out.split(sep) + return doc_out.split(sep) -seen = set() def block_filename(block): if not (m := re.match('File "([^"]*)", line ([0-9]+), in ', block)): return None return m.group(1) + def expanded_filename_args(): DD = DocTestDefaults(optional='all', warn_long=10000) DC = DocTestController(DD, input_filenames) @@ -377,7 +437,12 @@ def expanded_filename_args(): for source in DC.sources: yield source.path -def process_grouped_blocks(grouped_iterator): + +def process_grouped_blocks(grouped_iterator, distribution=None, venv=None, environment=None): + + seen = set() + + explainer = venv_explainer(distribution, venv, environment) for input, blocks in grouped_iterator: @@ -394,7 +459,11 @@ def process_grouped_blocks(grouped_iterator): file_optional_tags = set(parse_file_optional_tags(enumerate(src_in_lines))) for block in blocks: - process_block(block, src_in_lines, file_optional_tags) + try: + process_block(block, src_in_lines, file_optional_tags, venv_explainer=explainer) + except Exception: + print('sage-fixdoctests: Failure to process block') + print(block) # Now source line numbers do not matter any more, and lines can be real lines again src_in_lines = list(itertools.chain.from_iterable( @@ -411,7 +480,7 @@ def process_grouped_blocks(grouped_iterator): for tag, explanation in tags.items() if explanation or tag not in file_optional_tags} line = update_optional_tags(line, tags=persistent_optional_tags, force_rewrite='standard') - if not line.rstrip(): + if re.fullmatch(' *sage: *', line): # persistent (block-scoped or file-scoped) tag was removed, so remove the whole line line = None else: @@ -426,7 +495,7 @@ def process_grouped_blocks(grouped_iterator): if src_in_lines != shallow_copy_of_src_in_lines: if (output := output_filename(input)) is None: - print(f"Not saving modifications made in '{input}'") + print(f"sage-fixdoctests: Not saving modifications made in '{input}'") else: with open(output, 'w') as test_output: for line in src_in_lines: @@ -435,16 +504,156 @@ def process_grouped_blocks(grouped_iterator): test_output.write(line) test_output.write('\n') # Show summary of changes - if input != output : - print("The fixed doctests have been saved as '{0}'.".format(output)) + if input != output: + print("sage-fixdoctests: The fixed doctests have been saved as '{0}'.".format(output)) else: relative = os.path.relpath(output, SAGE_ROOT) - print(f"The input file '{output}' has been overwritten.") - if not relative.startswith('..'): + print(f"sage-fixdoctests: The input file '{output}' has been overwritten.") + if not args.no_diff and not relative.startswith('..'): subprocess.call(['git', '--no-pager', 'diff', relative], cwd=SAGE_ROOT) + for other_distribution, file_set in venv_files.items(): + if input not in venv_ignored_files[other_distribution]: + file_set.add(input) + else: + print(f"sage-fixdoctests: No fixes made in '{input}'") + if distribution is not None: + venv_files[distribution].discard(input) + + unprocessed_files.discard(input) + + +def fix_with_distribution(file_set, distribution=None, verbose=False): + if verbose: + print("#" * 78) + print(f"sage-fixdoctests: Fixing with --distribution={shlex.quote(distribution)}") + default_venv, default_environment = default_venv_environment_from_distribution(distribution) + venv = args.venv or default_venv + environment = args.environment or default_environment + file_set_to_process = sorted(file_set) + file_set.clear() + try: + doctests = doctest_blocks(args, file_set_to_process, + distribution=distribution, venv=venv, environment=environment) + process_grouped_blocks(itertools.groupby(doctests, block_filename), # modifies file_set + distribution=distribution, venv=venv, environment=environment) + except BadDistribution as e: + if args.ignore_bad_distributions: + print(f"sage-fixdoctests: {e}, ignoring") else: - print(f"No fixes made in '{input}'") + sys.exit(f"sage-fixdoctests: {e}") + + +if __name__ == "__main__": + + args = parser.parse_args() + + if args.verbose: + args.no_diff = True + + args.ignore_bad_distributions = False # This could also be a switch + + args.update_failures_distribution = args.distribution + + if args.distribution == ['all']: + args.distribution = ['sagemath-categories', + ''] # monolithic distribution + args.update_failures_distribution = args.distribution + ['sagemath-repl', # not included above because it knows too little and complains too much + ] + + args.ignore_bad_distributions = True + + if not args.filename: + if not args.update_known_test_failures: + sys.exit("sage-fixdoctests: At least one filename is required when --update-known-test-failures is not used") + if not args.distribution: + sys.exit("sage-fixdoctests: At least one --distribution argument is required for --update-known-test-failures") -process_grouped_blocks( - itertools.chain(itertools.groupby(doctests, block_filename), - ((filename, []) for filename in expanded_filename_args()))) + if args.distribution or args.venv or args.environment: + args.keep_both = args.full_tracebacks = True + + if len(args.distribution) > 1: + if args.venv or args.environment: + sys.exit("sage-fixdoctests: at most one --distribution argument can be combined with --venv and --environment") + elif not args.distribution: + args.distribution = [''] + + if len(args.filename) == 2 and not args.overwrite and not args.no_overwrite: + print("sage-fixdoctests: When passing two filenames, the second one is taken as an output filename; " + "this is deprecated. To pass two input filenames, use the option --overwrite.") + input_filenames = [args.filename[0]] + else: + input_filenames = args.filename + + try: + unprocessed_files = set(expanded_filename_args()) + for distribution in args.distribution: + venv_files[distribution] = set(unprocessed_files) # make copies + venv_ignored_files[distribution] = set() + if args.no_test: + pass + elif len(args.distribution) == 1 and not args.fixed_point: + fix_with_distribution(set(unprocessed_files), args.distribution[0]) + else: + for distribution, file_set in venv_files.items(): + fix_with_distribution(file_set, distribution, verbose=True) + if args.fixed_point: + if args.probe: + print(f"sage-fixdoctests: Turning off --probe for the following iterations") + # This forces convergence to a fixed point + args.probe = '' + while True: + # Run a distribution with largest number of files remaining to be checked + # because of the startup overhead of sage-runtests + distribution, file_set = max(venv_files.items(), key=lambda df: len(df[1])) + if not file_set: + break + while file_set: + fix_with_distribution(file_set, distribution, verbose=True) + # Immediately re-run with the same distribution to continue chains of + # "NameError" / "variable was set only in doctest" fixes + + # Each file must be processed by process_grouped_blocks at least once to clean up tags, + # even if sage-runtest does not have any complaints. + if unprocessed_files: + print(f"sage-fixdoctests: Processing unprocessed files") + process_grouped_blocks([(filename, []) + for filename in unprocessed_files]) + + if args.fixed_point: + print(f"sage-fixdoctests: Fixed point reached") + + if args.update_known_test_failures: + if args.update_failures_distribution == ['']: + print("sage-fixdoctests: Ignoring switch --update-known-test-failures because no --distribution was given") + else: + for distribution in sorted(args.update_failures_distribution): + if distribution == '': + continue + plain_distribution, extras = plain_distribution_and_extras(distribution) + default_venv, _ = default_venv_environment_from_distribution(distribution) + venv = args.venv or default_venv + try: + stats_filename = os.path.join(default_venv, '.sage/timings2.json') + with open(stats_filename, 'r') as stats_file: + stats = json.load(stats_file) + except FileNotFoundError: + print(f"sage-fixdoctests: {os.path.relpath(stats_filename, SAGE_ROOT)} " + "does not exist (ignoring)") + else: + for d in stats.values(): + del d['walltime'] + stats = {k: d for k, d in stats.items() + if d.get('failed') or d.get('ntests', True)} + if extras: + extras_suffix = '--' + '--'.join(extras.split(',')) + else: + extras_suffix = '' + failures_file = os.path.join(SAGE_ROOT, 'pkgs', plain_distribution, + f'known-test-failures{extras_suffix}.json') + with open(failures_file, 'w') as f: + json.dump(stats, f, sort_keys=True, indent=4) + print(f"sage-fixdoctests: Updated {os.path.relpath(failures_file, SAGE_ROOT)}") + + except Exception: + print(f"sage-fixdoctests: Internal error") + raise diff --git a/src/doc/en/developer/doctesting.rst b/src/doc/en/developer/doctesting.rst index db4fc7f9a83..589c9260134 100644 --- a/src/doc/en/developer/doctesting.rst +++ b/src/doc/en/developer/doctesting.rst @@ -1080,7 +1080,7 @@ under the control of gdb, use the ``--gdb`` flag:: [roed@localhost sage]$ ./sage -t --gdb \ src/sage/schemes/elliptic_curves/constructor.py - exec gdb --eval-commands="run" --args /home/roed/sage/local/var/lib/sage/venv-python3.9/bin/python3 sage-runtests --serial --timeout=0 --stats_path=/home/roed/.sage/timings2.json --optional=pip,sage,sage_spkg src/sage/schemes/elliptic_curves/constructor.py + exec gdb --eval-commands="run" --args /home/roed/sage/local/var/lib/sage/venv-python3.9/bin/python3 sage-runtests --serial --timeout=0 --stats-path=/home/roed/.sage/timings2.json --optional=pip,sage,sage_spkg src/sage/schemes/elliptic_curves/constructor.py GNU gdb 6.8-debian Copyright (C) 2008 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later @@ -1308,6 +1308,8 @@ code loads the globals from that file into the namespace before running tests. To disable this behaviour (and require imports to be explicitly specified), use the ``--force-lib`` option. +.. _section-doctest-auxiliary-files: + Auxiliary files ^^^^^^^^^^^^^^^ @@ -1338,10 +1340,11 @@ To specify a logfile (rather than use the default which is created for cumulative wall time: 4.3 seconds -To give a json file storing the timings for each file, use the -``--stats_path`` flag. These statistics are used in sorting files so -that slower tests are run first (and thus multiple processes are -utilized most efficiently):: +To give a json file storing the timings and pass/fail status for each file, use the +``--stats-path`` flag; the default location of this file is ``~/.sage/timings2.json``. +The doctester reads it if it exists, for the purpose of sorting the files +so that slower tests are run first (and thus multiple processes are utilized most +efficiently):: [roed@localhost sage]$ ./sage -tp 2 --stats-path ~/.sage/timings2.json --all Running doctests with ID 2012-07-07-01-28-34-2df4251d. @@ -1350,6 +1353,22 @@ utilized most efficiently):: Doctesting 2067 files using 2 threads. ... +At the end of the doctest run, Sage updates the json file if it exists or creates +a new one. + +The recorded pass/fail status of the files can be used for running only those files +that failed their most recent test by using the ``--failed`` flag (``-f`` for short). + +Using the option ``--baseline-stats-path known-test-failures.json``, +it is possible to distinguish files with known doctest failures +from new failures. The file ``known-test-failures.json`` should be +prepared in the same format as ``timings2.json``. + +Source files marked as failed there will be marked as "[failed in baseline]" +failures in the doctest report; and if there are only baseline failures, no +new failures, then ``sage -t`` will exit with status code 0 (success). + + .. _section-doctesting-venv: Options for testing in virtual environments @@ -1614,6 +1633,9 @@ To have the doctest fixer take care of the ``# optional/needs`` tags, but not change the expected results of examples, use the option ``--only-tags``. This mode is suitable for mostly unattended runs on many files. +With the option ``--verbose``, the doctest fixer shows the doctester's messages +one by one and reports the changes made. + .. warning:: While the doctest fixer guarantees to preserve any comments that @@ -1655,3 +1677,26 @@ to the doctest. Likewise, when the doctester runs into a :class:`ModuleNotFoundError`, the doctest fixer will automatically add a ``# needs ...`` tag. + +The switch ``--distribution`` can be repeated; the given distributions +will be tested in sequence. Using ``--distribution all`` is equivalent +to a preset list of ``--distribution`` switches. With the switch +``--fixed-point``, the doctest fixer runs the given distributions until +no more changes are made. + + +Updating baseline files +----------------------- + +The modularized distribution packages ``pkgs/sagemath-categories`` and +``pkgs/sagemath-repl`` contain files ``known-test-failures*.json`` for use +with the option ``--baseline-stats-path``, see section +:ref:`section-doctest-auxiliary-files`. + +After running the doctesters of the distributions, for example, via +``sage --fixdoctests``, you can use the test results stored in +``timings2.json`` files to update the ``known-test-failures*.json`` files. +This update can be done using the command:: + + [mkoeppe@localhost sage]$ ./sage --fixdoctests --no-test \ + --update-known-test-failures --distribution all diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index 66cedee9b75..11bf91a3aef 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -708,8 +708,8 @@ def load_baseline_stats(self, filename): try: with open(filename) as stats_file: self.baseline_stats.update(json.load(stats_file)) - except Exception: - self.log("Error loading baseline stats from %s"%filename) + except Exception as e: + self.log("Error loading baseline stats from %s: %s" % (filename, e)) def load_stats(self, filename): """ @@ -772,7 +772,7 @@ def save_stats(self, filename): """ from sage.misc.temporary_file import atomic_write with atomic_write(filename) as stats_file: - json.dump(self.stats, stats_file) + json.dump(self.stats, stats_file, sort_keys=True, indent=4) def log(self, s, end="\n"): """ @@ -1089,7 +1089,7 @@ def sort_sources(self): def sort_key(source): basename = source.basename - return -self.stats.get(basename, default).get('walltime'), basename + return -self.stats.get(basename, default).get('walltime', 0), basename self.sources = sorted(self.sources, key=sort_key) def run_doctests(self): diff --git a/src/sage/doctest/parsing.py b/src/sage/doctest/parsing.py index 19315eac6a2..3a2ac1b98d4 100644 --- a/src/sage/doctest/parsing.py +++ b/src/sage/doctest/parsing.py @@ -204,6 +204,9 @@ def parse_optional_tags(string, *, return_string_sans_tags=False): return {} first_line_sans_comments, comment = first_line[:sharp_index] % literals, first_line[sharp_index:] % literals + if not first_line_sans_comments.endswith(" ") and not first_line_sans_comments.rstrip().endswith("sage:"): + # Enforce two spaces before comment + first_line_sans_comments = first_line_sans_comments.rstrip() + " " if return_string_sans_tags: # skip non-tag comments that precede the first tag comment @@ -1215,6 +1218,12 @@ def check_and_clear_tag_counts(): if any(tag in external_software for tag in extra): # never probe "external" software continue + if any(tag in ['webbrowser'] for tag in extra): + # never probe + continue + if any(tag in ['got', 'expected', 'nameerror'] for tag in extra): + # never probe special tags added by sage-fixdoctests + continue if all(tag in persistent_optional_tags for tag in extra): # don't probe if test is only conditional # on file-level or block-level tags diff --git a/src/sage/doctest/reporting.py b/src/sage/doctest/reporting.py index c2b4537d596..361fea344c2 100644 --- a/src/sage/doctest/reporting.py +++ b/src/sage/doctest/reporting.py @@ -264,7 +264,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): Output so far... ********************************************************************** sage: DTR.stats - {'sage.doctest.reporting': {'failed': True, 'walltime': 1000000.0}} + {'sage.doctest.reporting': {'failed': True, + 'ntests': 0, + 'walltime': 1000000.0}} Or a process that returned a bad exit code:: @@ -275,7 +277,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): Output before trouble ********************************************************************** sage: DTR.stats - {'sage.doctest.reporting': {'failed': True, 'walltime': 1000000.0}} + {'sage.doctest.reporting': {'failed': True, + 'ntests': 0, + 'walltime': 1000000.0}} Or a process that segfaulted:: @@ -287,7 +291,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): Output before trouble ********************************************************************** sage: DTR.stats - {'sage.doctest.reporting': {'failed': True, 'walltime': 1000000.0}} + {'sage.doctest.reporting': {'failed': True, + 'ntests': 0, + 'walltime': 1000000.0}} Report a timeout with results and a ``SIGKILL``:: @@ -299,7 +305,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): Output before trouble ********************************************************************** sage: DTR.stats - {'sage.doctest.reporting': {'failed': True, 'walltime': 1000000.0}} + {'sage.doctest.reporting': {'failed': True, + 'ntests': 1, + 'walltime': 1000000.0}} This is an internal error since results is None:: @@ -310,12 +318,15 @@ def report(self, source, timeout, return_code, results, output, pid=None): All output ********************************************************************** sage: DTR.stats - {'sage.doctest.reporting': {'failed': True, 'walltime': 1000000.0}} + {'sage.doctest.reporting': {'failed': True, + 'ntests': 1, + 'walltime': 1000000.0}} Or tell the user that everything succeeded:: sage: doctests, extras = FDS.create_doctests(globals()) - sage: runner = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD, optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS) + sage: runner = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD, + ....: optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS) sage: Timer().start().stop().annotate(runner) sage: D = DictAsObject({'err':None}) sage: runner.update_results(D) @@ -323,7 +334,7 @@ def report(self, source, timeout, return_code, results, output, pid=None): sage: DTR.report(FDS, False, 0, (sum([len(t.examples) for t in doctests]), D), "Good tests") [... tests, ... s] sage: DTR.stats - {'sage.doctest.reporting': {'walltime': ...}} + {'sage.doctest.reporting': {'ntests': ..., 'walltime': ...}} Or inform the user that some doctests failed:: @@ -366,7 +377,8 @@ def report(self, source, timeout, return_code, results, output, pid=None): sage: DC = DocTestController(DD, [filename]) sage: DTR = DocTestReporter(DC) sage: doctests, extras = FDS.create_doctests(globals()) - sage: runner = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD, optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS) + sage: runner = SageDocTestRunner(SageOutputChecker(), verbose=False, sage_options=DD, + ....: optionflags=doctest.NORMALIZE_WHITESPACE|doctest.ELLIPSIS) sage: Timer().start().stop().annotate(runner) sage: D = DictAsObject({'err':None}) sage: runner.update_results(D) @@ -417,7 +429,7 @@ def report(self, source, timeout, return_code, results, output, pid=None): log(output) log("*"*70) postscript['lines'].append(cmd + " # %s"%fail_msg) - stats[basename] = dict(failed=True, walltime=1e6) + stats[basename] = dict(failed=True, walltime=1e6, ntests=ntests) if not the_baseline_stats.get('failed', False): self.error_status |= 4 elif return_code: @@ -433,7 +445,7 @@ def report(self, source, timeout, return_code, results, output, pid=None): log(output) log("*"*70) postscript['lines'].append(cmd + " # %s" % fail_msg) - stats[basename] = dict(failed=True, walltime=1e6) + stats[basename] = dict(failed=True, walltime=1e6, ntests=ntests) if not the_baseline_stats.get('failed', False): self.error_status |= (8 if return_code > 0 else 16) else: @@ -489,9 +501,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): if hasattr(result_dict, 'tb'): log(result_dict.tb) if hasattr(result_dict, 'walltime'): - stats[basename] = dict(failed=True, walltime=wall) + stats[basename] = dict(failed=True, walltime=wall, ntests=ntests) else: - stats[basename] = dict(failed=True, walltime=1e6) + stats[basename] = dict(failed=True, walltime=1e6, ntests=ntests) self.error_status |= 64 if result_dict.err is None or result_dict.err == 'tab': f = result_dict.failures @@ -503,9 +515,9 @@ def report(self, source, timeout, return_code, results, output, pid=None): if not the_baseline_stats.get('failed', False): self.error_status |= 1 if f or result_dict.err == 'tab': - stats[basename] = dict(failed=True, walltime=wall) + stats[basename] = dict(failed=True, walltime=wall, ntests=ntests) else: - stats[basename] = dict(walltime=wall) + stats[basename] = dict(walltime=wall, ntests=ntests) postscript['cputime'] += cpu postscript['walltime'] += wall