From 508e94d231d0b7ad19e720517db2417b3c071c0f Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 21 Sep 2019 13:36:30 +0200 Subject: [PATCH 01/13] [actions] Add push workflow (linting) --- .github/workflows/push.yml | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/push.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000000..0f2c88e2d8 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,51 @@ +name: Unit tests + +on: ['push', 'pull_request'] + +jobs: + + flake8: + name: Flake8 tests + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Set up Python 3.7 + uses: actions/setup-python@v1.1.0 + with: + python-version: 3.7 + - name: Run flake8 + # we use exactly the same exceptions than tox.ini file + run: | + python -m pip install --upgrade pip + pip install -U flake8 + flake8 --ignore=E123,E124,E126,E226,E402,E501,W503,W504 pythonforandroid/ tests/ ci/ + + linting: + name: Pytest [Python ${{ matrix.python-version }} | ${{ matrix.os }}] + needs: flake8 + runs-on: ${{ matrix.os }} + strategy: + matrix: + python-version: [3.6, 3.7] + os: [ubuntu-latest, macOs-latest] + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1.1.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -U virtualenv pytest pytest-cov backports.tempfile pyOpenSSL coveralls + pip install -e . + - name: Tests + run: | + python -m pytest tests/ --ignore tests/test_pythonpackage.py --cov pythonforandroid/ --cov-branch + - name: Coveralls + run: | + python -m coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} From 23b8907a57a01389df6a4a1119e55f8b1cd4a50e Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 21 Sep 2019 15:14:00 +0200 Subject: [PATCH 02/13] [actions] Fix `expected compiler` for tests Because the compiler path depends on the platform where the tests are run. --- tests/recipes/recipe_lib_test.py | 9 +++++---- tests/recipes/test_icu.py | 3 ++- tests/test_archs.py | 5 +++-- tests/test_bootstrap.py | 4 +++- tests/test_recipe.py | 6 ++++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/recipes/recipe_lib_test.py b/tests/recipes/recipe_lib_test.py index 5622397aa5..6c48474384 100644 --- a/tests/recipes/recipe_lib_test.py +++ b/tests/recipes/recipe_lib_test.py @@ -1,4 +1,5 @@ from unittest import mock +from platform import system from tests.recipes.recipe_ctx import RecipeCtx @@ -13,7 +14,7 @@ class BaseTestForMakeRecipe(RecipeCtx): recipe_name = None expected_compiler = ( - "{android_ndk}/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" + "{android_ndk}/toolchains/llvm/prebuilt/{system}-x86_64/bin/clang" ) sh_command_calls = ["./configure"] @@ -48,7 +49,7 @@ def test_get_recipe_env( some internal methods has been called. """ mock_find_executable.return_value = self.expected_compiler.format( - android_ndk=self.ctx._ndk_dir + android_ndk=self.ctx._ndk_dir, system=system().lower() ) mock_glob.return_value = ["llvm"] mock_check_recipe_choices.return_value = sorted( @@ -85,7 +86,7 @@ def test_build_arch( mock_current_directory, ): mock_find_executable.return_value = self.expected_compiler.format( - android_ndk=self.ctx._ndk_dir + android_ndk=self.ctx._ndk_dir, system=system().lower() ) mock_glob.return_value = ["llvm"] @@ -133,7 +134,7 @@ def test_build_arch( mock_current_directory, ): mock_find_executable.return_value = self.expected_compiler.format( - android_ndk=self.ctx._ndk_dir + android_ndk=self.ctx._ndk_dir, system=system().lower() ) mock_glob.return_value = ["llvm"] diff --git a/tests/recipes/test_icu.py b/tests/recipes/test_icu.py index 506cdb74b2..00cfbdbbe7 100644 --- a/tests/recipes/test_icu.py +++ b/tests/recipes/test_icu.py @@ -1,6 +1,7 @@ import os import unittest from unittest import mock +from platform import system from tests.recipes.recipe_ctx import RecipeCtx from pythonforandroid.recipes.icu import ICURecipe @@ -47,7 +48,7 @@ def test_build_arch( ): mock_find_executable.return_value = os.path.join( self.ctx._ndk_dir, - "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", + f"toolchains/llvm/prebuilt/{system().lower()}-x86_64/bin/clang", ) mock_archs_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") diff --git a/tests/test_archs.py b/tests/test_archs.py index ddb18b80e2..61887e0e51 100644 --- a/tests/test_archs.py +++ b/tests/test_archs.py @@ -2,6 +2,7 @@ import unittest from os import environ from unittest import mock +from platform import system from pythonforandroid.bootstrap import Bootstrap from pythonforandroid.distribution import Distribution @@ -70,8 +71,8 @@ def setUp(self): # Here we define the expected compiler, which, as per ndk >= r19, # should be the same for all the tests (no more gcc compiler) self.expected_compiler = ( - "/opt/android/android-ndk/toolchains/" - "llvm/prebuilt/linux-x86_64/bin/clang" + f"/opt/android/android-ndk/toolchains/" + f"llvm/prebuilt/{system().lower()}-x86_64/bin/clang" ) diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index 02d54f0d9e..b164985e38 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -4,6 +4,8 @@ import unittest from unittest import mock +from platform import system + from pythonforandroid.bootstrap import ( _cmp_bootstraps_by_priority, Bootstrap, expand_dependencies, ) @@ -546,7 +548,7 @@ def test_bootstrap_strip( ): mock_find_executable.return_value = os.path.join( self.ctx._ndk_dir, - "toolchains/llvm/prebuilt/linux-x86_64/bin/clang", + f"toolchains/llvm/prebuilt/{system().lower()}-x86_64/bin/clang", ) mock_glob.return_value = [ os.path.join(self.ctx._ndk_dir, "toolchains", "llvm") diff --git a/tests/test_recipe.py b/tests/test_recipe.py index c01ab83507..9ff1713c95 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -5,6 +5,8 @@ import warnings from unittest import mock from backports import tempfile +from platform import system + from pythonforandroid.build import Context from pythonforandroid.recipe import Recipe, import_recipe from pythonforandroid.archs import ArchAarch_64 @@ -278,8 +280,8 @@ def test_get_recipe_env_with( should be tested in the proper test. """ expected_compiler = ( - "/opt/android/android-ndk/toolchains/" - "llvm/prebuilt/linux-x86_64/bin/clang" + f"/opt/android/android-ndk/toolchains/" + f"llvm/prebuilt/{system().lower()}-x86_64/bin/clang" ) mock_find_executable.return_value = expected_compiler mock_glob.return_value = ["llvm"] From 3b6448f840c8173c34c38d70b5003f5e64da7048 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 21 Sep 2019 15:14:31 +0200 Subject: [PATCH 03/13] [actions] Mock environ variables for `test_create_no_sdk_dir` --- tests/test_toolchain.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index 7e3078133b..f95abcfdba 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -100,6 +100,10 @@ def test_create(self): ] assert m_run_distribute.call_args_list == [mock.call()] + @mock.patch( + 'pythonforandroid.build.environ', + # Make sure that no environ variable modifies `sdk_dir` + {'ANDROIDSDK': None, 'ANDROID_HOME': None}) def test_create_no_sdk_dir(self): """ The `--sdk-dir` is mandatory to `create` a distribution. From 5e0d7edab2e1e18c1deedd94664eb7bdbbc5dedd Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 21 Sep 2019 15:15:12 +0200 Subject: [PATCH 04/13] [actions] Fix `test_toolchain.test_create` for macos --- tests/test_toolchain.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_toolchain.py b/tests/test_toolchain.py index f95abcfdba..e299794ebb 100644 --- a/tests/test_toolchain.py +++ b/tests/test_toolchain.py @@ -78,10 +78,14 @@ def test_create(self): m_get_ndk_platform_dir.return_value = ( '/tmp/android-ndk/platforms/android-21/arch-arm', True) ToolchainCL() - assert m_get_available_apis.call_args_list == [ - mock.call('/tmp/android-sdk')] - assert m_get_toolchain_versions.call_args_list == [ - mock.call('/tmp/android-ndk', mock.ANY)] + assert m_get_available_apis.call_args_list in [ + [mock.call('/tmp/android-sdk')], # linux case + [mock.call('/private/tmp/android-sdk')] # macos case + ] + assert m_get_toolchain_versions.call_args_list in [ + [mock.call('/tmp/android-ndk', mock.ANY)], # linux case + [mock.call('/private/tmp/android-ndk', mock.ANY)], # macos case + ] build_order = [ 'hostpython3', 'libffi', 'openssl', 'sqlite3', 'python3', 'genericndkbuild', 'setuptools', 'six', 'pyjnius', 'android', From f0d6a62b5af9c7a706f757e5c22c2687b6ce25a8 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 21 Sep 2019 15:15:40 +0200 Subject: [PATCH 05/13] [actions] Fix venv/virtualenv tests --- tests/test_pythonpackage_basic.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_pythonpackage_basic.py b/tests/test_pythonpackage_basic.py index 3d1a156df6..a39d8a44e3 100644 --- a/tests/test_pythonpackage_basic.py +++ b/tests/test_pythonpackage_basic.py @@ -308,7 +308,9 @@ def test_virtualenv(self): sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "virtualenv", "bin", "python") ) - assert os.path.normpath(sys_python_path) == os.path.normpath(pybin) + assert os.path.normpath(sys_python_path).startswith( + os.path.normpath(pybin) + ) finally: shutil.rmtree(test_dir) @@ -341,6 +343,8 @@ def test_venv(self): sys_python_path = self.run__get_system_python_executable( os.path.join(test_dir, "venv", "bin", "python") ) - assert os.path.normpath(sys_python_path) == os.path.normpath(pybin) + assert os.path.normpath(sys_python_path).startswith( + os.path.normpath(pybin) + ) finally: shutil.rmtree(test_dir) From 23d35a65009f4e7196896c9ccdd032a066ed05f1 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 21 Sep 2019 16:31:13 +0200 Subject: [PATCH 06/13] [actions] Add build stage: pull docker image & build an testapp --- .github/workflows/push.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0f2c88e2d8..8289ca9c4a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,4 +1,4 @@ -name: Unit tests +name: Unit tests & Build Testapp on: ['push', 'pull_request'] @@ -49,3 +49,17 @@ jobs: python -m coveralls env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + + build: + name: Build testapp + needs: [flake8] + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Pull docker image + run: | + make docker/pull + - name: Build apk for Python 3 arm64-v8a + run: | + make docker/run/make/testapps/python3/arm64-v8a From 3611a9ef3742552dc5f1797012dc6e03174fe1ab Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 12:46:38 +0100 Subject: [PATCH 07/13] [actions] Make flake8 tests via tox --- .github/workflows/push.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 8289ca9c4a..b24fcfe91c 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -15,11 +15,10 @@ jobs: with: python-version: 3.7 - name: Run flake8 - # we use exactly the same exceptions than tox.ini file run: | python -m pip install --upgrade pip - pip install -U flake8 - flake8 --ignore=E123,E124,E126,E226,E402,E501,W503,W504 pythonforandroid/ tests/ ci/ + pip install tox>=2.0 + tox -e pep8 linting: name: Pytest [Python ${{ matrix.python-version }} | ${{ matrix.os }}] From 4f0f86ae429a6fc11d17b34ffdd070e6ad9590c1 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 14:19:31 +0100 Subject: [PATCH 08/13] [actions] Run pytest via tox We also remove the env var `COVERALLS_REPO_TOKEN`, so we test if coveralls is working with `gh-actions` Note: Since Python 2 it's almost at the end of his life we only perform the Python3 tests, but we still have the travis tests for Python 2 --- .github/workflows/push.yml | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b24fcfe91c..7cf324af21 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -35,19 +35,15 @@ jobs: uses: actions/setup-python@v1.1.0 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Tox Python 3 + # ignores test_pythonpackage.py since it runs for too long run: | python -m pip install --upgrade pip - pip install -U virtualenv pytest pytest-cov backports.tempfile pyOpenSSL coveralls - pip install -e . - - name: Tests + pip install tox>=2.0 pyOpenSSL coveralls + tox -e py3 -- tests/ --ignore tests/test_pythonpackage.py + - name: Send Coverage to coveralls run: | - python -m pytest tests/ --ignore tests/test_pythonpackage.py --cov pythonforandroid/ --cov-branch - - name: Coveralls - run: | - python -m coveralls - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + coveralls build: name: Build testapp From 8416bfe1074d615eed81de6610bf33a0211e83ff Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 14:38:16 +0100 Subject: [PATCH 09/13] [actions] Reintroduce `COVERALLS_REPO_TOKEN` --- .github/workflows/push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7cf324af21..075b9b3f82 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -44,6 +44,8 @@ jobs: - name: Send Coverage to coveralls run: | coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} build: name: Build testapp From b9e66eaebe06dbbb89ef4483c2995a5ac89a47fa Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 16:30:32 +0100 Subject: [PATCH 10/13] [actions] Make tox tests via Makefile The coveralls report will only be send with our travis tests, since the `CI` environment variable it will only exist for travis. --- .github/workflows/push.yml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 075b9b3f82..a5a53e36d4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -20,7 +20,7 @@ jobs: pip install tox>=2.0 tox -e pep8 - linting: + test: name: Pytest [Python ${{ matrix.python-version }} | ${{ matrix.os }}] needs: flake8 runs-on: ${{ matrix.os }} @@ -35,17 +35,11 @@ jobs: uses: actions/setup-python@v1.1.0 with: python-version: ${{ matrix.python-version }} - - name: Tox Python 3 - # ignores test_pythonpackage.py since it runs for too long + - name: Tox tests run: | python -m pip install --upgrade pip - pip install tox>=2.0 pyOpenSSL coveralls - tox -e py3 -- tests/ --ignore tests/test_pythonpackage.py - - name: Send Coverage to coveralls - run: | - coveralls - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + pip install tox>=2.0 + make test build: name: Build testapp From 643a7f712b4e42714327eb8b3855cd08c30f22a4 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 17:01:22 +0100 Subject: [PATCH 11/13] [actions] Add `rebuild updated recipes` test --- .github/workflows/push.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a5a53e36d4..254832ceaf 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -54,3 +54,17 @@ jobs: - name: Build apk for Python 3 arm64-v8a run: | make docker/run/make/testapps/python3/arm64-v8a + + rebuild_updated_recipes: + name: Test updated recipes + needs: [flake8] + runs-on: ubuntu-latest + steps: + - name: Checkout python-for-android + uses: actions/checkout@master + - name: Pull docker image + run: | + make docker/pull + - name: Rebuild updated recipes + run: | + make docker/run/make/rebuild_updated_recipes From c6c3ea8c9ae5b571be192c764f7626caf9af1cd7 Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 17:15:05 +0100 Subject: [PATCH 12/13] [actions] Shorten long line below 79 characters for libtorrent The purpose of this commit is to test the rebuild updated recipes behaviour with a long time consuming recipe --- pythonforandroid/recipes/libtorrent/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py index 0eb50672b7..ff405eacd9 100644 --- a/pythonforandroid/recipes/libtorrent/__init__.py +++ b/pythonforandroid/recipes/libtorrent/__init__.py @@ -75,7 +75,11 @@ def build_arch(self, arch): # Define build variables build_dir = self.get_build_dir(arch.arch) ctx_libs_dir = self.ctx.get_libs_dir(arch.arch) - encryption = 'openssl' if 'openssl' in recipe.ctx.recipe_build_order else 'built-in' + encryption = ( + 'openssl' + if 'openssl' in recipe.ctx.recipe_build_order + else 'built-in' + ) build_args = [ '-q', # '-a', # force build, useful to debug the build From 0a939087001b9c8c0a11acd463a62d6bef8a054c Mon Sep 17 00:00:00 2001 From: opacam Date: Sat, 23 Nov 2019 18:12:09 +0100 Subject: [PATCH 13/13] [actions] Revert "[actions] Shorten long line ..." This reverts commit c6c3ea8c, to keep travis happy --- pythonforandroid/recipes/libtorrent/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pythonforandroid/recipes/libtorrent/__init__.py b/pythonforandroid/recipes/libtorrent/__init__.py index ff405eacd9..0eb50672b7 100644 --- a/pythonforandroid/recipes/libtorrent/__init__.py +++ b/pythonforandroid/recipes/libtorrent/__init__.py @@ -75,11 +75,7 @@ def build_arch(self, arch): # Define build variables build_dir = self.get_build_dir(arch.arch) ctx_libs_dir = self.ctx.get_libs_dir(arch.arch) - encryption = ( - 'openssl' - if 'openssl' in recipe.ctx.recipe_build_order - else 'built-in' - ) + encryption = 'openssl' if 'openssl' in recipe.ctx.recipe_build_order else 'built-in' build_args = [ '-q', # '-a', # force build, useful to debug the build