From 4e82da23eccc360d07b9c12f538f9325bdfb5633 Mon Sep 17 00:00:00 2001 From: Alex Huszagh Date: Thu, 13 Oct 2022 19:21:00 -0500 Subject: [PATCH] More fixes. --- docker/Dockerfile.aarch64-linux-android | 5 +- docker/android-system.sh | 123 +++++++++++----- docker/android/android/__init__.py | 11 ++ .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 233 bytes docker/android/android/make.py | 66 +++++++++ docker/android/android/soong.py | 137 ++++++++++++++++++ docker/android/android/util.py | 3 + .../remove_tests.py} | 6 + docker/android/tests/Android.bp | 59 ++++++++ docker/android/tests/Android.mk | 102 +++++++++++++ docker/android/tests/README.md | 8 + .../test_make.cpython-310-pytest-7.1.3.pyc | Bin 0 -> 3192 bytes docker/android/tests/test_make.py | 27 ++++ docker/android/tests/test_soong.py | 32 ++++ 14 files changed, 538 insertions(+), 41 deletions(-) create mode 100644 docker/android/android/__init__.py create mode 100644 docker/android/android/__pycache__/__init__.cpython-310.pyc create mode 100644 docker/android/android/make.py create mode 100644 docker/android/android/soong.py create mode 100644 docker/android/android/util.py rename docker/{remove_android_tests.py => android/remove_tests.py} (94%) create mode 100644 docker/android/tests/Android.bp create mode 100644 docker/android/tests/Android.mk create mode 100644 docker/android/tests/README.md create mode 100644 docker/android/tests/__pycache__/test_make.cpython-310-pytest-7.1.3.pyc create mode 100644 docker/android/tests/test_make.py create mode 100644 docker/android/tests/test_soong.py diff --git a/docker/Dockerfile.aarch64-linux-android b/docker/Dockerfile.aarch64-linux-android index daecf3a74..bd0ba9582 100644 --- a/docker/Dockerfile.aarch64-linux-android +++ b/docker/Dockerfile.aarch64-linux-android @@ -17,13 +17,16 @@ ARG ANDROID_NDK=r21d ARG ANDROID_SDK=28 ARG ANDROID_VERSION=9.0.0_r1 ARG ANDROID_SYSTEM_COMPLETE=0 +ARG PYTHON_TMPDIR=/tmp/android COPY android-ndk.sh / RUN /android-ndk.sh arm64 ENV PATH=$PATH:/android-ndk/bin # TODO(ahuszagh) Restore... -COPY android-system.sh remove_android_tests.py / +COPY android-system.sh / +RUN mkdir -p $PYTHON_TMPDIR +COPY android $PYTHON_TMPDIR #RUN /android-system.sh arm64 # #COPY android-symlink.sh / diff --git a/docker/android-system.sh b/docker/android-system.sh index f27bccc85..1acb972db 100755 --- a/docker/android-system.sh +++ b/docker/android-system.sh @@ -64,7 +64,6 @@ main() { python \ python3 \ xz-utils - purge_list+=(default-jre) curl --retry 3 -sSfL https://storage.googleapis.com/git-repo-downloads/repo -O chmod +x repo @@ -128,30 +127,40 @@ main() { popd rm -rf "${td}" + rm -rf "${PYTHON_TMPDIR}" rm "${0}" } +# java isn't required for the build, but the build expects to +# find a java compiler. the supported android versions are: +# https://source.android.com/docs/setup/start/older-versions +# Android 7: OpenJDK-8 fake_java() { + local java_type= local java_version= - local icedtea_version= + local jre_info= + local build_info= + case "${MAJOR_VERSION}" in 5|6) + java_type=java java_version=1.7.0 - icedtea_version=2.6.9 + jre_info="IcedTea 2.6.9" + build_info="build 24.131-b00, mixed mode" ;; *) - java_version=1.8.0 - icedtea_version=2.6.9 + java_type=openjdk + java_version=1.8.0_342 + jre_info="build 1.8.0_342-8u342-b07-0ubuntu1~20.04-b07" + build_info="build 25.342-b07, mixed mode" ;; esac - # fake java and javac, it is not necessary for what we build, but the build - # script asks for it cat << EOF > /usr/bin/java #!/usr/bin/env bash -echo "java version \"${java_version}\"" -echo "OpenJDK Runtime Environment (IcedTea ${icedtea_version})" -echo "OpenJDK 64-Bit Server VM (build 24.131-b00, mixed mode)" +echo "${java_type} version \"${java_version}\"" +echo "OpenJDK Runtime Environment (${jre_info})" +echo "OpenJDK 64-Bit Server VM (${build_info})" EOF cat << EOF > /usr/bin/javac @@ -164,7 +173,7 @@ EOF # more faking export ANDROID_JAVA_HOME=/tmp - mkdir /tmp/lib/ + mkdir -p /tmp/lib/ touch /tmp/lib/tools.jar } @@ -225,6 +234,7 @@ android_repo_v5() { sync external/libselinux sync external/mksh sync external/openssl + sync external/pcre sync external/stlport sync prebuilts/clang/linux-x86/host/3.5 sync system/core @@ -262,9 +272,11 @@ android_repo_v6() { sync external/compiler-rt sync external/libcxx sync external/libcxxabi + sync external/libselinux sync external/elfutils sync external/jemalloc sync external/mksh + sync external/pcre sync external/safe-iop sync external/zlib sync libnativehelper @@ -300,6 +312,7 @@ android_repo_v6() { # tested on 7.0.0_r36 (SDK 24) # tested on 7.1.2_r39 (SDK 25, not supported) +# API level 25, requires for Android 7.1, is not provided in NDKs android_repo_v7() { sync bionic sync build @@ -321,6 +334,7 @@ android_repo_v7() { sync prebuilts/clang/host/linux-x86 sync prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8 sync prebuilts/misc + sync prebuilts/ndk sync prebuilts/ninja/linux-x86 sync system/core @@ -349,12 +363,26 @@ android_repo_v7() { # the unittests from it is a bit of work. rm bionic/tools/relocation_packer/Android.mk - ANDROID_MAJOR=7 python3 /remove_android_tests.py + remove_tests } # TODO: tested on 8.0.0_r51 (SDK 26) -# TODO: tested on 8.1.0_r81 (SDK 2278.0.0_r516) +# Currently failing due to libLLVM/libclang issues +# TODO: tested on 8.1.0_r81 (SDK 27) android_repo_v8() { + # external/compiler-rt/lib/sanitizer_common/tests/Android.bp:37:1 + # name: "san_test-Nolibc" + # + # libnativehelper/Android.bp + # subdirs = ["tests"] + # + # system/core/bootstat/Android.bp + # whole_static_libs: ["libgtest_prod"], + # + # external/compiler-rt/lib/asan/Android.bp + # name: "libasan_noinst_test", + # + sync bionic sync build sync build/make @@ -363,34 +391,42 @@ android_repo_v8() { sync external/boringssl sync external/clang sync external/compiler-rt + sync external/elfutils sync external/jemalloc sync external/libcxx sync external/libcxxabi - sync external/libnl + # TODO(ahuszagh) Remove? + #sync external/libnl + sync external/libevent sync external/libunwind sync external/libunwind_llvm sync external/llvm sync external/lzma sync external/mksh sync external/pcre + sync external/safe-iop sync external/selinux sync external/zlib - sync hardware/interfaces - sync hardware/libhardware - sync frameworks/native + # TODO(ahuszagh) Remove? + #sync hardware/interfaces + #sync hardware/libhardware + #sync frameworks/native sync libnativehelper sync prebuilts/build-tools sync prebuilts/clang/host/linux-x86 + # TODO(ahuszagh) clang-tools isn't available till 9.0 sync prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8 sync prebuilts/go/linux-x86 - # needed for libnativehelper_compat_libc++ sync prebuilts/ndk sync system/core - sync system/libfmq - sync system/libhidl - sync system/libhwbinder - sync system/media - sync system/tools/hidl + sync toolchain/binutils + + # TODO(ahuszagh) See if I can remove all of these + #sync system/libfmq + #sync system/libhidl + #sync system/libhwbinder + #sync system/media + #sync system/tools/hidl # TODO(ahuszagh) Android 8 is a mess # avoid build tests @@ -399,26 +435,21 @@ android_repo_v8() { rm bionic/tests/Android.bp rm bionic/benchmarks/Android.bp rm bionic/tests/libs/Android.bp - rm hardware/interfaces/tests/Android.bp - rm system/tools/hidl/test/Android.bp + # TODO(ahuszagh) Can probably remove this +# rm hardware/interfaces/tests/Android.bp +# rm system/tools/hidl/test/Android.bp # we don't need the relocation packer, and removing # the unittests from it is a bit of work. rm bionic/tools/relocation_packer/Android.bp rm bionic/tools/relocation_packer/Android.mk + rm system/core/libgrallocusage/Android.bp + rm system/core/libmemtrack/Android.bp + rm system/core/libsysutils/Android.bp + + # avoid java dependencies + rm external/lzma/Java/Tukaani/Android.mk - # extra utilities we don't need - rm frameworks/native/libs/vr/Android.bp - rm frameworks/native/services/Android.bp - rm frameworks/native/services/*/Android.bp - rm hardware/interfaces/automotive/Android.bp - rm hardware/interfaces/camera/Android.bp - rm system/libhidl/transport/Android.bp - rm system/media/alsa_utils/Android.bp - rm system/media/audio_route/Android.bp - rm system/media/audio_utils/tests/Android.bp - # TODO(ahuszagh) Need to remove the tests here - - ANDROID_MAJOR=8 python3 /remove_android_tests.py + remove_tests } # tested on 9.0.0_r1 (SDK 28) @@ -482,7 +513,7 @@ android_repo_v9() { rm bionic/tests/headers/Android.bp rm bionic/tests/headers/posix/Android.bp - ANDROID_MAJOR=9 python3 /remove_android_tests.py + remove_tests } # tested on 10.0.0_r47 (SDK 29) @@ -553,7 +584,19 @@ android_repo_v10() { rm bionic/tests/headers/Android.bp rm bionic/tests/headers/posix/Android.bp - ANDROID_MAJOR=10 python3 /remove_android_tests.py + remove_tests +} + +remove_tests() { + install_packages python3-pip + + export PYTHONPATH="${PYTHON_TMPDIR}/lib/python3.8/site-packages/":"${PYTHONPATH}" + mkdir -p "${PYTHON_TMPDIR}" + python3 -m pip install sly==0.4.0 --prefix "${PYTHON_TMPDIR}" + + # TODO(ahuszagh) Need to change this + # Need a custom entrypoint + ANDROID_MAJOR="${MAJOR_VERSION}" python3 "${PYTHON_TMPDIR}/remove_tests.py" } sync() { diff --git a/docker/android/android/__init__.py b/docker/android/android/__init__.py new file mode 100644 index 000000000..7becf09f2 --- /dev/null +++ b/docker/android/android/__init__.py @@ -0,0 +1,11 @@ +import sys + +# we run this script once every build, and we'd rather +# have much smaller image sizes, so copying without +# any bytecode is a better idea. +sys.dont_write_bytecode = True + +__all__ = [ + "make", + "soong", +] diff --git a/docker/android/android/__pycache__/__init__.cpython-310.pyc b/docker/android/android/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3dbd1b0858590c846f6339a5fb3d3d65bf60ea5c GIT binary patch literal 233 zcmYjMJ8l9o6nyrwWEF`j9XFuC2LMHssc1sA&6C+@?OOYJ$TmWo8|4x?h#QJjxk3tr zp+JoEW;7bjNRMZ;S4j9-e^lqBzc=#?nan1gpC=+9(Lo6$4WOWmCbcXjiBDMMe<_FU zE>3#gnlksyL%qmi=0_jjh=+hDXM@507)-@v;tQhgI>c{q(^S3F|4KF78>O40|I&4% vY8zDL*n34>wZ= 0.4 +- pytest >= 7 diff --git a/docker/android/tests/__pycache__/test_make.cpython-310-pytest-7.1.3.pyc b/docker/android/tests/__pycache__/test_make.cpython-310-pytest-7.1.3.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6144ab814e684ea8b95bc85b5e0f0b3e70a36033 GIT binary patch literal 3192 zcmcIm&2Jk;6yMnodu^}nq)F4JloCFQx(DKi+koFe{%a^bz%wYy1Efl}40oqaR&_PyVG@AqaM zRI6nLet-UOrFm6Tl)rG2{~0j32)F3LKov@S#Ub*kb~IHXN=J7LlI|3og4`)OMOIih z2PHX{S!GCu#Idq93*;&*!hVe^tU6TX+p1shOgIxDYgB)rIFppzRBU4xyeijhlIR_8 zgEetB`qe=&;l2#F_!V%5$~qaSp+?D2jg?T3lx;FFhDuDr!g_IFM#_-HYFLWYZ4zsd z7Lg4#)+1d6VR=xYYNW3bD%aQ4p-%PC8dQhMJw=>_{aRQW621&l9i=XzIyJ@`mr)~9 zLE{Q)padF*l(iwDg|X&ssR{YuU&>vOK(j!LJNk?g8`Pww$Y>Lg3*iK4Oh$zv*(TpZ z&UGc$0I#}!B-X;INP{SJT8@j69u*-HGcv_vcmpFbA(9HN%Sf!T_n**vI0fE!&^t=t z-O5GsGiu049-+qmNaUJ6B=T8=$f`RAU@g%atw$w^yPV-J=Wzpe9c~lQ0)9F%tHhP4 zvObM<*oM7Gw_q0{_=iqP+rSz5gzSxc(jz+>__8wlz@u?qDwrGe0XES7)U);<5@FNn z|FCUjwvGL^)4Y%E9({@%?0!LyzM$Qi1MzosBbk=2gjkOYf)`aAI#+Y z;GrsRWhkE037~NbG-NM4RQa2TZ`=*1M*RX>i>MXVh6J>hPzz<~8GO4Q<=83bhwBK4 zWZV&$Q6_C! zm>JU1Q0;oyVtg(u$vhPGos3BP8HJS#+o8{%(rF3El}i|MV`&dzps2ix)j~ zKNYnt{->L^khDWLvxyprMDKZFE78}3b~iC^t=_!le*Ec;L=Sqbo9LW*lw)=r^Rp4V zjh@GWz8Gp!zS!Hs@Li85lWx=VeV4NyZw4JuVu=OY?ww7*+3t#@;0K@~lH;yu1)DyV zs_tDsxa0Y5*XyvP4qEPC3mR64b8qX?T%i?{WOs=~NJLN`$ld$-C(!#jup0T*#{} z=O=sA-g9I!QUi;NdA)_axjh!n^GN{pi^AW6$CLLPt)RmiUTahIy}PZ(-FDb$@<50N z4VoK_H@x&LYhbiOjxL@_^S!M^mm~!%;uH)uV!`zrTwkfTNqXHUrusX%PaY8dHmKM~ z5_42Jei9avSvRfqe3~A}|2$`3^L7Yz8!sirY>l6QclZ>(sEI8h^PCiEo6AC*KrCMp z&vacpGhLU@fUZ;d{KoaGtCv#}7JxNyPA0`HFv&!TP9`KboJK$92c3X!`s_nIUqlUt PHx!dhs}_NHRZIOB6|re? literal 0 HcmV?d00001 diff --git a/docker/android/tests/test_make.py b/docker/android/tests/test_make.py new file mode 100644 index 000000000..2cb0a187b --- /dev/null +++ b/docker/android/tests/test_make.py @@ -0,0 +1,27 @@ +import os +import sys + +TEST_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, PROJECT_DIR) + +from android import make + +def test(): + path = os.path.join(TEST_DIR, 'Android.mk') + contents = open(path).read() + makefile = make.Makefile.parse(path) + assert str(makefile) == contents + assert len(makefile.sections) == 6 + + assert not makefile.sections[0].is_dev() + assert makefile.sections[1].is_dev() + assert makefile.sections[1].is_benchmark() + assert makefile.sections[2].is_dev() + assert makefile.sections[2].is_test() + assert makefile.sections[4].title == 'Other section.' + + filtered = makefile.filter_dev() + assert len(filtered.sections) == 2 + assert filtered.sections[0].title is None + assert filtered.sections[1].title == 'Other section.' diff --git a/docker/android/tests/test_soong.py b/docker/android/tests/test_soong.py new file mode 100644 index 000000000..ed318adb8 --- /dev/null +++ b/docker/android/tests/test_soong.py @@ -0,0 +1,32 @@ +import os +import sys + +TEST_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, PROJECT_DIR) + +from android import soong + +def test(): + path = os.path.join(TEST_DIR, 'Android.bp') + contents = open(path).read() + lexer = soong.Lexer() + tokens = list(lexer.tokenize(contents)) + import pdb; pdb.set_trace() + +# TODO(ahuszagh) Implement +# makefile = make.Makefile.parse(path) +# assert str(makefile) == contents +# assert len(makefile.sections) == 6 +# +# assert not makefile.sections[0].is_dev() +# assert makefile.sections[1].is_dev() +# assert makefile.sections[1].is_benchmark() +# assert makefile.sections[2].is_dev() +# assert makefile.sections[2].is_test() +# assert makefile.sections[4].title == 'Other section.' +# +# filtered = makefile.filter_dev() +# assert len(filtered.sections) == 2 +# assert filtered.sections[0].title is None +# assert filtered.sections[1].title == 'Other section.'