diff --git a/docker/Dockerfile.aarch64-linux-android b/docker/Dockerfile.aarch64-linux-android index daecf3a74..bd0ba9582 100644 --- a/docker/Dockerfile.aarch64-linux-android +++ b/docker/Dockerfile.aarch64-linux-android @@ -17,13 +17,16 @@ ARG ANDROID_NDK=r21d ARG ANDROID_SDK=28 ARG ANDROID_VERSION=9.0.0_r1 ARG ANDROID_SYSTEM_COMPLETE=0 +ARG PYTHON_TMPDIR=/tmp/android COPY android-ndk.sh / RUN /android-ndk.sh arm64 ENV PATH=$PATH:/android-ndk/bin # TODO(ahuszagh) Restore... -COPY android-system.sh remove_android_tests.py / +COPY android-system.sh / +RUN mkdir -p $PYTHON_TMPDIR +COPY android $PYTHON_TMPDIR #RUN /android-system.sh arm64 # #COPY android-symlink.sh / diff --git a/docker/android-system.sh b/docker/android-system.sh index f27bccc85..1acb972db 100755 --- a/docker/android-system.sh +++ b/docker/android-system.sh @@ -64,7 +64,6 @@ main() { python \ python3 \ xz-utils - purge_list+=(default-jre) curl --retry 3 -sSfL https://storage.googleapis.com/git-repo-downloads/repo -O chmod +x repo @@ -128,30 +127,40 @@ main() { popd rm -rf "${td}" + rm -rf "${PYTHON_TMPDIR}" rm "${0}" } +# java isn't required for the build, but the build expects to +# find a java compiler. the supported android versions are: +# https://source.android.com/docs/setup/start/older-versions +# Android 7: OpenJDK-8 fake_java() { + local java_type= local java_version= - local icedtea_version= + local jre_info= + local build_info= + case "${MAJOR_VERSION}" in 5|6) + java_type=java java_version=1.7.0 - icedtea_version=2.6.9 + jre_info="IcedTea 2.6.9" + build_info="build 24.131-b00, mixed mode" ;; *) - java_version=1.8.0 - icedtea_version=2.6.9 + java_type=openjdk + java_version=1.8.0_342 + jre_info="build 1.8.0_342-8u342-b07-0ubuntu1~20.04-b07" + build_info="build 25.342-b07, mixed mode" ;; esac - # fake java and javac, it is not necessary for what we build, but the build - # script asks for it cat << EOF > /usr/bin/java #!/usr/bin/env bash -echo "java version \"${java_version}\"" -echo "OpenJDK Runtime Environment (IcedTea ${icedtea_version})" -echo "OpenJDK 64-Bit Server VM (build 24.131-b00, mixed mode)" +echo "${java_type} version \"${java_version}\"" +echo "OpenJDK Runtime Environment (${jre_info})" +echo "OpenJDK 64-Bit Server VM (${build_info})" EOF cat << EOF > /usr/bin/javac @@ -164,7 +173,7 @@ EOF # more faking export ANDROID_JAVA_HOME=/tmp - mkdir /tmp/lib/ + mkdir -p /tmp/lib/ touch /tmp/lib/tools.jar } @@ -225,6 +234,7 @@ android_repo_v5() { sync external/libselinux sync external/mksh sync external/openssl + sync external/pcre sync external/stlport sync prebuilts/clang/linux-x86/host/3.5 sync system/core @@ -262,9 +272,11 @@ android_repo_v6() { sync external/compiler-rt sync external/libcxx sync external/libcxxabi + sync external/libselinux sync external/elfutils sync external/jemalloc sync external/mksh + sync external/pcre sync external/safe-iop sync external/zlib sync libnativehelper @@ -300,6 +312,7 @@ android_repo_v6() { # tested on 7.0.0_r36 (SDK 24) # tested on 7.1.2_r39 (SDK 25, not supported) +# API level 25, requires for Android 7.1, is not provided in NDKs android_repo_v7() { sync bionic sync build @@ -321,6 +334,7 @@ android_repo_v7() { sync prebuilts/clang/host/linux-x86 sync prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8 sync prebuilts/misc + sync prebuilts/ndk sync prebuilts/ninja/linux-x86 sync system/core @@ -349,12 +363,26 @@ android_repo_v7() { # the unittests from it is a bit of work. rm bionic/tools/relocation_packer/Android.mk - ANDROID_MAJOR=7 python3 /remove_android_tests.py + remove_tests } # TODO: tested on 8.0.0_r51 (SDK 26) -# TODO: tested on 8.1.0_r81 (SDK 2278.0.0_r516) +# Currently failing due to libLLVM/libclang issues +# TODO: tested on 8.1.0_r81 (SDK 27) android_repo_v8() { + # external/compiler-rt/lib/sanitizer_common/tests/Android.bp:37:1 + # name: "san_test-Nolibc" + # + # libnativehelper/Android.bp + # subdirs = ["tests"] + # + # system/core/bootstat/Android.bp + # whole_static_libs: ["libgtest_prod"], + # + # external/compiler-rt/lib/asan/Android.bp + # name: "libasan_noinst_test", + # + sync bionic sync build sync build/make @@ -363,34 +391,42 @@ android_repo_v8() { sync external/boringssl sync external/clang sync external/compiler-rt + sync external/elfutils sync external/jemalloc sync external/libcxx sync external/libcxxabi - sync external/libnl + # TODO(ahuszagh) Remove? + #sync external/libnl + sync external/libevent sync external/libunwind sync external/libunwind_llvm sync external/llvm sync external/lzma sync external/mksh sync external/pcre + sync external/safe-iop sync external/selinux sync external/zlib - sync hardware/interfaces - sync hardware/libhardware - sync frameworks/native + # TODO(ahuszagh) Remove? + #sync hardware/interfaces + #sync hardware/libhardware + #sync frameworks/native sync libnativehelper sync prebuilts/build-tools sync prebuilts/clang/host/linux-x86 + # TODO(ahuszagh) clang-tools isn't available till 9.0 sync prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8 sync prebuilts/go/linux-x86 - # needed for libnativehelper_compat_libc++ sync prebuilts/ndk sync system/core - sync system/libfmq - sync system/libhidl - sync system/libhwbinder - sync system/media - sync system/tools/hidl + sync toolchain/binutils + + # TODO(ahuszagh) See if I can remove all of these + #sync system/libfmq + #sync system/libhidl + #sync system/libhwbinder + #sync system/media + #sync system/tools/hidl # TODO(ahuszagh) Android 8 is a mess # avoid build tests @@ -399,26 +435,21 @@ android_repo_v8() { rm bionic/tests/Android.bp rm bionic/benchmarks/Android.bp rm bionic/tests/libs/Android.bp - rm hardware/interfaces/tests/Android.bp - rm system/tools/hidl/test/Android.bp + # TODO(ahuszagh) Can probably remove this +# rm hardware/interfaces/tests/Android.bp +# rm system/tools/hidl/test/Android.bp # we don't need the relocation packer, and removing # the unittests from it is a bit of work. rm bionic/tools/relocation_packer/Android.bp rm bionic/tools/relocation_packer/Android.mk + rm system/core/libgrallocusage/Android.bp + rm system/core/libmemtrack/Android.bp + rm system/core/libsysutils/Android.bp + + # avoid java dependencies + rm external/lzma/Java/Tukaani/Android.mk - # extra utilities we don't need - rm frameworks/native/libs/vr/Android.bp - rm frameworks/native/services/Android.bp - rm frameworks/native/services/*/Android.bp - rm hardware/interfaces/automotive/Android.bp - rm hardware/interfaces/camera/Android.bp - rm system/libhidl/transport/Android.bp - rm system/media/alsa_utils/Android.bp - rm system/media/audio_route/Android.bp - rm system/media/audio_utils/tests/Android.bp - # TODO(ahuszagh) Need to remove the tests here - - ANDROID_MAJOR=8 python3 /remove_android_tests.py + remove_tests } # tested on 9.0.0_r1 (SDK 28) @@ -482,7 +513,7 @@ android_repo_v9() { rm bionic/tests/headers/Android.bp rm bionic/tests/headers/posix/Android.bp - ANDROID_MAJOR=9 python3 /remove_android_tests.py + remove_tests } # tested on 10.0.0_r47 (SDK 29) @@ -553,7 +584,19 @@ android_repo_v10() { rm bionic/tests/headers/Android.bp rm bionic/tests/headers/posix/Android.bp - ANDROID_MAJOR=10 python3 /remove_android_tests.py + remove_tests +} + +remove_tests() { + install_packages python3-pip + + export PYTHONPATH="${PYTHON_TMPDIR}/lib/python3.8/site-packages/":"${PYTHONPATH}" + mkdir -p "${PYTHON_TMPDIR}" + python3 -m pip install sly==0.4.0 --prefix "${PYTHON_TMPDIR}" + + # TODO(ahuszagh) Need to change this + # Need a custom entrypoint + ANDROID_MAJOR="${MAJOR_VERSION}" python3 "${PYTHON_TMPDIR}/remove_tests.py" } sync() { diff --git a/docker/android/android/__init__.py b/docker/android/android/__init__.py new file mode 100644 index 000000000..7becf09f2 --- /dev/null +++ b/docker/android/android/__init__.py @@ -0,0 +1,11 @@ +import sys + +# we run this script once every build, and we'd rather +# have much smaller image sizes, so copying without +# any bytecode is a better idea. +sys.dont_write_bytecode = True + +__all__ = [ + "make", + "soong", +] diff --git a/docker/android/android/__pycache__/__init__.cpython-310.pyc b/docker/android/android/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 000000000..3dbd1b085 Binary files /dev/null and b/docker/android/android/__pycache__/__init__.cpython-310.pyc differ diff --git a/docker/android/android/make.py b/docker/android/android/make.py new file mode 100644 index 000000000..818edeb7b --- /dev/null +++ b/docker/android/android/make.py @@ -0,0 +1,66 @@ +''' + make + ==== + + utilities to process makefiles +''' + +import re + +from . import util + +class Section: + def __init__(self, title, contents): + self.title = title + self.contents = contents + + def is_test(self): + return self.title is not None and 'test' in self.title.lower() + + def is_benchmark(self): + return self.title is not None and 'benchmark' in self.title.lower() + + def is_dev(self): + return self.is_test() or self.is_benchmark() + + def __repr__(self): + return self.contents + +class Makefile: + def __init__(self, sections): + self.sections = sections + + @staticmethod + def parse(path): + with open(path, 'r') as f: + contents = f.read() + + sep1 = r'#\s+=+' + sep2 = r'#\s+-+' + comment = r'[A-Za-z0-9._ -]+' + pat1 = fr'(?:{sep1}\n)?#\s+({comment})\n{sep1}' + pat2 = fr'(?:{sep2}\n)?#\s+({comment})\n{sep2}' + pattern = fr'(?:{pat1})|(?:{pat2})' + + sections = [] + matches = list(re.finditer(pattern, contents)) + if len(matches) == 0: + sections.append(Section(None, contents)) + else: + first = matches[0] + last = matches[-1] + if first.start() != 0: + sections.append(Section(None, contents[:first.start()])) + for (prev, nxt) in util.windows(matches, 2): + title = prev.group(1) or prev.group(2) + sections.append(Section(title, contents[prev.start():nxt.start()])) + title = last.group(1) or prev.group(2) + sections.append(Section(title, contents[last.start():])) + + return Makefile(sections) + + def filter_dev(self): + return Makefile([i for i in self.sections if not i.is_dev()]) + + def __repr__(self): + return ''.join([str(i) for i in self.sections]) diff --git a/docker/android/android/soong.py b/docker/android/android/soong.py new file mode 100644 index 000000000..779c4a356 --- /dev/null +++ b/docker/android/android/soong.py @@ -0,0 +1,137 @@ +''' + soong + ===== + + utilities to process soong blueprint files. these are + a go-like, json-like data file format similar. they + support nested objects, arrays, bools, strings, and + use of variables. for example: + + array = ["..."] + cc_defaults { + name: "target", + options: array, + flags: ["..."], + } + cc_test { + name: "test", + defaults: ["target"], + srcs: ["test.cc"], + nested: { + array: { + option: false, + }, + }, + } + + the specification can be found here: + https://source.android.com/docs/core/tests/development/blueprints +''' + +import sly + +# TODO(ahuszagh) Implement... + +from sly import Lexer, Parser + +# base character defs +_H = r'[0-9a-f]' +_NL = r'\n|\r\n|\r|\f' +_UNICODE = fr'\\{_H}{1,6}(\r\n|[ \t\r\n\f])?' +_ESCAPE = r'{_UNICODE}|\\[^\r\n\f0-9a-f]' + +class Lexer(sly.Lexer): + tokens = { BOOL, NUMBER, IDENT, STRING } + ignore = ' \t' + literals = { '[', ']', '{', '}', ':', ',', '=' } + + # Tokens + STRING = fr'"([^\n\r\f\\"]|\\{_NL}|{_ESCAPE})*"' + NUMBER = r'(?:\d+\.\d*)|(?:\.\d+)|(?:\d+)' + BOOL = '(?:true)|(?:false)' + IDENT = r'[a-zA-Z_][a-zA-Z0-9_]*' + + @_(r'\n+') + def newline(self, t): + self.lineno += t.value.count('\n') + + def error(self, t): + print(f"Illegal character '{t.value[0]}'") + self.index += 1 + +#@_('LBRACE new_scope statements RBRACE') +#def statements(self, p): +# # Action code +# ... +# pop_scope() # Return to previous scope +# +#@_('') +#def new_scope(self, p): +# # Create a new scope for local variables +# create_scope() +# ... + +class Parser(sly.Parser): + tokens = Lexer.tokens + +# # TODO(ahuszagh) Implement... +# precedence = ( +# ('left', '+', '-'), +# ('left', '*', '/'), +# ('right', 'UMINUS'), +# ) +# + def __init__(self): + self.ast = Ast([]) + + @_('IDENT "=" expr') + def assign(self, p): + import pdb; pdb.set_trace() + self.names[p.NAME] = p.expr + +# @_('expr') +# def statement(self, p): +# print(p.expr) +# +# @_('expr "+" expr') +# def expr(self, p): +# return p.expr0 + p.expr1 +# +# @_('expr "-" expr') +# def expr(self, p): +# return p.expr0 - p.expr1 +# +# @_('expr "*" expr') +# def expr(self, p): +# return p.expr0 * p.expr1 +# +# @_('expr "/" expr') +# def expr(self, p): +# return p.expr0 / p.expr1 +# +# @_('"-" expr %prec UMINUS') +# def expr(self, p): +# return -p.expr +# +# @_('"(" expr ")"') +# def expr(self, p): +# return p.expr +# +# @_('NUMBER') +# def expr(self, p): +# return p.NUMBER +# +# @_('NAME') +# def expr(self, p): +# try: +# return self.names[p.NAME] +# except LookupError: +# print("Undefined name '%s'" % p.NAME) +# return 0 + +class Ast: + def __init__(self, nodes): + self.nodes = nodes + +class Node: + pass diff --git a/docker/android/android/util.py b/docker/android/android/util.py new file mode 100644 index 000000000..b64f8ff04 --- /dev/null +++ b/docker/android/android/util.py @@ -0,0 +1,3 @@ +def windows(sequence, count): + for i in range(len(sequence) - count + 1): + yield sequence[i:i + count] diff --git a/docker/remove_android_tests.py b/docker/android/remove_tests.py similarity index 94% rename from docker/remove_android_tests.py rename to docker/android/remove_tests.py index 11437a724..177f15b8c 100644 --- a/docker/remove_android_tests.py +++ b/docker/android/remove_tests.py @@ -9,11 +9,16 @@ import os import re +# TODO(ahuszagh) Need a better parser... +# use `android` for this. +# This is the basic one, so it will do for now + def windows(sequence, count): for i in range(len(sequence) - count + 1): yield sequence[i:i + count] def process_blueprint(contents, path): + # TODO(ahuszagh) Can probably do this better... gtest_paths = ['system/core/libmetricslogger/Android.bp'] subdir_test_paths = [ 'system/media/audio_utils/Android.bp', @@ -108,4 +113,5 @@ def main(): process_file('bionic/libc/malloc_debug/Android.mk', process_makefile) if __name__ == '__main__': + # TODO(ahuszagh) Handle testing? main() diff --git a/docker/android/tests/Android.bp b/docker/android/tests/Android.bp new file mode 100644 index 000000000..61568036b --- /dev/null +++ b/docker/android/tests/Android.bp @@ -0,0 +1,59 @@ +sample_array = [ + "value1", + "value2", +] +cc_defaults { + name: "target", + cflags: [ + "-Wall", + "-fstrict-aliasing", + ], + option: true, + tidy_checks: sample_array, + tidy_checks_as_errors: sample_array, + array: [ + "-short", + "--root='/path/to/dir'", + ], +} +cc_library_static { + name: "static_lib", + srcs: [ + "tree.cc", + "lib.cc", + ], + include_dirs: ["bionic/libc"], + export_include_dirs: ["."], +} +cc_library { + name: "lib", + srcs: [ + "tree.cc", + "lib.cc", + ], + include_dirs: ["bionic/libc"], + export_include_dirs: ["."], +} +cc_test { + name: "test", + defaults: ["target"], + srcs: ["test.cc"], + nested: { + array: { + option: false, + }, + }, +} +cc_test_host { + name: "host_test", + include_dirs: ["path/to/lib"], + compile_multilib: "64", + static_libs: [ + "libm", + "libz", + ], + host_ldlibs: [ + "-ldl", + "-lzstd", + ], +} diff --git a/docker/android/tests/Android.mk b/docker/android/tests/Android.mk new file mode 100644 index 000000000..f3db77183 --- /dev/null +++ b/docker/android/tests/Android.mk @@ -0,0 +1,102 @@ +LOCAL_PATH := $(call my-dir) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := config.c +LOCAL_MODULE := config +LOCAL_SHARED_LIBRARIES := libcutils +LOCAL_CFLAGS := -Werror + +include $(BUILD_HOST_EXECUTABLE) + +LOCAL_PATH := $(call my-dir) + +# ----------------------------------------------------------------------------- +# Benchmarks. +# ----------------------------------------------------------------------------- + +test_tags := tests + +benchmark_c_flags := \ + -Wall -Wextra \ + -Werror \ + -fno-builtin \ + +benchmark_src_files := \ + benchmark_main.cc \ + bench.cc + +# Build benchmarks for the device. Run with: +# adb shell liblog-benchmarks +include $(CLEAR_VARS) +LOCAL_MODULE := benchmarks +LOCAL_MODULE_TAGS := tests +LOCAL_CFLAGS += $(benchmark_c_flags) +LOCAL_SHARED_LIBRARIES += libm libdl +LOCAL_SRC_FILES := $(benchmark_src_files) + +# ----------------------------------------------------------------------------- +# Unit tests. +# ----------------------------------------------------------------------------- + +test_c_flags := \ + -g \ + -Wall \ + -Werror + +################################## +# test executable +LOCAL_MODULE := module +LOCAL_SRC_FILES := src.c +LOCAL_SHARED_LIBRARIES := libcutils +LOCAL_CFLAGS := $(test_c_flags) +LOCAL_MODULE_RELATIVE_PATH := config-tests + +# Unit tests. +# ========================================================= + +include $(CLEAR_VARS) +LOCAL_MODULE := init_tests +LOCAL_SRC_FILES := \ + init_parser_test.cc \ + property_service_test.cc \ + service_test.cc \ + util_test.cc \ + +################################## +# test executable +LOCAL_MODULE := module +LOCAL_SRC_FILES := src.c +LOCAL_SHARED_LIBRARIES := libcutils +LOCAL_CFLAGS := $(test_c_flags) +LOCAL_MODULE_RELATIVE_PATH := config-tests +LOCAL_SHARED_LIBRARIES += \ + libcutils \ + libbase \ + +LOCAL_STATIC_LIBRARIES := libinit +LOCAL_SANITIZE := integer +LOCAL_CLANG := true +LOCAL_CPPFLAGS := -Wall -Wextra -Werror +include $(BUILD_NATIVE_TEST) + +# Other section. +# ========================================================= +include $(call all-makefiles-under,$(LOCAL_PATH)) + +# ============================================================================= +# Unit tests. +# ============================================================================= + +test_c_flags := \ + -g \ + -Wall \ + -Werror + +################################## +# test executable +LOCAL_MODULE := mod2 +LOCAL_SRC_FILES := mod.c +LOCAL_SHARED_LIBRARIES := libcutils +LOCAL_CFLAGS := $(test_c_flags) +LOCAL_MODULE_RELATIVE_PATH := mod2-tests diff --git a/docker/android/tests/README.md b/docker/android/tests/README.md new file mode 100644 index 000000000..ca1eabc5e --- /dev/null +++ b/docker/android/tests/README.md @@ -0,0 +1,8 @@ +android_tests +============= + +Contains sample Soong blueprint files and Makefiles to test removal of unittests for build configurations. + +This requires a Python3 interpreter, and therefore is not run as part of the core test suite. Running the test suite requires: +- sly >= 0.4 +- pytest >= 7 diff --git a/docker/android/tests/__pycache__/test_make.cpython-310-pytest-7.1.3.pyc b/docker/android/tests/__pycache__/test_make.cpython-310-pytest-7.1.3.pyc new file mode 100644 index 000000000..6144ab814 Binary files /dev/null and b/docker/android/tests/__pycache__/test_make.cpython-310-pytest-7.1.3.pyc differ diff --git a/docker/android/tests/test_make.py b/docker/android/tests/test_make.py new file mode 100644 index 000000000..2cb0a187b --- /dev/null +++ b/docker/android/tests/test_make.py @@ -0,0 +1,27 @@ +import os +import sys + +TEST_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, PROJECT_DIR) + +from android import make + +def test(): + path = os.path.join(TEST_DIR, 'Android.mk') + contents = open(path).read() + makefile = make.Makefile.parse(path) + assert str(makefile) == contents + assert len(makefile.sections) == 6 + + assert not makefile.sections[0].is_dev() + assert makefile.sections[1].is_dev() + assert makefile.sections[1].is_benchmark() + assert makefile.sections[2].is_dev() + assert makefile.sections[2].is_test() + assert makefile.sections[4].title == 'Other section.' + + filtered = makefile.filter_dev() + assert len(filtered.sections) == 2 + assert filtered.sections[0].title is None + assert filtered.sections[1].title == 'Other section.' diff --git a/docker/android/tests/test_soong.py b/docker/android/tests/test_soong.py new file mode 100644 index 000000000..ed318adb8 --- /dev/null +++ b/docker/android/tests/test_soong.py @@ -0,0 +1,32 @@ +import os +import sys + +TEST_DIR = os.path.dirname(os.path.realpath(__file__)) +PROJECT_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, PROJECT_DIR) + +from android import soong + +def test(): + path = os.path.join(TEST_DIR, 'Android.bp') + contents = open(path).read() + lexer = soong.Lexer() + tokens = list(lexer.tokenize(contents)) + import pdb; pdb.set_trace() + +# TODO(ahuszagh) Implement +# makefile = make.Makefile.parse(path) +# assert str(makefile) == contents +# assert len(makefile.sections) == 6 +# +# assert not makefile.sections[0].is_dev() +# assert makefile.sections[1].is_dev() +# assert makefile.sections[1].is_benchmark() +# assert makefile.sections[2].is_dev() +# assert makefile.sections[2].is_test() +# assert makefile.sections[4].title == 'Other section.' +# +# filtered = makefile.filter_dev() +# assert len(filtered.sections) == 2 +# assert filtered.sections[0].title is None +# assert filtered.sections[1].title == 'Other section.'