Feature: Xcode archive errors

CHANGELOG.md

@@ -1,3 +1,14 @@
+Version 0.11.1
+-------------
+
+**Features**
+
+- Show unformatted Xcode build errors on `xcode-project build-ipa` invocations that fail due to errors on `xcodebuild archive`. [PR #153](https://github.com/codemagic-ci-cd/cli-tools/pull/153).
+
+**Development / Docs**
+
+- Add a generator `codemagic.utilities.backwards_file_reader.iter_backwards` that returns the lines of a file in reverse order. [PR #153](https://github.com/codemagic-ci-cd/cli-tools/pull/153).
+
 Version 0.11.0
 -------------
 

src/codemagic/__version__.py

@@ -1,5 +1,5 @@
 __title__ = 'codemagic-cli-tools'
 __description__ = 'CLI tools used in Codemagic builds'
-__version__ = '0.11.0'
+__version__ = '0.11.1'
 __url__ = 'https://github.com/codemagic-ci-cd/cli-tools'
 __licence__ = 'GNU General Public License v3.0'

src/codemagic/models/xcodebuild.py

@@ -2,6 +2,7 @@
 
 import os
 import pathlib
+import re
 import shlex
 import subprocess
 import sys
@@ -10,12 +11,15 @@
 from functools import reduce
 from operator import add
 from typing import IO
+from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Union
 
 from codemagic.cli import CliProcess
 from codemagic.mixins import RunningCliAppMixin
 from codemagic.utilities import log
+from codemagic.utilities.backwards_file_reader import iter_backwards
 from codemagic.utilities.levenshtein_distance import levenshtein_distance
 
 from .export_options import ExportOptions
@@ -203,7 +207,16 @@ def archive(self,
         xcarchive = pathlib.Path(temp_dir)
 
         cmd = self._construct_archive_command(xcarchive, export_options, xcargs, custom_flags)
-        self._run_command(cmd, f'Failed to archive {self.workspace or self.project}')
+        try:
+            self._run_command(cmd, f'Failed to archive {self.workspace or self.project}')
+        except IOError as error:
+            if not self.xcpretty:
+                raise
+            message, process = error.args
+            errors = _XcodebuildLogErrorFinder(self.logs_path).find_failure_logs()
+            if not errors:
+                raise
+            raise IOError('\n'.join([f'{message}. The following build commands failed:', '', errors]), process)
 
         return xcarchive
 
@@ -308,3 +321,69 @@ def execute(self, *args, **kwargs) -> XcodebuildCliProcess:
                 self._buffer = None
             if self.xcpretty:
                 self.xcpretty.flush()
+
+
+class _XcodebuildLogErrorFinder:
+
+    def __init__(self, log_path: Union[pathlib.Path, str]):
+        self._log_path = pathlib.Path(log_path)
+        self._backwards_log_iterator = iter_backwards(log_path)
+
+    def _get_failed_commands(self):
+        capture_lines = False
+        error_lines = []
+        for line in self._backwards_log_iterator:
+            line = line.strip()
+            if re.match(r'^\(\d+ failures?\)$', line):
+                capture_lines = True
+                continue
+            elif line == 'The following build commands failed:':
+                break
+            elif capture_lines and line:
+                error_lines.append(line)
+
+        return error_lines
+
+    def _get_failed_command_logs(self, failed_commands, max_lines) -> Dict[str, List[str]]:
+        lines_cache: List[str] = []
+        capture_lines = False
+        logs: Dict[str, List[str]] = {}
+        for line in self._backwards_log_iterator:
+            line = line.strip()
+            if not line:
+                continue
+            elif re.match(r'^\*\* [^ ]+ FAILED \*\*', line):  # Match lines like '** ARCHIVE FAILED **'
+                # From here on upwards we can start looking for error logs
+                capture_lines = True
+                continue
+            elif line in failed_commands:
+                # Found a line that refers to a failed command, save its logs
+                if line not in logs:
+                    # Capture up to 10 last lines of the logs
+                    log_lines = reversed(lines_cache[:max_lines])
+                    logs[line] = ['...', *log_lines] if len(lines_cache) > max_lines else list(log_lines)
+                if set(logs.keys()) == set(failed_commands):
+                    # All errors are processed, stop
+                    break
+                lines_cache = []
+            elif capture_lines:
+                lines_cache.append(line)
+
+        return logs
+
+    @classmethod
+    def _format_errors(cls, failed_command_logs: Dict[str, List[str]]) -> str:
+        lines = []
+        for error in sorted(failed_command_logs.keys()):
+            lines.append(error)
+            for log_line in failed_command_logs[error]:
+                lines.append(f'\t{log_line}')
+            lines.append('')
+        return '\n'.join(lines[:-1])
+
+    def find_failure_logs(self, max_lines_per_error=6) -> Optional[str]:
+        failed_commands = self._get_failed_commands()
+        failed_command_logs = self._get_failed_command_logs(failed_commands, max_lines_per_error)
+        if not failed_command_logs:
+            return None
+        return self._format_errors(failed_command_logs)

src/codemagic/utilities/backwards_file_reader.py

@@ -0,0 +1,39 @@
+import io
+from pathlib import Path
+from typing import Iterable
+from typing import Union
+
+
+def iter_backwards(file_path: Union[Path, str], buffer_size=8192) -> Iterable[str]:
+    """
+    A generator that returns the lines of a file in reverse order
+    """
+
+    file_path = Path(file_path)
+    file_size = file_path.stat().st_size
+
+    with file_path.open() as fd:
+        current_segment = None
+        offset_from_end = 0
+        unprocessed_size = fd.seek(0, io.SEEK_END)
+
+        while unprocessed_size > 0:
+            offset_from_end = min(file_size, offset_from_end + buffer_size)
+            fd.seek(file_size - offset_from_end)
+            buffer = fd.read(min(unprocessed_size, buffer_size))
+            unprocessed_size -= buffer_size
+            lines = buffer.splitlines()
+
+            if buffer.endswith('\n'):  # Previous segment was not a half line.
+                yield current_segment or ''
+            else:  # Previous segment did not end at a line break.
+                lines[-1] += current_segment or ''
+
+            # Retain the first line for next iteration as it might might have some
+            # portion not captured by current buffer.
+            current_segment = lines[0]
+            for line in reversed(lines[1:]):
+                yield line
+
+        if current_segment is not None:
+            yield current_segment

tests/utilities/test_backwards_file_reader.py

@@ -0,0 +1,30 @@
+import random
+import string
+import tempfile
+
+import pytest
+
+from codemagic.utilities.backwards_file_reader import iter_backwards
+
+
+@pytest.fixture
+def random_lines():
+    lines = []
+    for c in (string.ascii_letters + string.digits):
+        lines.extend([random.randint(80, 10000) * c, ''])
+    return lines
+
+
+@pytest.mark.parametrize('buffer_size', [2**i for i in range(5, 15)])
+def test_backwards_file_reader(buffer_size, random_lines):
+    with tempfile.NamedTemporaryFile(mode='w') as tf:
+        for i, line in enumerate(random_lines, 1):
+            # Do not write double line break in the very end
+            line_end = '\n' if i < len(random_lines) else ''
+            tf.write(f'{line}{line_end}')
+        tf.flush()
+
+        iterator = iter_backwards(tf.name, buffer_size)
+        backwards_lines = list(iterator)
+
+    assert list(reversed(backwards_lines)) == random_lines