Improve macos support (#109)

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: yandex-taxi-testsuite package tests
+name: (Ubuntu) yandex-taxi-testsuite package tests
 
 on:
   push:
@@ -23,6 +23,7 @@ jobs:
             - '3.10'
             - '3.11'
             - '3.12'
+            - '3.13'
     steps:
     - name: Install database dependencies
       run: |

.github/workflows/macos-ci.yml

@@ -0,0 +1,65 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: (MacOS) yandex-taxi-testsuite package tests
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  build:
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+            - '3.9'
+            - '3.10'
+            - '3.11'
+            - '3.12'
+            - '3.13'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install database dependencies
+      run: |
+        brew tap mongodb/brew
+        brew update
+        brew install redis clickhouse rabbitmq postgresql mariadb kafka mongodb-community
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install python dependencies
+      run: |
+          brew install pytest virtualenv
+          make setup-dev-venv
+    - name: Lint with flake8
+      run: |
+          make venv-linters
+    - name: Check PEP8 with black
+      run: |
+          make venv-check-black
+    - name: Test with pytest
+      env:
+          TESTSUITE_ALLOW_ROOT: 1
+          TESTSUITE_DEBUG: 1
+          PYTEST_ARGS: "-vvs"
+      run: |
+          make venv-tests
+
+    - name: Run examples tests
+      env:
+          TESTSUITE_ALLOW_ROOT: 1
+          TESTSUITE_DEBUG: 1
+          PYTEST: pytest
+          PYTEST_ARGS: "-vvs"
+      run: |
+          make venv-test-examples
+
+    - name: Build documentation
+      if: matrix.python-version != '3.13'
+      run: |
+          make build-docs

docs/kafka.rst

@@ -19,6 +19,8 @@ If you already have Kafka installed and its location differs from
 ``/etc/kafka`` please specify
 ``KAFKA_HOME`` environment variable accordingly.
 
+On MacOS just install Kafka with ``brew``: ``brew install kafka``
+
 Installed Kafka **must** support KRaft_ protocol.
 
 Environment variables
@@ -27,7 +29,8 @@ Environment variables
 KAFKA_HOME
 ~~~~~~~~~~
 
-Use to override Kafka binaries dir. Default is ``/etc/kafka``
+Use to override Kafka binaries dir.
+Default is ``/etc/kafka`` for Linux and ``/opt/homebrew/opt/kafka/libexec`` for MacOS
 
 TESTSUITE_KAFKA_SERVER_HOST
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/rabbitmq.rst

@@ -16,16 +16,20 @@ RabbitMQ installation
 Consult official docs at https://www.rabbitmq.com/download.html
 
 If you already have RabbitMQ installed and its location differs from
-`/usr/lib/rabbitmq` please specify 
+`/usr/lib/rabbitmq` please specify
 ``TESTSUITE_RABBITMQ_BINDIR`` environment variable accordingly.
 
+On MacOS just install Rabbitmq with ``brew``: ``brew install rabbitmq``
+
 Environment variables
 ---------------------
 
 TESTSUITE_RABBITMQ_BINDIR
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Use to override rabbitmq binary dir. Default is ``/usr/lib/rabbitmq/bin/``
+Use to override rabbitmq binary dir.
+Default is ``/usr/lib/rabbitmq/bin/`` for Linux
+and ``/opt/homebrew/sbin/`` for MacOS
 
 TESTSUITE_RABBITMQ_TCP_PORT
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -45,16 +49,16 @@ one may customize this timeout via environment variable ``TESTSUITE_RABBITMQ_SER
 Customize ports
 ---------------
 
-Testsuite may start RabbitMQ with custom ports if 
+Testsuite may start RabbitMQ with custom ports if
 ``TESTSUITE_RABBITMQ_TCP_PORT`` or ``TESTSUITE_RABBITMQ_EPMD_PORT``
 environment variables are specified.
 
 Use external instance
 ---------------------
 
-Usage of external instance is not officially supported for now, 
+Usage of external instance is not officially supported for now,
 but if your instance is local you may try setting environment variable
-``TESTSUITE_RABBITMQ_TCP_PORT`` and pytest option ``--rabbitmq=1`` 
+``TESTSUITE_RABBITMQ_TCP_PORT`` and pytest option ``--rabbitmq=1``
 and see if it works.
 
 Usage example

tests/plugins/mockserver/test_unix_mockserver.py

@@ -1,5 +1,6 @@
 import aiohttp
 import pytest
+import platform
 
 from testsuite._internal import fixture_types
 from testsuite.daemons import service_client
@@ -8,6 +9,11 @@
 from typing import Any
 from typing import Dict
 
+if platform.system() == 'Darwin':
+    _MOCKSERVER_SOCKET = 'socket'
+else:
+    _MOCKSERVER_SOCKET = 'mockserver.socket'
+
 
 @pytest.fixture
 def unix_mockserver(
@@ -25,7 +31,7 @@ def unix_mockserver(
 @pytest.fixture(scope='session')
 async def _unix_mockserver(pytestconfig, tmp_path_factory):
     async with server.create_unix_server(
-        tmp_path_factory.mktemp('mockserver') / 'mockserver.socket',
+        tmp_path_factory.mktemp('mockserver') / _MOCKSERVER_SOCKET,
         loop=None,
         pytestconfig=pytestconfig,
     ) as result:

testsuite/databases/kafka/scripts/find-kafka.sh

@@ -4,6 +4,12 @@ MIN_JAVA_VERSION="8"
 MIN_KAFKA_VERSION_MAJOR="3"
 MIN_KAFKA_VERSION_MINOR="3"
 
+if [ "$(uname)" = "Darwin" ]; then
+    DEFAULT_PATH="/opt/homebrew/opt/kafka/libexec"
+else
+    DEFAULT_PATH="/etc/kafka"
+fi
+
 check_java() {
     if [ -n "$JAVA_HOME" ] && [ -x "$JAVA_HOME/bin/java" ];  then
         _java="$JAVA_HOME/bin/java"
@@ -17,7 +23,7 @@ check_java() {
     fi
 
     if [ "$_java" ]; then
-        version=$("$_java" -version 2>&1 | grep -oP 'version "?(1\.)?\K\d+')
+        version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | cut -d . -f 1)
         echo "Current Java version is $version"
         if [ "$version" -ge $MIN_JAVA_VERSION ]; then
             return 0
@@ -28,18 +34,33 @@ check_java() {
     fi
 }
 
+check_home() {
+    [ -x "$1/bin/kafka-run-class.sh" ] && [ -e "$1/config/kraft/server.properties" ]
+}
+
 find_kafka() {
-    if [ "x$KAFKA_HOME" = "x" ]; then
-        echo "
-    KAFKA_HOME env is not specified!!!.
-    Please download Kafka from https://kafka.apache.org/downloads,
-    unpack the archive and place path to it in KAFKA_HOME"
-        return 1
+    if [ "x$KAFKA_HOME" != "x" ]; then
+        if check_home "$KAFKA_HOME"; then
+            return 0
+        fi
+    fi
+    if check_home "$DEFAULT_PATH"; then
+        KAFKA_HOME="$DEFAULT_PATH"
+        return 0
     fi
-    return 0
+
+    echo "
+Kafka sources not found in KAFKA_HOME and in $DEFAULT_PATH !!!.
+Please download Kafka from https://kafka.apache.org/downloads,
+unpack the archive and place path to it in KAFKA_HOME.
+Note: For MacOS just install KAFKA with 'brew install kafka'
+"
+
+    return 1
 }
 
 check_kafka() {
+    echo "Kafka home: $KAFKA_HOME"
     kafka_bin_dir="$KAFKA_HOME/bin"
     echo "Kafka bin directory: $kafka_bin_dir"
     cd "$kafka_bin_dir"

testsuite/databases/kafka/scripts/service-kafka

@@ -56,8 +56,8 @@ start_kafka() {
         exit 1
     fi
 
-    mkdir -p $KAFKA_SOURCE_DIR
-    mkdir -p $KAFKA_LOGS_DIR
+    mkdir -p "$KAFKA_SOURCE_DIR"
+    mkdir -p "$KAFKA_LOGS_DIR"
 
     echo "Copying Kafka home to tmp (to patch server configs): $KAFKA_HOME -> $KAFKA_SOURCE_DIR"
     cp -r "$KAFKA_HOME"/* "$KAFKA_SOURCE_DIR/"
@@ -66,21 +66,21 @@ start_kafka() {
     echo "Generating cluster UUID..."
     CLUSTER_UUID_FILE=$KAFKA_TMPDIR/cluster_uuid.txt
     "$KAFKA_BIN_DIR/kafka-storage.sh" random-uuid > "$CLUSTER_UUID_FILE"
-    echo "Cluster UUID is $(cat $CLUSTER_UUID_FILE)"
+    echo "Cluster UUID is $(cat "$CLUSTER_UUID_FILE")"
 
     # For some reasons, kafka-storage.sh does not support --override option
     # to override the log.dirs, node.id and  controller.quorum.voters configuration params.
     # Though, manually substitute it in the file.
     echo "Patching server.properties. Setting logs dir to $KAFKA_LOGS_DIR"
-    sed -i 's/log\.dirs=.*/log\.dirs='"$(echo "$KAFKA_LOGS_DIR" | sed 's/\//\\\//g')"'/' "$KAFKA_CONF_FILE"
-    sed -i 's/node\.id=.*/node\.id='"$KAFKA_NODE_ID"'/' "$KAFKA_CONF_FILE"
-    sed -i 's/controller\.quorum\.voters=.*/controller.quorum.voters='"$KAFKA_NODE_ID"'@'"$KAFKA_SERVER_HOST:""$KAFKA_CONTROLLER_PORT"'/' "$KAFKA_CONF_FILE"
+    sed -i.backup 's/log\.dirs=.*/log\.dirs='"$(echo "$KAFKA_LOGS_DIR" | sed 's/\//\\\//g')"'/' "$KAFKA_CONF_FILE"
+    sed -i.backup 's/node\.id=.*/node\.id='"$KAFKA_NODE_ID"'/' "$KAFKA_CONF_FILE"
+    sed -i.backup 's/controller\.quorum\.voters=.*/controller.quorum.voters='"$KAFKA_NODE_ID"'@'"$KAFKA_SERVER_HOST:""$KAFKA_CONTROLLER_PORT"'/' "$KAFKA_CONF_FILE"
     patched_config=$(cat "$KAFKA_CONF_FILE" | grep -E "(log\.dirs|node\.id|controller\.quorum\.voters)")
     echo "Patched server config:\n$patched_config"
 
     echo "Formatting logs dir..."
     "$KAFKA_BIN_DIR/kafka-storage.sh" format \
-        --cluster-id $(cat $CLUSTER_UUID_FILE) \
+        --cluster-id $(cat "$CLUSTER_UUID_FILE") \
         --config $KAFKA_CONF_FILE \
         --ignore-formatted
 

testsuite/databases/kafka/service.py

@@ -57,7 +57,6 @@ def create_kafka_service(
         working_dir=working_dir,
         environment={
             'KAFKA_TMPDIR': working_dir,
-            'KAFKA_HOME': os.getenv('KAFKA_HOME', '/etc/kafka'),
             'KAFKA_SERVER_HOST': settings.server_host,
             'KAFKA_SERVER_PORT': str(settings.server_port),
             'KAFKA_CONTROLLER_PORT': str(settings.controller_port),

testsuite/databases/rabbitmq/scripts/find-rabbitmq.sh

@@ -1,27 +1,51 @@
 #!/bin/sh
 
+if [ "$(uname)" = "Darwin" ]; then
+    DEFAULT_PATH="/opt/homebrew/sbin"
+else
+    DEFAULT_PATH="/usr/lib/rabbitmq/bin"
+fi
+
 find_rabbitmq() {
-    if [ ! -x "$RABBITMQ_BINDIR/rabbitmq-server" ]; then
-        die "No rabbitmq-server script found. 
-Please install RabbitMQ following official instructions at 
-https://www.rabbitmq.com/download.html 
+    RABBITMQ_SERVER=
+    for dir in "$TESTSUITE_RABBITMQ_BINDIR" "$DEFAULT_PATH"; do
+        if [ -x "$dir/rabbitmq-server" ]; then
+          RABBITMQ_SERVER="$dir/rabbitmq-server"
+          break
+        fi
+    done
+    if [ "x$RABBITMQ_SERVER" = "x" ]; then
+        die "No rabbitmq-server script found.
+Please install RabbitMQ following official instructions at
+https://www.rabbitmq.com/download.html
 or set TESTSUITE_RABBITMQ_BINDIR environment variable to the directory containing
 actual rabbitmq scripts (it's usually /usr/lib/rabbitmq/bin/). Note that this directory
-differs from \"which rabbitmq-server\", 
+differs from \"which rabbitmq-server\",
 actual location can be found via inspecting sbin/rabbitmq-server script.
 "
     fi
-    if [ ! -x "$RABBITMQ_BINDIR/rabbitmq-server" ]; then
-        die "No rabbitmqctl script found. 
-Please install RabbitMQ following official instructions at 
-https://www.rabbitmq.com/download.html 
+
+    RABBITMQ_CTL=
+    for dir in "$TESTSUITE_RABBITMQ_BINDIR" "$DEFAULT_PATH"; do
+        if [ -x "$dir/rabbitmqctl" ]; then
+          RABBITMQ_CTL="$dir/rabbitmqctl"
+          break
+        fi
+    done
+    if [ "x$RABBITMQ_CTL" = "x" ]; then
+        die "No rabbitmqctl script found.
+Please install RabbitMQ following official instructions at
+https://www.rabbitmq.com/download.html
 or set TESTSUITE_RABBITMQ_BINDIR environment variable to the directory containing
 actual rabbitmq scripts (it's usually /usr/lib/rabbitmq/bin/). Note that this directory
-differs from \"which rabbitmqctl\", 
+differs from \"which rabbitmqctl\",
 actual location can be found via inspecting sbin/rabbitmqctl script.
 "
     fi
 
+    echo "Rabbitmq server: $RABBITMQ_SERVER"
+    echo "Rabbitmq ctl: $RABBITMQ_CTL"
+
     return 0
 }
 

testsuite/databases/rabbitmq/scripts/service-rabbitmq

@@ -18,6 +18,12 @@ fi
 if [ "x$RABBITMQ_EPMD_PORT" = "x" ]; then
     die "RABBITMQ_EPMD_PORT must be set"
 fi
+if [ "x$RABBITMQ_SERVER" = "x" ]; then
+    die "RABBITMQ_SERVER must be set"
+fi
+if [ "x$RABBITMQ_CTL" = "x" ]; then
+    die "RABBITMQ_CTL must be set"
+fi
 
 RABBITMQ_DATA_DIR=$RABBITMQ_TMPDIR/mnesia
 RABBITMQ_LOG_DIR=$RABBITMQ_TMPDIR/log
@@ -36,7 +42,7 @@ start_rabbitmq() {
     mkdir -p "$RABBITMQ_DATA_DIR"
     mkdir -p "$RABBITMQ_LOG_DIR"
     # '-detached' doesn't work with systemwide installation for some reason
-    "$RABBITMQ_BINDIR/rabbitmq-server" &
+    "$RABBITMQ_SERVER" &
 }
 
 start() {
@@ -45,7 +51,7 @@ start() {
 
 stop() {
     unset RABBITMQ_ERLANG_COOKIE
-    "$RABBITMQ_BINDIR/rabbitmqctl" shutdown -n "$RABBITMQ_NODENAME" --erlang-cookie="$V_RABBITMQ_ERLANG_COOKIE"
+    "$RABBITMQ_CTL" shutdown -n "$RABBITMQ_NODENAME" --erlang-cookie="$V_RABBITMQ_ERLANG_COOKIE"
     epmd -kill
     rm -rf "$RABBITMQ_TMPDIR"
 }

testsuite/databases/rabbitmq/service.py

@@ -38,10 +38,6 @@ def create_rabbitmq_service(
             'RABBITMQ_TMPDIR': working_dir,
             'RABBITMQ_TCP_PORT': str(settings.tcp_port),
             'RABBITMQ_EPMD_PORT': str(settings.epmd_port),
-            'RABBITMQ_BINDIR': utils.getenv_str(
-                key='TESTSUITE_RABBITMQ_BINDIR',
-                default='/usr/lib/rabbitmq/bin/',
-            ),
         },
         check_ports=[settings.tcp_port, settings.epmd_port],
         start_timeout=utils.getenv_float(