[tests-only] [full-ci] Add acceptance tests

.drone.star

@@ -47,7 +47,16 @@ config = {
     "phpstan": True,
     "javascript": False,
     "phpunit": True,
-    "acceptance": False,
+    "acceptance": {
+        "cli": {
+            "suites": [
+                "cliMain",
+            ],
+            "servers": [
+                "daily-master-qa",
+            ],
+        },
+    },
 }
 
 def main(ctx):

Makefile

@@ -29,6 +29,7 @@ PHPUNITDBG=phpdbg -qrr -d memory_limit=4096M -d zend.enable_gc=0 "$(PWD)/../../l
 PHP_CS_FIXER=php -d zend.enable_gc=0 vendor-bin/owncloud-codestyle/vendor/bin/php-cs-fixer
 PHAN=php -d zend.enable_gc=0 vendor-bin/phan/vendor/bin/phan
 PHPSTAN=php -d zend.enable_gc=0 vendor-bin/phpstan/vendor/bin/phpstan
+BEHAT_BIN=vendor-bin/behat/vendor/bin/behat
 
 market_doc_files=LICENSE README.md CHANGELOG.md
 market_src_dirs=appinfo img js lib templates vendor
@@ -39,6 +40,7 @@ dist_dir=$(build_dir)/dist
 # internal aliases
 composer_deps=vendor/
 js_deps=node_modules/
+acceptance_test_deps=vendor-bin/behat/vendor
 
 #
 # Catch-all rules
@@ -156,18 +158,18 @@ test-php-phpstan: vendor-bin/phpstan/vendor
 
 .PHONY: test-acceptance-api
 test-acceptance-api: ## Run API acceptance tests
-test-acceptance-api:
-	../../tests/acceptance/run.sh --remote --type api
+test-acceptance-api: $(acceptance_test_deps)
+	BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type api
 
 .PHONY: test-acceptance-cli
 test-acceptance-cli: ## Run CLI acceptance tests
-test-acceptance-cli:
-	../../tests/acceptance/run.sh --remote --type cli
+test-acceptance-cli: $(acceptance_test_deps)
+	BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type cli
 
 .PHONY: test-acceptance-webui
 test-acceptance-webui: ## Run webUI acceptance tests
-test-acceptance-webui:
-	../../tests/acceptance/run.sh --remote --type webUI
+test-acceptance-webui: $(acceptance_test_deps)
+	BEHAT_BIN=$(BEHAT_BIN) ../../tests/acceptance/run.sh --remote --type webUI
 
 .PHONY: test-php-codecheck
 test-php-codecheck:
@@ -204,3 +206,9 @@ vendor-bin/phpstan/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/phpstan
 
 vendor-bin/phpstan/composer.lock: vendor-bin/phpstan/composer.json
 	@echo phpstan composer.lock is not up to date.
+
+vendor-bin/behat/vendor: vendor/bamarni/composer-bin-plugin vendor-bin/behat/composer.lock
+	composer bin behat install --no-progress
+
+vendor-bin/behat/composer.lock: vendor-bin/behat/composer.json
+	@echo behat composer.lock is not up to date.

lib/Command/InstallApp.php

@@ -134,7 +134,7 @@ protected function execute(InputInterface $input, OutputInterface $output) {
 						$output->writeln("$appId: App installed.");
 					}
 				} catch (\Exception $ex) {
-					$output->writeln("<error> $appId: {$ex->getMessage()} </error>");
+					$output->writeln("<error>$appId: {$ex->getMessage()}</error>");
 					$this->exitCode = 1;
 				}
 			}

lib/Command/UpgradeApp.php

@@ -167,6 +167,7 @@ function ($elem) {
 					}
 				} else {
 					$output->writeln("$appId: Not installed ...");
+					$this->exitCode = 1;
 				}
 			} catch (\Exception $ex) {
 				$output->writeln("<error>$appId: {$ex->getMessage()}</error>");

tests/acceptance/config/behat.yml

@@ -0,0 +1,22 @@
+default:
+  autoload:
+     '': '%paths.base%/../features/bootstrap'
+
+  suites:
+    cliMain:
+      paths:
+        - '%paths.base%/../features/cliMain'
+      contexts:
+        - MarketContext:
+        - FeatureContext: &common_feature_context_params
+            baseUrl:  http://localhost:8080
+            adminUsername: admin
+            adminPassword: admin
+            regularUserPassword: 123456
+            ocPath: apps/testing/api/v1/occ
+        - LoggingContext:
+        - OccContext:
+        - OccAppManagementContext:
+
+  extensions:
+    Cjm\Behat\StepThroughExtension: ~

tests/acceptance/features/bootstrap/MarketContext.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+/**
+ * ownCloud
+ *
+ * @author Phil Davis <info@jankaritech.com>
+ * @copyright Copyright (c) 2022 Phil Davis info@jankaritech.com
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU AFFERO GENERAL PUBLIC LICENSE for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public
+ * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+use Behat\Behat\Context\Context;
+
+require_once 'bootstrap.php';
+
+/**
+ * Context for market specific steps
+ */
+class MarketContext implements Context {
+}

tests/acceptance/features/bootstrap/bootstrap.php

@@ -0,0 +1,30 @@
+<?php declare(strict_types=1);
+/**
+ * ownCloud
+ *
+ * @author Phil Davis <info@jankaritech.com>
+ * @copyright Copyright (c) 2022 Phil Davis info@jankaritech.com
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU AFFERO GENERAL PUBLIC LICENSE for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public
+ * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+require_once __DIR__ . '/../../../../../../tests/acceptance/features/bootstrap/bootstrap.php';
+
+$classLoader = new \Composer\Autoload\ClassLoader();
+$classLoader->addPsr4(
+	"",
+	__DIR__ . "/../../../../../../tests/acceptance/features/bootstrap",
+	true
+);
+$classLoader->register();

tests/acceptance/features/cliMain/marketInstallUpgradeUninstall.feature

@@ -0,0 +1,87 @@
+@cli
+Feature: install, upgrade and uninstall apps that are available in the market-place
+
+  # Note: testing this feature requires that the real market-place be online,
+  # working and reachable from the system-under-test.
+  # The activity app must not yet be installed on the system-under-test.
+  # Happy-path install, upgrade and uninstall are tested all in one scenario as a "user journey"
+  # because we need to install anyway, in order to test a normal uninstall.
+  Scenario: install, attempt to reinstall, upgrade and uninstall an app that is available in the market-place
+    # Note: use the activity app as the example to install
+    # it should be an app that is always available
+    When the administrator invokes occ command "market:install activity"
+    Then the command should have been successful
+    And the command output should be:
+    """
+    activity: Installing new app ...
+    activity: App installed.
+    """
+    And app "activity" should be enabled
+    # Attempt to install again and check the different message
+    When the administrator invokes occ command "market:install activity"
+    Then the command should have been successful
+    And the command output should be:
+    """
+    activity: App already installed and no update available
+    """
+    And app "activity" should be enabled
+    # Attempt to upgrade and check that no update is available
+    When the administrator invokes occ command "market:upgrade activity"
+    Then the command should have been successful
+    And the command output should be:
+    """
+    activity: No update available.
+    """
+    And app "activity" should be enabled
+    # Uninstall the app - to make sure that uninstall works, and to cleanup
+    When the administrator invokes occ command "market:uninstall activity"
+    Then the command should have been successful
+    And the command output should be:
+    """
+    activity: Un-Installing ...
+    activity: App uninstalled.
+    """
+    And app "activity" should not be in the apps list
+
+  Scenario: install an app that is not available in the market-place
+    When the administrator invokes occ command "market:install nonexistentapp"
+    Then the command should have failed with exit code 1
+    And the command output should be:
+    """
+    nonexistentapp: Installing new app ...
+    nonexistentapp: Unknown app (nonexistentapp)
+    """
+
+  Scenario: upgrade an app that is not available in the market-place
+    When the administrator invokes occ command "market:upgrade nonexistentapp"
+    Then the command should have failed with exit code 1
+    And the command output should be:
+    """
+    nonexistentapp: Not installed ...
+    """
+
+  Scenario: upgrade an app that is available in the market-place but not installed locally
+    When the administrator invokes occ command "market:upgrade activity"
+    Then the command should have failed with exit code 1
+    And the command output should be:
+    """
+    activity: Not installed ...
+    """
+
+  Scenario: uninstall an app that is not available in the market-place
+    When the administrator invokes occ command "market:uninstall nonexistentapp"
+    Then the command should have failed with exit code 1
+    And the command output should be:
+    """
+    nonexistentapp: Un-Installing ...
+    nonexistentapp: App (nonexistentapp) could not be uninstalled. Please check the server logs.
+    """
+
+  Scenario: uninstall an app that is available in the market-place but not installed locally
+    When the administrator invokes occ command "market:uninstall activity"
+    Then the command should have failed with exit code 1
+    And the command output should be:
+    """
+    activity: Un-Installing ...
+    activity: App (activity) could not be uninstalled. Please check the server logs.
+    """

tests/acceptance/features/cliMain/marketList.feature

@@ -0,0 +1,11 @@
+@cli
+Feature: list apps that are available in the market-place
+
+  # Note: testing this feature requires that the real market-place be online,
+  # working and reachable from the system-under-test.
+  Scenario: list the apps available in the market-place
+    When the administrator invokes occ command "market:list"
+    Then the command should have been successful
+    # The command lists all the apps that are available on the market-place
+    # Just check for an example app that should always be there
+    And the command output should contain the text "activity"

vendor-bin/behat/composer.json

@@ -0,0 +1,21 @@
+{
+    "config" : {
+        "platform": {
+            "php": "7.4"
+        }
+    },
+    "require": {
+        "behat/behat": "^3.9",
+        "behat/gherkin": "^4.9",
+        "behat/mink": "1.7.1",
+        "friends-of-behat/mink-extension": "^2.5",
+        "behat/mink-selenium2-driver": "^1.5",
+        "ciaranmcnulty/behat-stepthroughextension" : "dev-master",
+        "rdx/behat-variables": "^1.2",
+        "sensiolabs/behat-page-object-extension": "^2.3",
+        "symfony/translation": "^4.4",
+        "sabre/xml": "^2.2",
+        "guzzlehttp/guzzle": "^7.4",
+        "phpunit/phpunit": "^9.5"
+    }
+}