Assign jobs for FINISHED agents immediately after FINISHED heartbeat,…

… if data is already saved (#319)

### What's done:
* Check if TestExecutions for agent are updated
* Assign jobs for FINISHED agents immediately after FINISHED heartbeat
* Added notes on nginx deployment

Closes #309

save-backend/src/main/kotlin/org/cqfn/save/backend/controllers/TestExecutionController.kt

@@ -2,12 +2,14 @@ package org.cqfn.save.backend.controllers
 
 import org.cqfn.save.agent.TestExecutionDto
 import org.cqfn.save.backend.service.TestExecutionService
+import org.cqfn.save.domain.TestResultStatus
 import org.cqfn.save.test.TestDto
 import org.springframework.dao.DataAccessException
 import org.springframework.http.HttpStatus
 import org.springframework.http.ResponseEntity
 import org.springframework.transaction.annotation.Transactional
 import org.springframework.web.bind.annotation.GetMapping
+import org.springframework.web.bind.annotation.PathVariable
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.bind.annotation.RequestParam
@@ -36,6 +38,17 @@ class TestExecutionController(private val testExecutionService: TestExecutionSer
             .map { it.toDto() }
     }
 
+    /**
+     * @param agentContainerId id of agent's container
+     * @param status status for test executions
+     * @return a list of test executions
+     */
+    @GetMapping("/testExecutions/agent/{agentId}/{status}")
+    fun getTestExecutionsForAgentWithStatus(@PathVariable("agentId") agentContainerId: String,
+                                            @PathVariable status: TestResultStatus
+    ) = testExecutionService.getTestExecutions(agentContainerId, status)
+        .map { it.toDto() }
+
     /**
      * Returns number of TestExecutions with this [executionId]
      *

save-backend/src/main/kotlin/org/cqfn/save/backend/repository/TestExecutionRepository.kt

@@ -36,6 +36,15 @@ interface TestExecutionRepository : BaseEntityRepository<TestExecution> {
      */
     fun findByExecutionId(executionId: Long, pageable: Pageable): List<TestExecution>
 
+    /**
+     * Returns test executions for agent [agentContainerId] and status [status]
+     *
+     * @param agentContainerId
+     * @param status
+     * @return a list of test executions
+     */
+    fun findByAgentContainerIdAndStatus(agentContainerId: String, status: TestResultStatus): List<TestExecution>
+
     /**
      * Returns a TestExecution matched by a set of fields
      *

save-backend/src/main/kotlin/org/cqfn/save/backend/service/TestExecutionService.kt

@@ -43,6 +43,16 @@ class TestExecutionService(private val testExecutionRepository: TestExecutionRep
     internal fun getTestExecutions(executionId: Long, page: Int, pageSize: Int) = testExecutionRepository
         .findByExecutionId(executionId, PageRequest.of(page, pageSize))
 
+    /**
+     * Get test executions by [agentContainerId] and [status]
+     *
+     * @param agentContainerId
+     * @param status
+     * @return a list of test executions
+     */
+    internal fun getTestExecutions(agentContainerId: String, status: TestResultStatus) = testExecutionRepository
+        .findByAgentContainerIdAndStatus(agentContainerId, status)
+
     /**
      * Returns number of TestExecutions with this [executionId]
      *

save-deploy/README.md

@@ -75,4 +75,5 @@ manually placed in `save-orchestrator/build/resources/main` before build, and it
 # Server configuration
 ## Nginx
 Nginx is used as a reverse proxy, which allows access from external network to backend and some other services.
-File `save-deploy/reverse-proxy.conf` should be copied to `/etc/nginx/sites-available`.
+File `save-deploy/reverse-proxy.conf` should be copied to `/etc/nginx/sites-available`. Symlink should be created:
+`sudo ln -s /etc/nginx/sites-available/reverse-proxy.conf /etc/nginx/sites-enabled/` (or to `/etc/nginx/conf.d` on some distributions).

save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/controller/HeartbeatController.kt

@@ -3,6 +3,7 @@ package org.cqfn.save.orchestrator.controller
 import org.cqfn.save.agent.AgentState
 import org.cqfn.save.agent.ContinueResponse
 import org.cqfn.save.agent.Heartbeat
+import org.cqfn.save.agent.HeartbeatResponse
 import org.cqfn.save.agent.WaitResponse
 import org.cqfn.save.entities.AgentStatusDto
 import org.cqfn.save.orchestrator.config.ConfigProperties
@@ -60,15 +61,14 @@ class HeartbeatController(private val agentService: AgentService,
                     // if agent sends the first heartbeat, we try to assign work for it
                     AgentState.STARTING -> agentService.getNewTestsIds(heartbeat.agentId)
                     // if agent idles, we try to assign work, but also check if it should be terminated
-                    AgentState.IDLE -> agentService.getNewTestsIds(heartbeat.agentId)
-                        .doOnSuccess {
-                            if (it is WaitResponse) {
-                                initiateShutdownSequence(heartbeat.agentId)
-                            }
+                    AgentState.IDLE -> handleVacantAgent(heartbeat.agentId)
+                    AgentState.FINISHED -> agentService.checkSavedData(heartbeat.agentId).flatMap { isSavingSuccessful ->
+                        if (isSavingSuccessful) {
+                            handleVacantAgent(heartbeat.agentId)
+                        } else {
+                            // todo: if failure is repeated multiple times, re-assign missing tests once more
+                            Mono.just(WaitResponse)
                         }
-                    AgentState.FINISHED -> {
-                        agentService.checkSavedData()
-                        Mono.just(WaitResponse)
                     }
                     AgentState.BUSY -> Mono.just(ContinueResponse)
                     AgentState.BACKEND_FAILURE, AgentState.BACKEND_UNREACHABLE, AgentState.CLI_FAILED, AgentState.STOPPED_BY_ORCH -> Mono.just(WaitResponse)
@@ -79,6 +79,13 @@ class HeartbeatController(private val agentService: AgentService,
             }
     }
 
+    private fun handleVacantAgent(agentId: String): Mono<HeartbeatResponse> = agentService.getNewTestsIds(agentId)
+        .doOnSuccess {
+            if (it is WaitResponse) {
+                initiateShutdownSequence(agentId)
+            }
+        }
+
     /**
      * If agent was IDLE and there are no new tests - we check if the Execution is completed.
      * We get all agents for the same execution, if they are all done.

save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/service/AgentService.kt

@@ -3,7 +3,9 @@ package org.cqfn.save.orchestrator.service
 import org.cqfn.save.agent.AgentState
 import org.cqfn.save.agent.HeartbeatResponse
 import org.cqfn.save.agent.NewJobResponse
+import org.cqfn.save.agent.TestExecutionDto
 import org.cqfn.save.agent.WaitResponse
+import org.cqfn.save.domain.TestResultStatus
 import org.cqfn.save.entities.Agent
 import org.cqfn.save.entities.AgentStatus
 import org.cqfn.save.entities.AgentStatusDto
@@ -105,10 +107,16 @@ class AgentService {
                 .toBodilessEntity()
 
     /**
-     * @return nothing for now Fixme
+     * Check that no TestExecution for agent [agentId] have status READY
+     *
+     * @param agentId agent for which data is checked
+     * @return true if all executions have status other than `READY`
      */
-    @Suppress("FunctionOnlyReturningConstant")
-    fun checkSavedData() = true
+    fun checkSavedData(agentId: String): Mono<Boolean> = webClientBackend.get()
+        .uri("/testExecutions/agent/$agentId/${TestResultStatus.READY}")
+        .retrieve()
+        .bodyToMono<List<TestExecutionDto>>()
+        .map { it.isEmpty() }
 
     /**
      * If an error occurs, should try to resend tests