diff --git a/.github/workflows/markdown.yml b/.github/workflows/doc-links.yml similarity index 65% rename from .github/workflows/markdown.yml rename to .github/workflows/doc-links.yml index b78ebbc..7b36823 100644 --- a/.github/workflows/markdown.yml +++ b/.github/workflows/doc-links.yml @@ -1,15 +1,16 @@ +name: Documentation links + on: + push: + branches: [ main ] workflow_dispatch: -name: Markdown files test and TOC update - permissions: - contents: write - pull-requests: write + contents: read jobs: - markdown-test-and-update: - name: Markdown files test and update + markdown-test: + name: Markdown files test runs-on: ubuntu-latest steps: @@ -34,12 +35,3 @@ jobs: cd extension awesome_bot --files README.md --allow-dupe --allow-redirect --allow 401 --skip-save-results --base-url http://localhost:8080/ awesome_bot docs/*.md --allow-dupe --allow-redirect --allow 401 --skip-save-results --base-url http://localhost:8080/docs/ - - - name: Generate table of contents - uses: technote-space/toc-generator@v4 - with: - MAX_HEADER_LEVEL: 5 - COMMIT_NAME: CrowdSec Dev Bot - TARGET_PATHS: 'docs/*.md' - CHECK_ONLY_DEFAULT_BRANCH: true - CREATE_PR: true diff --git a/docs/DEVELOPER.md b/docs/DEVELOPER.md index b0bb46b..b0e45ec 100644 --- a/docs/DEVELOPER.md +++ b/docs/DEVELOPER.md @@ -11,6 +11,7 @@ - [Virtual environment](#virtual-environment) - [Install dependencies](#install-dependencies) - [Unit tests](#unit-tests) +- [Update documentation table of contents](#update-documentation-table-of-contents) - [Release process](#release-process) @@ -39,6 +40,24 @@ pip install -r requirements-dev.txt python -m pytest ``` + +## Update documentation table of contents + +To update the table of contents in the documentation, you can use [the `doctoc` tool](https://github.com/thlorenz/doctoc). + +First, install it: + +```bash +npm install -g doctoc +``` + +Then, run it in the documentation folder: + +```bash +doctoc docs/* +``` + + ## Release process We use [Semantic Versioning](https://semver.org/spec/v2.0.0.html) approach to determine the next version number of the SDK. diff --git a/src/cscapi/client.py b/src/cscapi/client.py index 4bd418c..e847fe2 100644 --- a/src/cscapi/client.py +++ b/src/cscapi/client.py @@ -9,7 +9,7 @@ import httpx import jwt -from cscapi.storage import MachineModel, ReceivedDecision, SignalModel, StorageInterface +from .storage import MachineModel, ReceivedDecision, SignalModel, StorageInterface from more_itertools import batched __version__ = metadata.version("cscapi").split("+")[0] @@ -153,55 +153,79 @@ def _send_signals_by_machine_id( self.logger.error( f"Machine {machine_to_process.machine_id} is marked as failing" ) + # machine_to_process contains only machine_id data + # To avoid erasing data, we need to fetch the full machine from the database + database_machine = self.storage.get_machine_by_id( + machine_to_process.machine_id + ) + machine_to_upsert = ( + database_machine if database_machine else machine_to_process + ) self.storage.update_or_create_machine( - replace(machine_to_process, is_failing=True) + replace(machine_to_upsert, is_failing=True) ) break for machine_to_process in machines_to_process_attempts: - machine_to_process = self._prepare_machine(machine_to_process) + try: + machine_to_process = self._prepare_machine(machine_to_process) + except httpx.HTTPStatusError as exc: + self.logger.error( + f"Error while preparing machine {machine_to_process.machine_id}: {exc}" + ) + machine_to_process.token = None + retry_machines_to_process_attempts.append(machine_to_process) + continue if machine_to_process.is_failing: self.logger.error( - f"skipping sending signals for machine {machine_to_process.machine_id} as it's marked as failing" + f"skipping sending signals for machine {machine_to_process.machine_id}" + f" as it's marked as failing" ) continue - self.logger.info( - f"sending signals for machine {machine_to_process.machine_id}" - ) sent_signal_ids = [] try: - sent_signal_ids = self._send_signals_to_capi( - machine_to_process.token, - signals_by_machineid[machine_to_process.machine_id], + self.logger.info( + f"sending signals for machine {machine_to_process.machine_id}" ) - sent_signal_ids_count = len(sent_signal_ids) - total_sent += sent_signal_ids_count - self.logger.info(f"sent {sent_signal_ids_count} signals") + if machine_to_process.token is not None: + sent_signal_ids = self._send_signals_to_capi( + machine_to_process.token, + signals_by_machineid[machine_to_process.machine_id], + ) + sent_signal_ids_count = len(sent_signal_ids) + total_sent += sent_signal_ids_count + self.logger.info(f"sent {sent_signal_ids_count} signals") + else: + self.logger.error( + f"skipping sending signals for machine {machine_to_process.machine_id}" + f" as it has no token" + ) + continue except httpx.HTTPStatusError as exc: self.logger.error( f"error while sending signals: {exc} for machine {machine_to_process.machine_id}" ) if exc.response.status_code == 401: - if attempt_count >= self.max_retries: - self.storage.update_or_create_machine( - replace(machine_to_process, is_failing=True) - ) - continue machine_to_process.token = None retry_machines_to_process_attempts.append(machine_to_process) continue if prune_after_send and sent_signal_ids: - self.logger.info( - f"pruning sent signals for machine {machine_to_process.machine_id}" - ) - self.storage.delete_signals(sent_signal_ids) - - self.logger.info( - f"sending metrics for machine {machine_to_process.machine_id}" - ) + try: + self.logger.info( + f"pruning sent signals for machine {machine_to_process.machine_id}" + ) + self.storage.delete_signals(sent_signal_ids) + except Exception as exc: + self.logger.error( + f"error while pruning signals: {exc} for machine {machine_to_process.machine_id}", + exc_info=True, + ) try: + self.logger.info( + f"sending metrics for machine {machine_to_process.machine_id}" + ) self._send_metrics_for_machine(machine_to_process) except httpx.HTTPStatusError as exc: self.logger.error( diff --git a/src/cscapi/storage.py b/src/cscapi/storage.py index 8e727f2..bb78e2b 100644 --- a/src/cscapi/storage.py +++ b/src/cscapi/storage.py @@ -97,7 +97,7 @@ def get_signals( raise NotImplementedError @abstractmethod - def get_machine_by_id(self, machine_id: str) -> MachineModel: + def get_machine_by_id(self, machine_id: str) -> Optional[MachineModel]: raise NotImplementedError @abstractmethod