Skip to content

Commit

Permalink
Only configure partitions on ingest statefulset command (#2266)
Browse files Browse the repository at this point in the history
  • Loading branch information
lferran authored Jun 20, 2024
1 parent d2d1f0f commit 0240d91
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 4 deletions.
3 changes: 1 addition & 2 deletions nucliadb/src/nucliadb/ingest/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,8 +175,6 @@ async def main_subscriber_workers(): # pragma: no cover
def setup_configuration(): # pragma: no cover
setup_logging()

assign_partitions(settings)

errors.setup_error_handling(importlib.metadata.distribution("nucliadb").version)

if asyncio._get_running_loop() is not None:
Expand All @@ -190,6 +188,7 @@ def run_consumer() -> None: # pragma: no cover
- pull worker
"""
setup_configuration()
assign_partitions(settings)
asyncio.run(main_consumer())


Expand Down
13 changes: 11 additions & 2 deletions nucliadb/src/nucliadb/search/search/paragraphs.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,13 @@ async def get_paragraph_from_full_text(
"""
extracted_text = await get_field_extracted_text(field, cache=extracted_text_cache)
if extracted_text is None:
logger.warning(f"{field} extracted_text does not exist on DB yet")
logger.warning(
"Extracted_text for field does not exist on DB. This should not happen.",
extra={
"field_id": field.resource_unique_id,
"kbid": field.kbid,
},
)
return ""

if split not in (None, ""):
Expand Down Expand Up @@ -154,7 +160,10 @@ async def get_paragraph_text(
if orm_resource is None:
orm_resource = await get_resource_from_cache(kbid, rid)
if orm_resource is None:
logger.error(f"{kbid}/{rid}:{field} does not exist on DB")
logger.warning(
"Resource does not exist on DB. This should not happen.",
extra={"resource_id": rid, "kbid": kbid, "field": field},
)
return ""

_, field_type, field = field.split("/")
Expand Down

2 comments on commit 0240d91

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmark

Benchmark suite Current: 0240d91 Previous: 0d03d9f Ratio
tests/search/unit/search/test_fetch.py::test_highligh_error 2252.3361595221486 iter/sec (stddev: 0.0000024574457428006355) 2841.0684406726436 iter/sec (stddev: 0.000004954958228416619) 1.26

This comment was automatically generated by workflow using github-action-benchmark.

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmark

Benchmark suite Current: 0240d91 Previous: 0d03d9f Ratio
tests/search/unit/search/test_fetch.py::test_highligh_error 2218.1381259413447 iter/sec (stddev: 0.0000012331727902891375) 2841.0684406726436 iter/sec (stddev: 0.000004954958228416619) 1.28

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.