diff --git a/nucliadb/nucliadb/ingest/tests/fixtures.py b/nucliadb/nucliadb/ingest/tests/fixtures.py index bf64dc1fc9..b41232a0d1 100644 --- a/nucliadb/nucliadb/ingest/tests/fixtures.py +++ b/nucliadb/nucliadb/ingest/tests/fixtures.py @@ -251,7 +251,7 @@ async def indexing_utility_ingest(natsd): @pytest.fixture(scope="function") -async def _natsd_reset(natsd, event_loop): +async def _natsd_reset(natsd): nc = await nats.connect(servers=[natsd]) js = nc.jetstream() try: diff --git a/nucliadb/nucliadb/reader/tests/fixtures.py b/nucliadb/nucliadb/reader/tests/fixtures.py index 0f3537792f..30d8264ed6 100644 --- a/nucliadb/nucliadb/reader/tests/fixtures.py +++ b/nucliadb/nucliadb/reader/tests/fixtures.py @@ -50,7 +50,7 @@ def test_settings_reader(cache, gcs, fake_node, maindb_driver): # type: ignore @pytest.fixture(scope="function") -async def reader_api(test_settings_reader: None, local_files, event_loop): # type: ignore +async def reader_api(test_settings_reader: None, local_files): # type: ignore from nucliadb.reader.app import create_application application = create_application() diff --git a/nucliadb/nucliadb/train/tests/fixtures.py b/nucliadb/nucliadb/train/tests/fixtures.py index aff8e7b835..b0faac6865 100644 --- a/nucliadb/nucliadb/train/tests/fixtures.py +++ b/nucliadb/nucliadb/train/tests/fixtures.py @@ -321,7 +321,7 @@ def test_settings_train(cache, gcs, fake_node, maindb_driver): # type: ignore @pytest.fixture(scope="function") -async def train_api(test_settings_train: None, local_files, event_loop): # type: ignore +async def train_api(test_settings_train: None, local_files): # type: ignore from nucliadb.train.utils import start_train_grpc, stop_train_grpc await start_shard_manager() diff --git a/nucliadb/nucliadb/writer/api/v1/upload.py b/nucliadb/nucliadb/writer/api/v1/upload.py index 2b7812c654..b46f1484c2 100644 --- a/nucliadb/nucliadb/writer/api/v1/upload.py +++ b/nucliadb/nucliadb/writer/api/v1/upload.py @@ -294,7 +294,10 @@ async def _tus_post( await storage_manager.start(dm, path=path, kbid=kbid) await dm.save() - location = f"{request['path']}/{upload_id}" + # Find the URL for upload, with the same parameter as this call + location = api.url_path_for( + "Upload information", upload_id=upload_id, **request.path_params + ) return Response( status_code=201, headers={ @@ -325,7 +328,7 @@ async def tus_head_rslug_prefix( @api.head( - f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}/file/{{field}}/{TUSUPLOAD}/{{upload_id}}", + f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{path_rid}}/file/{{field}}/{TUSUPLOAD}/{{upload_id}}", tags=["Resource field TUS uploads"], status_code=200, openapi_extra={"x-operation-order": 3}, @@ -336,7 +339,7 @@ async def tus_head_rslug_prefix( async def tus_head_rid_prefix( request: Request, kbid: str, - rid: str, + path_rid: str, field: str, upload_id: str, ) -> Response: diff --git a/nucliadb/nucliadb/writer/tests/fixtures.py b/nucliadb/nucliadb/writer/tests/fixtures.py index a438c32439..9466dfdd27 100644 --- a/nucliadb/nucliadb/writer/tests/fixtures.py +++ b/nucliadb/nucliadb/writer/tests/fixtures.py @@ -49,7 +49,6 @@ async def writer_api( transaction_utility, processing_utility, tus_manager, - event_loop, ) -> AsyncIterator[Callable[[list[Enum], str, str], AsyncClient]]: nucliadb_settings.nucliadb_ingest = grpc_servicer.host from nucliadb.writer.app import create_application diff --git a/nucliadb/nucliadb/writer/tests/test_files.py b/nucliadb/nucliadb/writer/tests/test_files.py index a38d333027..10682c0964 100644 --- a/nucliadb/nucliadb/writer/tests/test_files.py +++ b/nucliadb/nucliadb/writer/tests/test_files.py @@ -28,7 +28,7 @@ from nucliadb_protos.resources_pb2 import FieldType from nucliadb_protos.writer_pb2 import BrokerMessage, ResourceFieldId -from nucliadb.writer.api.v1.router import KB_PREFIX, RSLUG_PREFIX +from nucliadb.writer.api.v1.router import KB_PREFIX, RESOURCE_PREFIX, RSLUG_PREFIX from nucliadb.writer.api.v1.upload import maybe_b64decode from nucliadb.writer.tus import TUSUPLOAD, UPLOAD, get_storage_manager from nucliadb_models.resource import NucliaDBRoles @@ -102,7 +102,6 @@ async def test_knowledgebox_file_tus_upload_root(writer_api, knowledgebox_writer data = io_bytes.read(min_chunk_size) while data != b"": resp = await client.head(url) - assert resp.headers["Upload-Length"] == f"0" assert resp.headers["Upload-Offset"] == f"{offset}" @@ -536,6 +535,47 @@ async def test_file_tus_upload_field_by_slug(writer_api, knowledgebox_writer, re assert len(data.read()) == len(raw_bytes) +@pytest.mark.asyncio +async def test_file_tus_upload_urls_field_by_resource_id( + writer_api, knowledgebox_writer, resource +): + kb = knowledgebox_writer + + async with writer_api(roles=[NucliaDBRoles.WRITER]) as client: + language = base64.b64encode(b"ca").decode() + filename = base64.b64encode(b"image.jpg").decode() + md5 = base64.b64encode(b"7af0916dba8b70e29d99e72941923529").decode() + headers = { + "tus-resumable": "1.0.0", + "upload-metadata": f"filename {filename},language {language},md5 {md5}", + "content-type": "image/jpg", + "upload-defer-length": "1", + } + + resp = await client.post( + f"/{KB_PREFIX}/{kb}/resource/idonotexist/file/field1/{TUSUPLOAD}", + headers=headers, + ) + assert resp.status_code == 404 + + resp = await client.post( + f"/{KB_PREFIX}/{kb}/resource/{resource}/file/field1/{TUSUPLOAD}", + headers=headers, + ) + assert resp.status_code == 201 + url = resp.headers["location"] + + # Check that we are using the resource for the whole file upload + assert f"{RESOURCE_PREFIX}/{resource}" in url + + # Make sure the returned URL works + resp = await client.head(url) + assert resp.status_code == 200 + + assert resp.headers["Upload-Length"] == "0" + assert resp.headers["Upload-Offset"] == "0" + + @pytest.mark.asyncio async def test_multiple_tus_file_upload_tries( writer_api, knowledgebox_writer, resource diff --git a/nucliadb/requirements.lock.txt b/nucliadb/requirements.lock.txt index 852dfbd60e..04fcca9ee7 100644 --- a/nucliadb/requirements.lock.txt +++ b/nucliadb/requirements.lock.txt @@ -1,63 +1,62 @@ -aiobotocore==2.5.2 +aiobotocore==2.11.2 aiodns==3.1.1 aiofiles==23.2.1 -aiohttp==3.9.1 +aiohttp==3.9.3 aioitertools==0.11.0 aiosignal==1.3.1 -anyio==3.7.1 +anyio==4.2.0 asgiref==3.7.2 -async-lru==2.0.2 +async-lru==2.0.4 async-timeout==4.0.3 asyncpg==0.29.0 -attrs==23.1.0 +attrs==23.2.0 backoff==2.2.1 -botocore==1.29.161 +botocore==1.34.34 cachetools==5.3.2 -certifi==2023.11.17 +certifi==2024.2.2 cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 -cryptography==41.0.7 +cryptography==42.0.2 Deprecated==1.2.14 ed25519==1.5 -exceptiongroup==1.2.0 -fastapi==0.103.1 +fastapi==0.109.2 fastapi-versioning==0.10.0 -frozenlist==1.4.0 +frozenlist==1.4.1 gcloud==0.18.3 -google-api-core==2.14.0 -google-api-python-client==2.109.0 -google-auth==2.23.4 -google-auth-httplib2==0.1.1 -google-cloud-core==2.3.3 -google-cloud-storage==2.13.0 +google-api-core==2.17.0 +google-api-python-client==2.117.0 +google-auth==2.27.0 +google-auth-httplib2==0.2.0 +google-cloud-core==2.4.1 +google-cloud-storage==2.14.0 google-crc32c==1.5.0 -google-resumable-media==2.6.0 +google-resumable-media==2.7.0 googleapis-common-protos==1.59.1 -grpc-stubs==1.53.0.3 -grpcio==1.59.3 -grpcio-channelz==1.59.3 -grpcio-health-checking==1.59.3 -grpcio-reflection==1.59.3 -grpcio-status==1.59.3 -grpcio-testing==1.59.3 -grpcio-tools==1.59.3 +grpc-stubs==1.53.0.5 +grpcio==1.60.1 +grpcio-channelz==1.60.1 +grpcio-health-checking==1.60.1 +grpcio-reflection==1.60.1 +grpcio-status==1.60.1 +grpcio-testing==1.60.1 +grpcio-tools==1.60.1 h11==0.14.0 httpcore==1.0.2 httplib2==0.22.0 -httpx==0.25.2 +httpx==0.26.0 idna==3.6 -importlib-metadata==6.8.0 +importlib-metadata==6.11.0 jmespath==1.0.1 -jwcrypto==1.5.1 -kubernetes-asyncio==28.2.1 +jwcrypto==1.5.3 +kubernetes_asyncio==29.0.0 lru-dict==1.3.0 memorylru==1.1.2 -mmh3==4.0.1 +mmh3==4.1.0 mrflagly==0.2.5 -multidict==6.0.4 +multidict==6.0.5 mypy-protobuf==3.4.0 -nats-py==2.6.0 +nats-py==2.5.0 nkeys==0.1.0 oauth2client==4.1.3 opentelemetry-api==1.21.0 @@ -73,40 +72,42 @@ opentelemetry-proto==1.21.0 opentelemetry-sdk==1.21.0 opentelemetry-semantic-conventions==0.42b0 opentelemetry-util-http==0.42b0 -orjson==3.9.10 +orjson==3.9.13 prometheus-client==0.19.0 -protobuf==4.22.3 +protobuf==4.25.2 +psutil==5.9.8 pyasn1==0.5.1 pyasn1-modules==0.3.0 pycares==4.4.0 pycparser==2.21 -pydantic==1.10.13 +pydantic==1.10.14 pydantic-argparse==0.8.0 PyJWT==2.8.0 pyparsing==3.1.1 python-dateutil==2.8.2 PyYAML==6.0.1 -redis==5.0.1 +redis==5.0.0 requests==2.31.0 rsa==4.9 -sentry-sdk==1.38.0 +sentry-sdk==1.40.3 six==1.16.0 sniffio==1.3.0 -starlette==0.27.0 +starlette==0.36.3 thrift==0.16.0 tikv_client==0.0.3 -types-aiofiles==23.2.0.0 -types-cachetools==5.3.0.5 +types-aiofiles==23.2.0.20240106 +types-cachetools==5.3.0.7 types-orjson==3.6.2 types-pkg-resources==0.1.3 types-protobuf==3.20.4.6 +types-psutil==5.9.5.20240205 types-requests==2.31.0.6 -types-setuptools==69.0.0.0 +types-setuptools==69.0.0.20240125 types-urllib3==1.26.25.14 -typing_extensions==4.8.0 +typing_extensions==4.9.0 uritemplate==4.1.1 urllib3==1.26.18 uvicorn==0.18.3 wrapt==1.16.0 -yarl==1.9.3 +yarl==1.9.4 zipp==3.17.0 diff --git a/nucliadb/requirements.txt b/nucliadb/requirements.txt index d2f83f897f..498dfa2464 100644 --- a/nucliadb/requirements.txt +++ b/nucliadb/requirements.txt @@ -2,14 +2,14 @@ uvicorn<0.19.0 pydantic_argparse nucliadb-node-binding>=2.26.0 -aiohttp>=3.9.0,<3.9.2 +aiohttp>=3.9.3 lru-dict>=1.1.7 backoff aiofiles>=0.8.0 -psutil==5.9.7 -types-psutil==5.9.5.17 +psutil>=5.9.7 +types-psutil>=5.9.5.17 types-aiofiles>=0.8.3 -protobuf==4.22.3 +protobuf>=4.22.3 types-protobuf>=3.19.20,<4.0 grpcio>=1.44.0 grpcio-health-checking>=1.44.0 @@ -23,7 +23,7 @@ types-setuptools # pydantic 2 need a careful migration pydantic>=1.9.0,<2.0 aiobotocore>=2.5.2 -botocore<1.30.0 +botocore>=1.34.0 google-cloud-storage gcloud oauth2client @@ -31,7 +31,7 @@ jwcrypto>=1.5.1 fastapi-versioning>=0.10.0 # At some point FastAPI will drop support for pydantic v1 -fastapi>=0.95.2,<=0.103.1 +fastapi>=0.95.2 sentry-sdk>=1.5.12 pyjwt>=2.4.0 mmh3>=3.0.0 @@ -60,8 +60,8 @@ jmespath>=1.0.0 idna>=3.3 sniffio>=1.2.0 -async_lru==2.0.4 -async-timeout==4.0.3 -cachetools==5.3.2 -types-cachetools==5.3.0.5 +async_lru>=2.0.4 +async-timeout>=4.0.3 +cachetools>=5.3.2 +types-cachetools>=5.3.0.5 kubernetes_asyncio \ No newline at end of file diff --git a/nucliadb_utils/requirements.txt b/nucliadb_utils/requirements.txt index 19a782abf9..e21f675936 100644 --- a/nucliadb_utils/requirements.txt +++ b/nucliadb_utils/requirements.txt @@ -1,9 +1,9 @@ pydantic>=1.8.2,<2.0 -aiohttp>=3.9.0,<3.9.2 +aiohttp>=3.9.3 prometheus-client>=0.12.0 types-requests>=2.27.7 mmh3>=3.0.0 -nats-py[nkeys]==2.6.0 +nats-py[nkeys]>=2.6.0 pyjwt>=2.4.0 memorylru>=1.1.2 mrflagly diff --git a/test-requirements.txt b/test-requirements.txt index f6de441272..8569020025 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -mypy==1.5.1 +mypy==1.6.1 pytest==7.4.2 pytest-asyncio~=0.18.0 pytest-cov==3.0.0