feat: ConditionalPutCommitHandler for concurrency on S3, faster commit (lancedb#3483)

* Concurrent writes are now safe on S3. Closes lancedb#2793
* Moves all object stores to use new `ConditionalPutCommitHandler`,
reducing the number of IOPS to write a manifest from 3 (put,
copy-if-not-exists, delete) to just 1.
* Added optional `size` field to dynamodb, eliminating need for `HEAD`
IOP when opening a table. This means we can open a table with only 1
object store IOP when using dynamodb manifest store. Closes lancedb#2995

Author: wjones127

.github/workflows/python.yml

@@ -27,6 +27,9 @@ env:
   RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
   CI: "true"
+  # Color output for pytest is off by default.
+  PYTEST_ADDOPTS: "--color=yes"
+  FORCE_COLOR: "1"
 
 jobs:
   lint:
@@ -203,22 +206,6 @@ jobs:
       run:
         shell: bash
         working-directory: python
-    services:
-      minio:
-        image: lazybit/minio
-        ports:
-          - 9000:9000
-        env:
-          MINIO_ACCESS_KEY: ACCESSKEY
-          MINIO_SECRET_KEY: SECRETKEY
-        options: --name=minio --health-cmd "curl http://localhost:9000/minio/health/live"
-      dynamodb-local:
-        image: amazon/dynamodb-local
-        ports:
-          - 8000:8000
-        env:
-          AWS_ACCESS_KEY_ID: ACCESSKEY
-          AWS_SECRET_ACCESS_KEY: SECRETKEY
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/run_integtests/action.yml

@@ -9,6 +9,10 @@ runs:
     shell: bash
     run: |
       pip3 install $(ls target/wheels/pylance-*.whl)[tests,ray]
+  - name: Start localstack
+    shell: bash
+    run: |
+      docker compose -f docker-compose.yml up -d --wait
   - name: Run python tests
     shell: bash
     working-directory: python

.github/workflows/rust.yml

@@ -83,9 +83,8 @@ jobs:
           sudo apt update
           sudo apt install -y protobuf-compiler libssl-dev
           rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
-      - name: Start DynamoDB local for tests
-        run: |
-          docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local
+      - name: Start DynamodDB and S3
+        run: docker compose -f docker-compose.yml up -d --wait
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Run tests
@@ -130,9 +129,8 @@ jobs:
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
           cargo test --locked --features ${ALL_FEATURES} --no-run
-      - name: Start DynamoDB local for tests
-        run: |
-          docker run -d -e AWS_ACCESS_KEY_ID=DUMMYKEY -e AWS_SECRET_ACCESS_KEY=DUMMYKEY -p 8000:8000 amazon/dynamodb-local
+      - name: Start DynamodDB and S3
+        run: docker compose -f docker-compose.yml up -d --wait
       - name: Run tests
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`