build: update exon, add s3 tests (#81)

.github/workflows/test.yml

@@ -27,6 +27,13 @@ jobs:
 
       # setup a virtual environment
       - name: Setup Python and Run Tests
+        env:
+          AWS_DEFAULT_REGION: us-west-2
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+          AWS_ENDPOINT_URL: http://localhost:4566
+          AWS_ENDPOINT: http://localhost:4566
+          AWS_ALLOW_HTTP: true
         run: |
           python -m pip install --upgrade pip
           pip install virtualenv
@@ -38,10 +45,7 @@ jobs:
           cargo build
 
           # Develop and test w/o extras
-          maturin develop
-          pytest
-
-          # Develop and test with extras
           maturin develop --extras polars
-          pip install -e ".[polars]"
-          pytest
+
+          # Run tests
+          bash bin/test.sh

Cargo.toml

@@ -10,7 +10,7 @@ name = "biobear"
 [dependencies]
 arrow = {version = "49.0.0", features = ["pyarrow"]}
 datafusion = "34"
-exon = {version = "0.6.0", features = ["all"]}
+exon = {version = "0.7.0", features = ["all"]}
 pyo3 = "0.20"
 thiserror = "1.0"
 tokio = {version = "1", features = ["rt"]}

README.md

@@ -65,10 +65,6 @@ import biobear as bb
 
 session = bb.connect()
 
-# If you want to read from S3, you can register an object store
-# must have proper credentials set up
-session.register_object_store_from_url('s3://BUCKET')
-
 session.sql("""
 CREATE EXTERNAL TABLE gene_annotations_s3 STORED AS GFF LOCATION 's3://BUCKET/TenflaDSM28944/IMG_Data/Ga0451106_prodigal.gff'
 """)

bin/test.sh

@@ -0,0 +1,54 @@
+set -e
+
+DOCKER_IMAGE="public.ecr.aws/localstack/localstack:latest"
+
+source venv/bin/activate
+
+# Setup the trap.
+function teardown {
+    echo "Tearing down..."
+
+    # Find the docker container using the image.
+    CONTAINER_ID=$(docker ps -q -f ancestor=$DOCKER_IMAGE)
+
+    # Stop the container.
+    docker stop $CONTAINER_ID
+
+    echo "Teardown completed."
+}
+
+# check docker and aws cli are installed
+if ! command -v docker &> /dev/null
+then
+    echo "docker could not be found"
+    exit
+fi
+
+if ! command -v aws &> /dev/null
+then
+    echo "aws cli could not be found"
+    exit
+fi
+
+
+# Setup
+echo "Setting up..."
+
+# Start the localstack container.
+docker run -d -p 4566:4566 $DOCKER_IMAGE
+
+# Wait for the stack to start.
+sleep 2
+
+trap teardown EXIT
+
+# Create the test bucket.
+aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
+
+# Upload the test data.
+aws --endpoint-url=http://localhost:4566 s3api put-object --bucket test-bucket --key test.fasta --body ./python/tests/data/test.fasta
+
+# Make the bucket public.
+aws --endpoint-url=http://localhost:4566 s3api put-bucket-acl --bucket test-bucket --acl public-read
+
+pytest

python/tests/test_session.py

@@ -101,3 +101,13 @@ def test_to_record_batch_reader():
         rows += len(batch)
 
     assert rows == 2
+
+
+def test_read_from_s3():
+    """Test reading from s3."""
+    session = connect()
+
+    query = "SELECT * FROM fasta_scan('s3://test-bucket/test.fasta')"
+    arrow_table = session.sql(query).to_arrow()
+
+    assert len(arrow_table) == 2

requirements-dev.txt

@@ -1,3 +1,6 @@
 maturin
 pytest
 ruff
+awscli
+awscli-local
+polars