feat: add gcp token-based auth support (lancedb#3511)

Author: alex766

rust/lance-io/src/object_store.rs

@@ -21,7 +21,7 @@ use lance_core::utils::tokio::get_num_compute_intensive_cpus;
 use object_store::aws::{
     AmazonS3ConfigKey, AwsCredential as ObjectStoreAwsCredential, AwsCredentialProvider,
 };
-use object_store::gcp::GoogleCloudStorageBuilder;
+use object_store::gcp::{GcpCredential, GoogleCloudStorageBuilder};
 use object_store::{
     aws::AmazonS3Builder, azure::AzureConfigKey, gcp::GoogleConfigKey, local::LocalFileSystem,
     memory::InMemory, CredentialProvider, Error as ObjectStoreError, Result as ObjectStoreResult,
@@ -839,6 +839,10 @@ impl StorageOptions {
             })
             .collect()
     }
+
+    pub fn get(&self, key: &str) -> Option<&String> {
+        self.0.get(key)
+    }
 }
 
 impl From<HashMap<String, String>> for StorageOptions {
@@ -929,6 +933,14 @@ async fn configure_store(
             for (key, value) in storage_options.as_gcs_options() {
                 builder = builder.with_config(key, value);
             }
+            let token_key = "google_storage_token";
+            if let Some(storage_token) = storage_options.get(token_key) {
+                let credential = GcpCredential {
+                    bearer: storage_token.to_string(),
+                };
+                let credential_provider = Arc::new(StaticCredentialProvider::new(credential)) as _;
+                builder = builder.with_credentials(credential_provider);
+            }
             let store = builder.build()?;
             let store = Arc::new(store).traced();