Skip to content

Commit

Permalink
test bringup
Browse files Browse the repository at this point in the history
Signed-off-by: shamb0 <r.raajey@gmail.com>
  • Loading branch information
shamb0 committed Nov 21, 2024
1 parent 4f0da81 commit 2977ad4
Show file tree
Hide file tree
Showing 12 changed files with 264 additions and 361 deletions.
212 changes: 0 additions & 212 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 1 addition & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,7 @@ arrow-array = { version = "52.2", default-features = false }
arrow = { version = "52.2", default-features = false }
parquet = { version = "52.2", default-features = false, features = ["async"] }
redb = { version = "2.2" }
sqlx = { version = "0.8.2", features = [
"postgres",
"uuid",
], default-features = false }
sqlx = { version = "0.8.2", features = ["postgres", "uuid"] }
aws-config = "1.5"
pgvector = { version = "0.4.0", features = ["sqlx"], default-features = false }
aws-credential-types = "1.2"
Expand Down
5 changes: 4 additions & 1 deletion examples/index_md_into_pgvector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}

tracing::info!("Starting indexing pipeline");

indexing::Pipeline::from_loader(FileLoader::new(test_dataset_path).with_extensions(&["md"]))
.then_chunk(ChunkMarkdown::from_chunk_range(10..2048))
.then(MetadataQAText::new(llm_client.clone()))
Expand All @@ -101,7 +102,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.run()
.await?;

tracing::info!("PgVector Indexing test completed successfully");
tracing::info!("PgVector Indexing completed successfully");

for (i, question) in [
"What is SwiftIDE? Provide a clear, comprehensive summary in under 50 words.",
"How can I use SwiftIDE to connect with the Ethereum blockchain? Please provide a concise, comprehensive summary in less than 50 words.",
Expand All @@ -121,5 +123,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
}

tracing::info!("PgVector Indexing & retrieval test completed successfully");

Ok(())
}
7 changes: 6 additions & 1 deletion swiftide-core/src/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
//! `states::Answered`: The query has been answered
use derive_builder::Builder;

use crate::{util::debug_long_utf8, Embedding, SparseEmbedding};
use crate::{util::debug_long_utf8, AdvanceEmbedding, Embedding, SparseEmbedding};

type Document = String;

Expand All @@ -34,6 +34,9 @@ pub struct Query<STATE: QueryState> {

#[builder(default)]
pub sparse_embedding: Option<SparseEmbedding>,

#[builder(default)]
pub adv_embedding: Option<AdvanceEmbedding>,
}

impl<STATE: std::fmt::Debug + QueryState> std::fmt::Debug for Query<STATE> {
Expand All @@ -44,6 +47,7 @@ impl<STATE: std::fmt::Debug + QueryState> std::fmt::Debug for Query<STATE> {
.field("state", &self.state)
.field("transformation_history", &self.transformation_history)
.field("embedding", &self.embedding.is_some())
.field("adv_embedding", &self.adv_embedding.is_some())
.finish()
}
}
Expand Down Expand Up @@ -71,6 +75,7 @@ impl<STATE: Clone + QueryState> Query<STATE> {
transformation_history: self.transformation_history,
embedding: self.embedding,
sparse_embedding: self.sparse_embedding,
adv_embedding: self.adv_embedding,
}
}

Expand Down
18 changes: 18 additions & 0 deletions swiftide-core/src/type_aliases.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#![cfg_attr(coverage_nightly, coverage(off))]

use crate::indexing::EmbeddedField;
use serde::{Deserialize, Serialize};

pub type Embedding = Vec<f32>;
Expand All @@ -20,3 +21,20 @@ impl std::fmt::Debug for SparseEmbedding {
.finish()
}
}

#[derive(Serialize, Deserialize, Clone, PartialEq)]
pub struct AdvanceEmbedding {
pub embedded_field: EmbeddedField,
pub field_value: Vec<f32>,
}
pub type AdvanceEmbeddings = Vec<AdvanceEmbedding>;

impl std::fmt::Debug for AdvanceEmbedding {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// Start the debug struct formatting
f.debug_struct("AdvanceEmbedding")
.field("embedded_field", &self.embedded_field)
.field("field_value", &self.field_value)
.finish()
}
}
2 changes: 2 additions & 0 deletions swiftide-integrations/src/pgvector/fixtures.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ pub(crate) struct PgVectorTestData<'a> {
pub metadata: Option<indexing::Metadata>,
/// Vector embeddings with their corresponding fields
pub vectors: Vec<(indexing::EmbeddedField, Vec<f32>)>,
pub expected_in_results: bool,
pub use_adv_embedding_query: bool,
}

impl<'a> PgVectorTestData<'a> {
Expand Down
Loading

0 comments on commit 2977ad4

Please sign in to comment.