diff --git a/core/lib/canary/index/trieve/client.ex b/core/lib/canary/index/trieve/client.ex index 28a0d6e3..11eaf74b 100644 --- a/core/lib/canary/index/trieve/client.ex +++ b/core/lib/canary/index/trieve/client.ex @@ -4,7 +4,6 @@ defmodule Canary.Index.Trieve.Client do dataset = Application.fetch_env!(:canary, :trieve_dataset) Canary.rest_client( - receive_timeout: 2_000, base_url: "https://api.trieve.ai/api", headers: [ {"Content-Type", "application/json"}, @@ -62,19 +61,49 @@ defmodule Canary.Index.Trieve.Client do tags: tags } = chunk + tag_set = + if is_nil(tags) or tags == [] do + [ + format_for_tagset(:source_id, source_id), + format_for_tagset(:empty_tags) + ] + else + [ + format_for_tagset(:source_id, source_id) + | Enum.map(tags, &format_for_tagset(:tag, &1)) + ] + end + %{ tracking_id: tracking_id, group_tracking_ids: [group_tracking_id], link: url, chunk_html: content, metadata: meta, - tag_set: [ - format_for_tag(:source_id, source_id) - | Enum.map(tags, &format_for_tag(:tag, &1)) - ], + tag_set: tag_set, convert_html_to_text: false, upsert_by_tracking_id: true } + |> then(fn data -> + if is_struct(chunk[:created_at], DateTime) do + data + |> Map.merge(%{ + time_stamp: DateTime.to_iso8601(chunk.created_at) + }) + else + data + end + end) + |> then(fn data -> + if is_binary(chunk[:title]) do + data + |> Map.merge(%{ + fulltext_boost: %{boost_factor: 10, phrase: chunk.title} + }) + else + data + end + end) end) # https://docs.trieve.ai/api-reference/chunk/create-or-upsert-chunk-or-chunks @@ -108,53 +137,61 @@ defmodule Canary.Index.Trieve.Client do def search(query, opts \\ []) do tags = opts[:tags] source_ids = Keyword.fetch!(opts, :source_ids) - search_type = if(question?(query), do: :fulltext, else: :hybrid) + search_type = if(question?(query), do: :hybrid, else: :fulltext) + receive_timeout = if(question?(query), do: 3_000, else: 1_500) highlight_options = if question?(query) do %{ - highlight_window: 1, - highlight_max_length: 4, + highlight_window: 8, + highlight_max_length: 5, highlight_threshold: 0.5, highlight_strategy: :v1 } else %{ - highlight_window: 1, + highlight_window: 8, highlight_max_length: 2, highlight_threshold: 0.9, highlight_strategy: :exactmatch } end + filters = %{ + must: + [ + %{ + field: "tag_set", + match_any: Enum.map(source_ids, &format_for_tagset(:source_id, &1)) + }, + if(not is_nil(tags) and tags != [], + do: %{ + field: "tag_set", + match_any: [ + format_for_tagset(:empty_tags) + | Enum.map(tags, &format_for_tagset(:tag, &1)) + ] + }, + else: nil + ) + ] + |> Enum.reject(&is_nil/1) + } + # https://docs.trieve.ai/api-reference/chunk-group/search-over-groups case base() |> Req.post( + receive_timeout: receive_timeout, url: "/chunk_group/group_oriented_search", json: %{ query: query, - filters: %{ - must: - [ - %{ - field: "tag_set", - match_any: Enum.map(source_ids, &format_for_tag(:source_id, &1)) - }, - if(not is_nil(tags) and tags != [], - do: %{ - field: "tag_set", - match_any: Enum.map(tags, &format_for_tag(:tag, &1)) - }, - else: nil - ) - ] - |> Enum.reject(&is_nil/1) - }, + filters: filters, page: 1, page_size: 8, group_size: 3, search_type: search_type, score_threshold: 0.1, + recency_bias: 0.5, remove_stop_words: true, slim_chunks: false, typo_options: %{correct_typos: true}, @@ -178,9 +215,10 @@ defmodule Canary.Index.Trieve.Client do query |> String.split(" ") |> Enum.reject(&(&1 == "")) - |> Enum.count() > 2 + |> Enum.count() > 3 end - defp format_for_tag(:source_id, value), do: "source_id:#{value}" - defp format_for_tag(:tag, value), do: "tag:#{value}" + defp format_for_tagset(:empty_tags), do: "__empty_tags__" + defp format_for_tagset(:source_id, value), do: "__source_id:#{value}__" + defp format_for_tagset(:tag, value), do: "__tag:#{value}__" end diff --git a/core/lib/canary/searcher.ex b/core/lib/canary/searcher.ex index a68a1d9f..0fc7a159 100644 --- a/core/lib/canary/searcher.ex +++ b/core/lib/canary/searcher.ex @@ -1,37 +1,35 @@ defmodule Canary.Searcher do - @callback run(list(any()), String.t(), keyword()) :: {:ok, list(map())} | {:error, any()} + @callback run(String.t(), keyword()) :: {:ok, list(map())} | {:error, any()} - def run(sources, query, opts \\ []) do + def run(query, opts \\ []) do {cache, opts} = Keyword.pop(opts, :cache, false) if cache do - with {:error, _} <- get_cache(sources, query, opts), - {:ok, result} <- impl().run(sources, query, opts) do - set_cache(sources, query, opts, result) + with {:error, _} <- get_cache(query, opts), + {:ok, result} <- impl().run(query, opts) do + set_cache(query, opts, result) {:ok, result} end else - impl().run(sources, query, opts) + impl().run(query, opts) end end - defp set_cache(sources, query, opts, result) do - Cachex.put(:cache, key(sources, query, opts), result, ttl: :timer.minutes(3)) + defp set_cache(query, opts, result) do + Cachex.put(:cache, key(query, opts), result, ttl: :timer.minutes(3)) end - defp get_cache(sources, query, opts) do - case Cachex.get(:cache, key(sources, query, opts)) do + defp get_cache(query, opts) do + case Cachex.get(:cache, key(query, opts)) do {:ok, nil} -> {:error, :not_found} {:ok, hit} -> {:ok, hit} end end - defp key(sources, query, opts) do - sources - |> Enum.map(& &1.id) - |> Enum.join(",") - |> Kernel.<>(":" <> query) + defp key(query, opts) do + query |> Kernel.<>(":" <> Jason.encode!(opts[:tags])) + |> Kernel.<>(":" <> Jason.encode!(opts[:source_ids])) end defp impl(), do: Application.get_env(:canary, :searcher, Canary.Searcher.Default) @@ -42,53 +40,126 @@ defmodule Canary.Searcher.Default do require Ash.Query - def run(sources, query, _opts) do - {:ok, groups} = - Canary.Index.Trieve.Client.search(query, source_ids: Enum.map(sources, & &1.id)) + def run(query, opts) do + with {:ok, groups} <- Canary.Index.Trieve.Client.search(query, opts) do + matches = + groups + |> Enum.map(&transform_result/1) + |> Enum.reject(&is_nil/1) - matches = - groups - |> Enum.map(fn %{"group" => group, "chunks" => chunks} -> - chunks = - chunks - |> Enum.map(fn chunk -> + {:ok, matches} + end + end + + defp transform_result(%{ + "group" => %{"metadata" => %{"type" => "webpage"} = group_meta}, + "chunks" => chunks + }) do + chunks = + chunks + |> Enum.map(fn chunk -> + %{ + "chunk" => %{"metadata" => meta, "link" => url}, + "highlights" => highlights + } = chunk + + cond do + meta["title"] == group_meta["title"] -> + nil + + Enum.at(highlights, 0, nil) == nil -> + nil + + true -> %{ - "chunk" => %{"metadata" => meta, "link" => url}, - "highlights" => highlights - } = chunk - - cond do - meta["title"] == group["metadata"]["title"] -> - nil - - Enum.at(highlights, 0, nil) == nil -> - nil - - true -> - %{ - meta: meta, - url: url, - title: meta["title"], - excerpt: Enum.at(highlights, 0) - } - end - end) - |> Enum.reject(&is_nil/1) - - if chunks == [] do - nil + meta: meta, + url: url, + title: meta["title"], + excerpt: Enum.at(highlights, 0) + } + end + end) + |> Enum.reject(&is_nil/1) + + if chunks == [] do + nil + else + %{ + type: group_meta["type"], + url: group_meta["url"], + title: group_meta["title"], + meta: %{}, + sub_results: chunks + } + end + end + + defp transform_result(%{ + "group" => %{"metadata" => %{"type" => "github_issue"} = group_meta}, + "chunks" => chunks + }) do + chunks = + chunks + |> Enum.map(fn chunk -> + %{ + "chunk" => %{"metadata" => _meta, "link" => url}, + "highlights" => highlights + } = chunk + + if Enum.at(highlights, 0) do + %{ + url: url, + excerpt: Enum.at(highlights, 0) + } else + nil + end + end) + |> Enum.reject(&is_nil/1) + + %{ + type: group_meta["type"], + url: group_meta["url"], + title: group_meta["title"], + meta: %{ + closed: group_meta["closed"] + }, + sub_results: chunks + } + end + + defp transform_result(%{ + "group" => %{"metadata" => %{"type" => "github_discussion"} = group_meta}, + "chunks" => chunks + }) do + chunks = + chunks + |> Enum.map(fn chunk -> + %{ + "chunk" => %{"metadata" => _meta, "link" => url}, + "highlights" => highlights + } = chunk + + if Enum.at(highlights, 0) do %{ - type: group["metadata"]["type"], - url: group["metadata"]["url"], - title: group["metadata"]["title"], - meta: %{}, - sub_results: chunks + url: url, + excerpt: Enum.at(highlights, 0) } + else + nil end end) |> Enum.reject(&is_nil/1) - {:ok, matches} + %{ + type: group_meta["type"], + url: group_meta["url"], + title: group_meta["title"], + meta: %{ + closed: group_meta["closed"], + answered: group_meta["answered"] + }, + sub_results: chunks + } end end diff --git a/core/lib/canary/sources/document_create.ex b/core/lib/canary/sources/document_create.ex index 43cb7470..09e5ae60 100644 --- a/core/lib/canary/sources/document_create.ex +++ b/core/lib/canary/sources/document_create.ex @@ -1,6 +1,11 @@ defmodule Canary.Sources.Document.Create do use Ash.Resource.Change + alias Canary.Index.Trieve + alias Canary.Sources.Chunk + alias Canary.Sources.Webpage + alias Canary.Sources.GithubIssue + alias Canary.Sources.GithubDiscussion def init(opts) do if [ @@ -19,8 +24,6 @@ defmodule Canary.Sources.Document.Create do @impl true def batch_change(changesets, opts, _context) do - IO.inspect(changesets) - changesets |> Enum.map(fn changeset -> data = Ash.Changeset.get_argument(changeset, opts[:data_argument]) @@ -43,10 +46,10 @@ defmodule Canary.Sources.Document.Create do end) end - defp transform_fetcher_result(%Canary.Sources.Webpage.FetcherResult{} = data) do + defp transform_fetcher_result(%Webpage.FetcherResult{} = data) do local_chunks = data.items - |> Enum.map(fn _ -> %Canary.Sources.Chunk{index_id: Ash.UUID.generate()} end) + |> Enum.map(fn _ -> %Chunk{index_id: Ash.UUID.generate()} end) remote_chunks = data.items @@ -83,6 +86,87 @@ defmodule Canary.Sources.Document.Create do } end + defp transform_fetcher_result(%GithubIssue.FetcherResult{} = data) do + local_chunks = + data.items + |> Enum.map(fn _ -> %Chunk{index_id: Ash.UUID.generate()} end) + + remote_chunks = + data.items + |> Enum.map(fn item -> + %{ + content: item.content, + url: data.root.url, + title: data.root.title, + created_at: item.created_at, + meta: %{} + } + end) + + local_doc_meta = %Ash.Union{ + type: :github_issue, + value: %{ + node_id: data.root.node_id + } + } + + remote_group_meta = %{ + type: :github_issue, + title: data.root.title, + url: data.root.url, + closed: data.root.closed + } + + %{ + local_chunks: local_chunks, + remote_chunks: remote_chunks, + local_doc_meta: local_doc_meta, + remote_group_meta: remote_group_meta, + remote_tags: [] + } + end + + defp transform_fetcher_result(%GithubDiscussion.FetcherResult{} = data) do + local_chunks = + data.items + |> Enum.map(fn _ -> %Chunk{index_id: Ash.UUID.generate()} end) + + remote_chunks = + data.items + |> Enum.map(fn item -> + %{ + url: item.url, + content: item.content, + title: data.root.title, + created_at: item.created_at, + meta: %{} + } + end) + + local_doc_meta = %Ash.Union{ + type: :github_discussion, + value: %{ + node_id: data.root.node_id + } + } + + remote_group_meta = %{ + type: :github_discussion, + title: data.root.title, + url: data.root.url, + closed: data.root.closed, + answered: data.root.answered + } + + %{ + local_chunks: local_chunks, + remote_chunks: remote_chunks, + local_doc_meta: local_doc_meta, + remote_group_meta: remote_group_meta, + remote_tags: [] + } + end + @impl true def after_batch(changesets_and_results, opts, _context) do with :ok = create_groups(changesets_and_results, opts), diff --git a/core/lib/canary/sources/github_discussion_document_meta.ex b/core/lib/canary/sources/github_discussion_document_meta.ex index a7ce8506..eb4e5f30 100644 --- a/core/lib/canary/sources/github_discussion_document_meta.ex +++ b/core/lib/canary/sources/github_discussion_document_meta.ex @@ -2,27 +2,10 @@ defmodule Canary.Sources.GithubDiscussion.DocumentMeta do use Ash.Resource, data_layer: :embedded attributes do - attribute :title, :string, allow_nil?: false - attribute :url, :string, allow_nil?: false - attribute :closed, :boolean, allow_nil?: true - attribute :answered, :boolean, allow_nil?: true + attribute :node_id, :string, allow_nil?: false end actions do - defaults [:read, :destroy] - - create :create do - primary? true - - accept [:title, :url, :closed, :answered] - change {Canary.Change.NormalizeURL, input_argument: :url, output_attribute: :url} - end - - update :update do - primary? true - - accept [:title, :url, :closed, :answered] - change {Canary.Change.NormalizeURL, input_argument: :url, output_attribute: :url} - end + defaults [:read, :destroy, create: [:node_id], update: [:node_id]] end end diff --git a/core/lib/canary/sources/github_discussion_fetcher.ex b/core/lib/canary/sources/github_discussion_fetcher.ex index 0f285cf9..3385ffd7 100644 --- a/core/lib/canary/sources/github_discussion_fetcher.ex +++ b/core/lib/canary/sources/github_discussion_fetcher.ex @@ -1,31 +1,56 @@ defmodule Canary.Sources.GithubDiscussion.FetcherResult do - defstruct [ + alias Canary.Sources.GithubDiscussion + + defstruct [:root, :items] + @type t :: %__MODULE__{root: GithubDiscussion.Root.t(), items: list(GithubDiscussion.Item.t())} +end + +defmodule Canary.Sources.GithubDiscussion do + @fields [ :node_id, - :title, - :content, :url, + :content, :created_at, :author_name, - :author_avatar_url, - :comment, - :closed, - :answered + :author_avatar_url ] + def base_fields(), do: @fields +end + +defmodule Canary.Sources.GithubDiscussion.Root do + alias Canary.Sources.GithubDiscussion + + defstruct GithubDiscussion.base_fields() ++ [:title, :closed, :answered] + @type t :: %__MODULE__{ node_id: String.t(), - title: String.t(), - content: String.t(), url: String.t(), + content: String.t(), created_at: DateTime.t(), author_name: String.t(), author_avatar_url: String.t(), - comment: boolean(), + title: String.t(), closed: boolean(), answered: boolean() } end +defmodule Canary.Sources.GithubDiscussion.Item do + alias Canary.Sources.GithubDiscussion + + defstruct GithubDiscussion.base_fields() + + @type t :: %__MODULE__{ + node_id: String.t(), + url: String.t(), + content: String.t(), + created_at: DateTime.t(), + author_name: String.t(), + author_avatar_url: String.t() + } +end + defmodule Canary.Sources.GithubDiscussion.Fetcher do @default_discussion_n 100 @default_comment_n 100 @@ -92,34 +117,31 @@ defmodule Canary.Sources.GithubDiscussion.Fetcher do end defp transform_discussion_node(discussion) do - top = %GithubDiscussion.FetcherResult{ + root = %GithubDiscussion.Root{ node_id: discussion["id"], - title: discussion["title"], - content: discussion["body"], url: discussion["url"], + content: discussion["body"], created_at: discussion["createdAt"], author_name: discussion["author"]["login"], author_avatar_url: discussion["author"]["avatarUrl"], - comment: false, + title: discussion["title"], closed: discussion["closed"], answered: discussion["isAnswered"] } - comments = + items = discussion["comments"]["nodes"] |> Enum.map(fn comment -> - %GithubDiscussion.FetcherResult{ + %GithubDiscussion.Item{ node_id: comment["id"], - title: "", - content: comment["body"], url: comment["url"], + content: comment["body"], created_at: comment["createdAt"], author_name: comment["author"]["login"], - author_avatar_url: comment["author"]["avatarUrl"], - comment: true + author_avatar_url: comment["author"]["avatarUrl"] } end) - [top | comments] + %GithubDiscussion.FetcherResult{root: root, items: items} end end diff --git a/core/lib/canary/sources/github_discussion_syncer.ex b/core/lib/canary/sources/github_discussion_syncer.ex index b184d62f..cc084a40 100644 --- a/core/lib/canary/sources/github_discussion_syncer.ex +++ b/core/lib/canary/sources/github_discussion_syncer.ex @@ -18,12 +18,12 @@ defmodule Canary.Sources.GithubDiscussion.Syncer do defp destroy(source_id) do Document |> Ash.Query.filter(source_id == ^source_id) - |> Ash.bulk_destroy(:destroy, %{}, return_errors?: true, batch_size: 50) + |> Ash.bulk_destroy(:destroy, %{}, return_errors?: true) end defp create(source_id, incomings) do incomings - |> Enum.map(&%{source_id: source_id, fetcher_results: &1}) - |> Ash.bulk_create(Document, :create_github_discussion, return_errors?: true, batch_size: 50) + |> Enum.map(&%{source_id: source_id, fetcher_result: &1}) + |> Ash.bulk_create(Document, :create, return_errors?: true) end end diff --git a/core/lib/canary/sources/github_issue_document_meta.ex b/core/lib/canary/sources/github_issue_document_meta.ex index 9659f619..177bb195 100644 --- a/core/lib/canary/sources/github_issue_document_meta.ex +++ b/core/lib/canary/sources/github_issue_document_meta.ex @@ -2,26 +2,10 @@ defmodule Canary.Sources.GithubIssue.DocumentMeta do use Ash.Resource, data_layer: :embedded attributes do - attribute :title, :string, allow_nil?: false - attribute :url, :string, allow_nil?: false - attribute :closed, :boolean, allow_nil?: true + attribute :node_id, :string, allow_nil?: false end actions do - defaults [:read, :destroy] - - create :create do - primary? true - - accept [:title, :url, :closed] - change {Canary.Change.NormalizeURL, input_argument: :url, output_attribute: :url} - end - - update :update do - primary? true - - accept [:title, :url, :closed] - change {Canary.Change.NormalizeURL, input_argument: :url, output_attribute: :url} - end + defaults [:read, :destroy, create: [:node_id], update: [:node_id]] end end diff --git a/core/lib/canary/sources/github_issue_fetcher.ex b/core/lib/canary/sources/github_issue_fetcher.ex index 9b57ba02..1b0268ee 100644 --- a/core/lib/canary/sources/github_issue_fetcher.ex +++ b/core/lib/canary/sources/github_issue_fetcher.ex @@ -1,34 +1,55 @@ defmodule Canary.Sources.GithubIssue.FetcherResult do - defstruct [:items] - @type t :: %__MODULE__{items: list(Canary.Sources.GithubIssue.FetcherResultItem.t())} + alias Canary.Sources.GithubIssue + + defstruct [:root, :items] + @type t :: %__MODULE__{root: GithubIssue.Root.t(), items: list(GithubIssue.Item.t())} end -defmodule Canary.Sources.GithubIssue.FetcherResultItem do - defstruct [ +defmodule Canary.Sources.GithubIssue do + @fields [ :node_id, - :title, - :content, :url, + :content, :created_at, :author_name, - :author_avatar_url, - :comment, - :closed + :author_avatar_url ] + def base_fields(), do: @fields +end + +defmodule Canary.Sources.GithubIssue.Root do + alias Canary.Sources.GithubIssue + + defstruct GithubIssue.base_fields() ++ [:title, :closed] + @type t :: %__MODULE__{ node_id: String.t(), - title: String.t(), - content: String.t(), url: String.t(), + content: String.t(), created_at: DateTime.t(), author_name: String.t(), author_avatar_url: String.t(), - comment: boolean(), + title: String.t(), closed: boolean() } end +defmodule Canary.Sources.GithubIssue.Item do + alias Canary.Sources.GithubIssue + + defstruct GithubIssue.base_fields() + + @type t :: %__MODULE__{ + node_id: String.t(), + url: String.t(), + content: String.t(), + created_at: DateTime.t(), + author_name: String.t(), + author_avatar_url: String.t() + } +end + defmodule Canary.Sources.GithubIssue.Fetcher do @default_issue_n 100 @default_comment_n 100 @@ -97,33 +118,30 @@ defmodule Canary.Sources.GithubIssue.Fetcher do end defp transform_issue_node(issue) do - top = %GithubIssue.FetcherResultItem{ + root = %GithubIssue.Root{ node_id: issue["id"], - title: issue["title"], - content: issue["body"], url: issue["bodyUrl"], + content: issue["body"], created_at: issue["createdAt"], author_name: issue["author"]["login"], author_avatar_url: issue["author"]["avatarUrl"], - comment: false, + title: issue["title"], closed: issue["closed"] } - comments = + items = issue["comments"]["nodes"] |> Enum.map(fn comment -> - %GithubIssue.FetcherResultItem{ + %GithubIssue.Item{ node_id: comment["id"], - title: "", - content: comment["body"], url: comment["url"], + content: comment["body"], created_at: comment["createdAt"], author_name: comment["author"]["login"], - author_avatar_url: comment["author"]["avatarUrl"], - comment: true + author_avatar_url: comment["author"]["avatarUrl"] } end) - [top | comments] + %GithubIssue.FetcherResult{root: root, items: items} end end diff --git a/core/lib/canary/sources/github_issue_syncer.ex b/core/lib/canary/sources/github_issue_syncer.ex index 92f16117..be094f6e 100644 --- a/core/lib/canary/sources/github_issue_syncer.ex +++ b/core/lib/canary/sources/github_issue_syncer.ex @@ -18,12 +18,12 @@ defmodule Canary.Sources.GithubIssue.Syncer do defp destroy(source_id) do Document |> Ash.Query.filter(source_id == ^source_id) - |> Ash.bulk_destroy(:destroy, %{}, return_errors?: true, batch_size: 50) + |> Ash.bulk_destroy(:destroy, %{}, return_errors?: true) end defp create(source_id, incomings) do incomings - |> Enum.map(&%{source_id: source_id, fetcher_results: &1}) - |> Ash.bulk_create(Document, :create_github_issue, return_errors?: true, batch_size: 50) + |> Enum.map(&%{source_id: source_id, fetcher_result: &1}) + |> Ash.bulk_create(Document, :create, return_errors?: true) end end diff --git a/core/lib/canary/workers/github_discussion_processor.ex b/core/lib/canary/workers/github_discussion_processor.ex index c4f3d8bb..33ccca65 100644 --- a/core/lib/canary/workers/github_discussion_processor.ex +++ b/core/lib/canary/workers/github_discussion_processor.ex @@ -27,7 +27,7 @@ defmodule Canary.Workers.GithubDiscussionProcessor do defp process(%Source{id: source_id} = source) do with {:ok, incomings} <- GithubDiscussion.Fetcher.run(source), - :ok <- GithubDiscussion.Syncer.run(source_id, incomings) do + :ok <- GithubDiscussion.Syncer.run(source_id, Enum.to_list(incomings)) do :ok end end diff --git a/core/lib/canary/workers/github_issue_processor.ex b/core/lib/canary/workers/github_issue_processor.ex index 961bbed2..4861b282 100644 --- a/core/lib/canary/workers/github_issue_processor.ex +++ b/core/lib/canary/workers/github_issue_processor.ex @@ -1,7 +1,7 @@ defmodule Canary.Workers.GithubIssueProcessor do use Oban.Worker, queue: :github_processor, - max_attempts: 2, + max_attempts: 1, unique: [ period: cond do @@ -27,7 +27,7 @@ defmodule Canary.Workers.GithubIssueProcessor do defp process(%Source{id: source_id} = source) do with {:ok, incomings} <- GithubIssue.Fetcher.run(source), - :ok <- GithubIssue.Syncer.run(source_id, incomings) do + :ok <- GithubIssue.Syncer.run(source_id, Enum.to_list(incomings)) do :ok end end diff --git a/core/lib/canary_web/operations_controller.ex b/core/lib/canary_web/operations_controller.ex index 8049e166..af963e3e 100644 --- a/core/lib/canary_web/operations_controller.ex +++ b/core/lib/canary_web/operations_controller.ex @@ -28,7 +28,9 @@ defmodule CanaryWeb.OperationsController do end def search(conn, %{"query" => %{"text" => query, "tags" => tags}} = params) do - case Canary.Searcher.run(conn.assigns.sources, query, tags: tags, cache: cache?()) do + source_ids = Enum.map(conn.assigns.sources, & &1.id) + + case Canary.Searcher.run(query, source_ids: source_ids, tags: tags, cache: cache?()) do {:ok, matches} -> data = %{ matches: matches,