Get rid of broadway

.gitignore

@@ -23,4 +23,8 @@ erl_crash.dump
 faktory_worker-*.tar
 
 # ignore elixir language server files
-/.elixir_ls
+/.elixir_ls
+
+.idea/
+.DS_Store
+.vscode

README.md

@@ -131,6 +131,17 @@ $ docker-compose up -d
 Creating faktory_worker_test          ... done
 Creating faktory_worker_test_tls      ... done
 Creating faktory_worker_password_test ... done
+```
 
-$ mix test
+
+Faktory have free open-source solution and enterprise edition. 
+
+If you don't have enterprise license then tests will fail on enterprise features (batching operations etc). In this case you can exclude them by tag `:enterprise`
+```sh
+$ mix test --exclude enterprise 
 ```
+
+If you are enterprise user all tests should pass
+```sh
+$ mix test
+```

config/config.exs

@@ -1,7 +1,7 @@
-use Mix.Config
+import Config
 
 if Mix.env() == :test do
-  config :logger, backends: []
+  config :logger, backends: [:console]
 
   # Set the worker startup delay to 1ms to ensure tests are speedy
   config :faktory_worker, worker_startup_delay: 1

lib/faktory_worker.ex

@@ -38,7 +38,6 @@ defmodule FaktoryWorker do
         [
           {FaktoryWorker.QueueManager, opts},
           {FaktoryWorker.Pool, opts},
-          {FaktoryWorker.PushPipeline, opts},
           {FaktoryWorker.JobSupervisor, opts},
           {FaktoryWorker.WorkerSupervisor, opts}
         ]

lib/faktory_worker/batch.ex

@@ -7,11 +7,6 @@ defmodule FaktoryWorker.Batch do
   all jobs in a batch have completed, Faktory will queue a callback job. This
   allows building complex job workflows with dependencies.
 
-  Jobs pushed as part of a batch _must_ be pushed synchronously. This can be
-  done using the `skip_pipeline: true` option when calling `perform_async/2`. If
-  a job isn't pushed synchronously, you may encounter a race condition where the
-  batch is committed before all jobs have been pushed.
-
   ## Creating a batch
 
   A batch is created using `new!/1` and must provide a description and declare
@@ -25,9 +20,9 @@ defmodule FaktoryWorker.Batch do
   alias FaktoryWorker.Batch
 
   {:ok, bid} = Batch.new!(on_success: {MyApp.EmailReportJob, [], []})
-  MyApp.Job.perform_async([1, 2], custom: %{"bid" => bid}, skip_pipeline: true)
-  MyApp.Job.perform_async([3, 4], custom: %{"bid" => bid}, skip_pipeline: true)
-  MyApp.Job.perform_async([5, 6], custom: %{"bid" => bid}, skip_pipeline: true)
+  MyApp.Job.perform_async([1, 2], custom: %{"bid" => bid})
+  MyApp.Job.perform_async([3, 4], custom: %{"bid" => bid})
+  MyApp.Job.perform_async([5, 6], custom: %{"bid" => bid})
   Batch.commit(bid)
   ```
 
@@ -45,7 +40,7 @@ defmodule FaktoryWorker.Batch do
     def perform(arg1, arg2, bid) do
       Batch.open(bid)
 
-      MyApp.OtherJob.perform_async([1, 2], custom: %{"bid" => bid}, skip_pipeline: true)
+      MyApp.OtherJob.perform_async([1, 2], custom: %{"bid" => bid})
 
       Batch.commit(bid)
     end

lib/faktory_worker/connection.ex

@@ -115,7 +115,7 @@ defmodule FaktoryWorker.Connection do
 
   defp put_worker_args(args, opts) do
     process_wid = Keyword.get(opts, :process_wid)
-    sys_pid = System.get_pid()
+    sys_pid = System.pid()
     {:ok, hostname} = :inet.gethostname()
 
     worker_args = %{

lib/faktory_worker/job.ex

@@ -59,13 +59,10 @@ defmodule FaktoryWorker.Job do
 
   ## Synchronous job pushing
 
-  By default, jobs are pushed asynchronously to the Faktory server. To ensure a
-  job has been successfully submitted before continuing, jobs can be pushed
-  synchronously instead. To do this, pass the `:skip_pipeline` option with the
-  value of `true` to `perform_async/2`.
-
-  Synchronous pushing is required in certain situations to guarantee ordering,
-  such as when using the Faktory Enterprise batching feature.
+  Previous version used Broadway to send jobs and `:skip_pipeline` parameter was used to do it synchronously.
+  `:skip_pipeline` is not supported anymore.
+  Since Batch operations is a feature of Faktory Enterprise this library now sends any single job synchronously
+  and makes HTTP call to faktory server (see `FaktoryWorker.Batch`).
 
   ## Worker Configuration
 
@@ -88,7 +85,7 @@ defmodule FaktoryWorker.Job do
   means only values that implement the `Jason.Encoder` protocol are valid when calling the `perform_async/2` function.
   """
 
-  alias FaktoryWorker.{ConnectionManager, Random, Pool, Sandbox, Telemetry}
+  alias FaktoryWorker.{ConnectionManager, Random, Pool, Telemetry, Sandbox}
 
   # Look at supporting the following optional fields when pushing a job
   # priority
@@ -128,36 +125,17 @@ defmodule FaktoryWorker.Job do
 
   @doc false
   def perform_async(payload, opts) do
-    case Keyword.get(opts, :skip_pipeline, false) do
-      false ->
-        opts
-        |> push_pipeline_name()
-        |> perform_async(payload, opts)
-
-      true ->
-        opts
-        |> faktory_name()
-        |> push(payload)
-    end
-  end
-
-  @doc false
-  def perform_async(_, {:error, _} = error, _), do: error
-
-  def perform_async(pipeline_name, payload, opts) do
     if Sandbox.active?() do
       Sandbox.enqueue_job(
         String.to_existing_atom("Elixir." <> payload.jobtype),
         payload.args,
         opts
       )
+      {:ok, payload}
     else
-      message = %Broadway.Message{
-        acknowledger: {FaktoryWorker.PushPipeline.Acknowledger, :push_message, []},
-        data: {pipeline_name, payload}
-      }
-
-      Broadway.push_messages(pipeline_name, [message])
+      opts
+      |> faktory_name()
+      |> push(payload)
     end
   end
 
@@ -170,6 +148,7 @@ defmodule FaktoryWorker.Job do
   end
 
   @doc false
+  def push(_, invalid_payload = {:error, _}), do: invalid_payload
   def push(faktory_name, job) do
     faktory_name
     |> Pool.format_pool_name()
@@ -215,12 +194,6 @@ defmodule FaktoryWorker.Job do
     "The field '#{Atom.to_string(field)}' has an invalid value '#{inspect(value)}'"
   end
 
-  defp push_pipeline_name(opts) do
-    opts
-    |> faktory_name()
-    |> FaktoryWorker.PushPipeline.format_pipeline_name()
-  end
-
   defp faktory_name(opts) do
     Keyword.get(opts, :faktory_name, FaktoryWorker)
   end

mix.exs

@@ -27,7 +27,6 @@ defmodule FaktoryWorker.MixProject do
 
   defp deps do
     [
-      {:broadway, "~> 1.0.0"},
       {:certifi, "~> 2.5"},
       {:excoveralls, "~> 0.10", only: :test},
       {:jason, "~> 1.1"},

test/faktory_worker/connection_manager_test.exs

@@ -91,7 +91,7 @@ defmodule FaktoryWorker.ConnectionManagerTest do
                {{:ok, result}, _} = ConnectionManager.send_command(state, {:push, payload})
 
                assert result == "halt reason"
-             end) =~ "[warn]  [123456] Halt: halt reason"
+             end) |> String.match?(~r/\[warn.*(?<!ing)\]*\[123456\]*[Halt: halt reason]/)
     end
 
     test "should unset the connection when there is a socket failure" do

test/faktory_worker/job/batch_integration_test.exs

@@ -34,8 +34,7 @@ defmodule FaktoryWorker.BatchIntegrationTest do
 
       job_opts = [
         faktory_name: faktory_name,
-        custom: %{"bid" => bid},
-        skip_pipeline: true
+        custom: %{"bid" => bid}
       ]
 
       DefaultWorker.perform_async(["1"], job_opts)
@@ -72,8 +71,7 @@ defmodule FaktoryWorker.BatchIntegrationTest do
 
       job_opts = [
         faktory_name: faktory_name,
-        custom: %{"bid" => bid},
-        skip_pipeline: true
+        custom: %{"bid" => bid}
       ]
 
       DefaultWorker.perform_async(["1"], job_opts)

test/faktory_worker/job/job_integration_test.exs

@@ -9,7 +9,7 @@ defmodule FaktoryWorker.JobIntegrationTest do
 
   setup :flush_faktory!
 
-  describe "perform_async/3" do
+  describe "perform_async/2" do
     test "should send a new job to faktory" do
       faktory_name = :"Test_#{Random.string()}"
 
@@ -21,7 +21,8 @@ defmodule FaktoryWorker.JobIntegrationTest do
 
       job = Job.build_payload(DefaultWorker, %{hey: "there!"}, opts)
 
-      Job.perform_async(job, opts)
+      {:ok, job_sent} = Job.perform_async(job, opts)
+      assert job_sent == job
 
       assert_queue_size("default", 1)
 

test/faktory_worker/job/job_test.exs

@@ -184,15 +184,4 @@ defmodule FaktoryWorker.Job.JobTest do
     end
   end
 
-  describe "perform_async/3" do
-    test "should not send a bad payload" do
-      data = %{hey: "there!"}
-      opts = [queue: 123]
-      {:error, _} = payload = Job.build_payload(Test.Worker, data, opts)
-
-      {:error, error} = Job.perform_async(TestPipeline, payload, [])
-
-      assert error == "The field 'queue' has an invalid value '123'"
-    end
-  end
 end