From bb8d3b033f38574b33fffa3e970d07c9673abbd1 Mon Sep 17 00:00:00 2001 From: Dmitry Vorotilin Date: Mon, 9 May 2022 19:32:55 +0200 Subject: [PATCH] Refactor --- CHANGELOG.md | 2 + README.md | 47 ++++------ lib/ferrum.rb | 1 - lib/ferrum/page.rb | 10 +-- lib/ferrum/page/screenshot.rb | 6 +- lib/ferrum/page/stream.rb | 38 ++++++++ lib/ferrum/page/tracing.rb | 103 ++++++++++------------ lib/ferrum/utils/stream.rb | 43 --------- spec/browser_spec.rb | 158 ---------------------------------- spec/page/tracing_spec.rb | 125 +++++++++++++++++++++++++++ 10 files changed, 231 insertions(+), 302 deletions(-) create mode 100644 lib/ferrum/page/stream.rb delete mode 100644 lib/ferrum/utils/stream.rb create mode 100644 spec/page/tracing_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 713cbb00..ad357ad6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ a block with this page, after which the page is closed. - `Ferrum::Cookies#set` ability to set cookie using `Ferrum::Cookies::Cookie` object - `Ferrum::Network#emulate_network_conditions` activates emulation of network conditions - `Ferrum::Network#offline_mode` puts browser into offline mode +- `Ferrum::Page#tracing` - instance of `Ferrum::Page::Tracing` for trace capabilities. +- `Ferrum::Page::Tracing#record(&block)` start/stop tracing for steps provided in passed block ### Changed diff --git a/README.md b/README.md index 7fe7f034..dd521dfa 100644 --- a/README.md +++ b/README.md @@ -1151,46 +1151,29 @@ browser.at_xpath("//*[select]").select(["1", "2"]) ## Tracing -You can use `tracing.record` to create a trace file which can be opened in Chrome DevTools or [timeline viewer](https://chromedevtools.github.io/timeline-viewer/). +You can use `tracing.record` to create a trace file which can be opened in Chrome DevTools or +[timeline viewer](https://chromedevtools.github.io/timeline-viewer/). ```ruby -browser.page.tracing.record(path: "trace.json") do - browser.go_to("https://www.google.com") +page.tracing.record(path: "trace.json") do + page.go_to("https://www.google.com") end ``` -#### tracing.record(\*\*options) : `Hash` +#### tracing.record(\*\*options) : `String` -- By default: returns Trace data from `Tracing.tracingComplete` event. -- When `path` specified: return `true` and store Trace data from `Tracing.tracingComplete` event to file. +Accepts block, records trace and by default returns trace data from `Tracing.tracingComplete` event as output. When +`path` is specified returns `true` and stores trace data into file. * options `Hash` - * `:path` (String) - `String` to save a Trace data output on the disk, not specified by default. - * `:encoding` (Symbol) - `:base64` | `:binary` setting only for memory Trace data output, `:binary` by default. - * `:timeout` (Integer) - [Timeout of promise](https://github.com/ruby-concurrency/concurrent-ruby/blob/52c08fca13cc3811673ea2f6fdb244a0e42e0ebe/lib/concurrent-ruby/concurrent/promises.rb#L986) to wait til event `Tracing.Complete` triggered that fills buffer/file, `nil` by default, means "wait forever". - * `:screenshots` (Boolean) - When true - Captures screenshots in the trace, `false` by default. - * `:included_categories` (Array[String]) - An array of categories that be included to tracing data, by default: - ```ruby - ["devtools.timeline", - "v8.execute", - "disabled-by-default-devtools.timeline", - "disabled-by-default-devtools.timeline.frame", - "toplevel", - "blink.console", - "blink.user_timing", - "latencyInfo", - "disabled-by-default-devtools.timeline.stack", - "disabled-by-default-v8.cpu_profiler", - "disabled-by-default-v8.cpu_profiler.hires"] - ``` - * `:excluded_categories` (Array[String]) - An array of categories that be excluded from tracing data, by default: - ```ruby - ["*"] - ``` - - See all categories by `browser.client.command("Tracing.getCategories")` - -Only one trace can be active at a time per browser. + * :path `String` save data on the disk, `nil` by default + * :encoding `Symbol` `:base64` | `:binary` encode output as Base64 or plain text. `:binary` by default + * :timeout `Float` wait until file streaming finishes in the specified time or raise error, defaults to `nil` + * :screenshots `Boolean` capture screenshots in the trace, `false` by default + * :trace_config `Hash` config for + [trace](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#type-TraceConfig), for categories + see [getCategories](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#method-getCategories), + only one trace config can be active at a time per browser. ## Thread safety ## diff --git a/lib/ferrum.rb b/lib/ferrum.rb index 2e6a0368..6354f2b5 100644 --- a/lib/ferrum.rb +++ b/lib/ferrum.rb @@ -3,7 +3,6 @@ require "ferrum/utils/platform" require "ferrum/utils/elapsed_time" require "ferrum/utils/attempt" -require "ferrum/utils/stream" require "ferrum/errors" require "ferrum/browser" require "ferrum/node" diff --git a/lib/ferrum/page.rb b/lib/ferrum/page.rb index 11d54ac6..3fdf74a6 100644 --- a/lib/ferrum/page.rb +++ b/lib/ferrum/page.rb @@ -11,6 +11,7 @@ require "ferrum/page/screenshot" require "ferrum/page/animation" require "ferrum/page/tracing" +require "ferrum/page/stream" require "ferrum/browser/client" module Ferrum @@ -40,11 +41,13 @@ def reset include Animation include Screenshot include Frames + include Stream attr_accessor :referrer attr_reader :target_id, :browser, :headers, :cookies, :network, - :mouse, :keyboard, :event + :mouse, :keyboard, :event, + :tracing def initialize(target_id, browser) @frames = {} @@ -63,6 +66,7 @@ def initialize(target_id, browser) @headers = Headers.new(self) @cookies = Cookies.new(self) @network = Network.new(self) + @tracing = Tracing.new(self) subscribe prepare_page @@ -216,10 +220,6 @@ def subscribed?(event) @client.subscribed?(event) end - def tracing - @tracing ||= Tracing.new(client: @client) - end - private def subscribe diff --git a/lib/ferrum/page/screenshot.rb b/lib/ferrum/page/screenshot.rb index 37d04178..5409d24c 100644 --- a/lib/ferrum/page/screenshot.rb +++ b/lib/ferrum/page/screenshot.rb @@ -26,8 +26,6 @@ module Screenshot A6: { width: 4.13, height: 5.83 } }.freeze - STREAM_CHUNK = 128 * 1024 - def screenshot(**opts) path, encoding = common_options(**opts) options = screenshot_options(path, **opts) @@ -42,9 +40,7 @@ def pdf(**opts) path, encoding = common_options(**opts) options = pdf_options(**opts).merge(transferMode: "ReturnAsStream") handle = command("Page.printToPDF", **options).fetch("stream") - Utils::Stream.fetch(encoding: encoding, path: path) do |read_stream| - read_stream.call(client: self, handle: handle) - end + stream_to(path: path, encoding: encoding, handle: handle) end def mhtml(path: nil) diff --git a/lib/ferrum/page/stream.rb b/lib/ferrum/page/stream.rb new file mode 100644 index 00000000..d69c2b6d --- /dev/null +++ b/lib/ferrum/page/stream.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Ferrum + class Page + module Stream + STREAM_CHUNK = 128 * 1024 + + def stream_to(path:, encoding:, handle:) + if path.nil? + stream_to_memory(encoding: encoding, handle: handle) + else + stream_to_file(path: path, handle: handle) + end + end + + def stream_to_file(path:, handle:) + File.open(path, "wb") { |f| stream(output: f, handle: handle) } + true + end + + def stream_to_memory(encoding:, handle:) + data = String.new # Mutable string has << and compatible to File + stream(output: data, handle: handle) + encoding == :base64 ? Base64.encode64(data) : data + end + + def stream(output:, handle:) + loop do + result = command("IO.read", handle: handle, size: STREAM_CHUNK) + chunk = result.fetch("data") + chunk = Base64.decode64(chunk) if result["base64Encoded"] + output << chunk + break if result["eof"] + end + end + end + end +end diff --git a/lib/ferrum/page/tracing.rb b/lib/ferrum/page/tracing.rb index bd4d26c2..da1851fc 100644 --- a/lib/ferrum/page/tracing.rb +++ b/lib/ferrum/page/tracing.rb @@ -3,79 +3,66 @@ module Ferrum class Page class Tracing - INCLUDED_CATEGORIES = %w[ - devtools.timeline - v8.execute - disabled-by-default-devtools.timeline - disabled-by-default-devtools.timeline.frame - toplevel - blink.console - blink.user_timing - latencyInfo - disabled-by-default-devtools.timeline.stack - disabled-by-default-v8.cpu_profiler - disabled-by-default-v8.cpu_profiler.hires - ].freeze - EXCLUDED_CATEGORIES = %w[ - * - ].freeze + EXCLUDED_CATEGORIES = %w[*].freeze + SCREENSHOT_CATEGORIES = %w[disabled-by-default-devtools.screenshot].freeze + INCLUDED_CATEGORIES = %w[devtools.timeline v8.execute disabled-by-default-devtools.timeline + disabled-by-default-devtools.timeline.frame toplevel blink.console + blink.user_timing latencyInfo disabled-by-default-devtools.timeline.stack + disabled-by-default-v8.cpu_profiler disabled-by-default-v8.cpu_profiler.hires].freeze + DEFAULT_TRACE_CONFIG = { + includedCategories: INCLUDED_CATEGORIES, + excludedCategories: EXCLUDED_CATEGORIES + }.freeze - def initialize(client:) - @client = client + def initialize(page) + @page = page + @subscribed_tracing_complete = false end - def record(options = {}, &block) - @options = { - timeout: nil, - screenshots: false, - encoding: :binary, - included_categories: INCLUDED_CATEGORIES, - excluded_categories: EXCLUDED_CATEGORIES, - **options - } - @promise = Concurrent::Promises.resolvable_future - subscribe_on_tracing_event - start - block.call - @client.command("Tracing.end") - @promise.value!(@options[:timeout]) + def record(path: nil, encoding: :binary, timeout: nil, trace_config: nil, screenshots: false) + @path, @encoding = path, encoding + @result = Concurrent::Promises.resolvable_future + trace_config ||= DEFAULT_TRACE_CONFIG.dup + + if screenshots + included = trace_config.fetch(:includedCategories, []) + trace_config.merge!(includedCategories: included | SCREENSHOT_CATEGORIES) + end + + subscribe_tracing_complete + + start(trace_config) + yield + stop + + @result.value!(timeout) end private - def start - @client.command( - "Tracing.start", - transferMode: "ReturnAsStream", - traceConfig: { - includedCategories: included_categories, - excludedCategories: @options[:excluded_categories] - } - ) + def start(config) + @page.command("Tracing.start", transferMode: "ReturnAsStream", traceConfig: config) end - def included_categories - included_categories = @options[:included_categories] - if @options[:screenshots] == true - included_categories = @options[:included_categories] | ["disabled-by-default-devtools.screenshot"] - end - included_categories + def stop + @page.command("Tracing.end") end - def subscribe_on_tracing_event - @client.on("Tracing.tracingComplete") do |event, index| - next if index.to_i != 0 + def subscribe_tracing_complete + return if @subscribed_tracing_complete - @promise.fulfill(stream(event.fetch("stream"))) - rescue StandardError => e - @promise.reject(e) + @page.on("Tracing.tracingComplete") do |event, index| + next if index.to_i != 0 + @result.fulfill(stream_handle(event["stream"])) + rescue => e + @result.reject(e) end + + @subscribed_tracing_complete = true end - def stream(handle) - Utils::Stream.fetch(encoding: @options[:encoding], path: @options[:path]) do |read_stream| - read_stream.call(client: @client, handle: handle) - end + def stream_handle(handle) + @page.stream_to(path: @path, encoding: @encoding, handle: handle) end end end diff --git a/lib/ferrum/utils/stream.rb b/lib/ferrum/utils/stream.rb deleted file mode 100644 index 2b689eb9..00000000 --- a/lib/ferrum/utils/stream.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -module Ferrum - module Utils - module Stream - STREAM_CHUNK = 128 * 1024 - - module_function - - def fetch(path:, encoding:, &block) - if path.nil? - stream_to_memory(encoding: encoding, &block) - else - stream_to_file(path: path, &block) - end - end - - def stream_to_file(path:, &block) - File.open(path, "wb") { |f| stream_to(f, &block) } - true - end - - def stream_to_memory(encoding:, &block) - data = String.new("") # Mutable string has << and compatible to File - stream_to(data, &block) - encoding == :base64 ? Base64.encode64(data) : data - end - - def stream_to(output, &block) - loop do - read_stream = lambda do |client:, handle:| - client.command("IO.read", handle: handle, size: STREAM_CHUNK) - end - result = block.call(read_stream) - data_chunk = result["data"] - data_chunk = Base64.decode64(data_chunk) if result["base64Encoded"] - output << data_chunk - break if result["eof"] - end - end - end - end -end diff --git a/spec/browser_spec.rb b/spec/browser_spec.rb index 3f88a778..ca3e4826 100644 --- a/spec/browser_spec.rb +++ b/spec/browser_spec.rb @@ -824,163 +824,5 @@ module Ferrum end end end - - context "tracing" do - let(:file_path) { "#{PROJECT_ROOT}/spec/tmp/trace.json" } - - it "outputs a trace" do - browser.page.tracing.record(path: file_path) do - browser.go_to("https://www.google.com") - end - expect(File.exist?(file_path)).to be(true) - ensure - FileUtils.rm_f(file_path) - end - - it "runs with custom options" do - browser.page.tracing.record( - path: file_path, - included_categories: ["disabled-by-default-devtools.timeline"], - excluded_categories: ["*"] - ) do - browser.go_to - end - expect(File.exist?(file_path)).to be(true) - content = File.read(file_path) - trace_config = JSON.parse(content)["metadata"]["trace-config"] - expect(JSON.parse(trace_config)["excluded_categories"]).to eq(["*"]) - expect(JSON.parse(trace_config)["included_categories"]).to eq(["disabled-by-default-devtools.timeline"]) - expect(JSON.parse(content)["traceEvents"].any? { |object| object["cat"] == "toplevel" }).to eq(false) - ensure - FileUtils.rm_f(file_path) - end - - it "runs with default categories" do - browser.page.tracing.record(path: file_path) do - browser.go_to - end - expect(File.exist?(file_path)).to be(true) - content = File.read(file_path) - trace_config = JSON.parse(content)["metadata"]["trace-config"] - expect(JSON.parse(trace_config)["excluded_categories"]).to eq(["*"]) - expect(JSON.parse(trace_config)["included_categories"]) - .to match_array(%w[ - devtools.timeline - v8.execute - disabled-by-default-devtools.timeline - disabled-by-default-devtools.timeline.frame - toplevel - blink.console - blink.user_timing - latencyInfo - disabled-by-default-devtools.timeline.stack - disabled-by-default-v8.cpu_profiler - disabled-by-default-v8.cpu_profiler.hires - ]) - expect(JSON.parse(content)["traceEvents"].any? { |object| object["cat"] == "toplevel" }).to eq(true) - ensure - FileUtils.rm_f(file_path) - end - - it "throws an exception if tracing on two pages" do - page = browser.create_page - browser.page.tracing.record(path: file_path) do - expect do - page.tracing.record(path: file_path) do - browser.go_to - end - end.to raise_exception(Ferrum::BrowserError) do |e| - expect(e.message).to eq("Tracing has already been started (possibly in another tab).") - end - end - end - - specify "handles tracing.Complete event once" do - file_path = "#{PROJECT_ROOT}/spec/tmp/trace.json" - browser.page.tracing.record(path: file_path) do - browser.go_to - expect(Utils::Stream).to receive(:stream_to_file).with(path: file_path).once.and_call_original - end - expect(File.exist?(file_path)).to be(true) - file_path2 = "#{PROJECT_ROOT}/spec/tmp/trace2.json" - browser.page.tracing.record(path: file_path2) do - browser.go_to - expect(Utils::Stream).to receive(:stream_to_file).with(path: file_path2).once.and_call_original - end - expect(File.exist?(file_path2)).to be(true) - file_path3 = "#{PROJECT_ROOT}/spec/tmp/trace3.json" - browser.page.tracing.record(path: file_path3) do - browser.go_to - expect(Utils::Stream).to receive(:stream_to_file).with(path: file_path3).once.and_call_original - end - expect(File.exist?(file_path3)).to be(true) - ensure - FileUtils.rm_f(file_path) - FileUtils.rm_f(file_path2) - FileUtils.rm_f(file_path3) - end - - it "returns encoded 64 buffer" do - trace = browser.page.tracing.record(encoding: :base64) do - browser.go_to - end - expect(File.exist?(file_path)).to be(false) - decode64_trace = Base64.decode64(trace) - expect(JSON.parse(decode64_trace)["traceEvents"].any?).to eq(true) - end - - it "returns buffer with no encoding" do - trace = browser.page.tracing.record do - browser.go_to - end - expect(File.exist?(file_path)).to be(false) - expect(JSON.parse(trace)["traceEvents"].any?).to eq(true) - end - - context "screenshots enabled" do - it "fills file with screenshot data" do - browser.page.tracing.record(path: file_path, screenshots: true) do - browser.go_to("/ferrum/grid") - end - expect(File.exist?(file_path)).to be(true) - content = JSON.parse(File.read(file_path)) - trace_events = content["traceEvents"] - trace_config = content["metadata"]["trace-config"] - expect(JSON.parse(trace_config)["included_categories"]).to include("disabled-by-default-devtools.screenshot") - expect(trace_events.any? { |object| object["name"] == "Screenshot" }).to eq(true) - ensure - FileUtils.rm_f(file_path) - end - - it "returns a buffer with screenshot data" do - trace = browser.page.tracing.record(screenshots: true) do - browser.go_to("/ferrum/grid") - end - expect(File.exist?(file_path)).to be(false) - trace_config = JSON.parse(trace)["metadata"]["trace-config"] - expect(JSON.parse(trace_config)["included_categories"]).to include("disabled-by-default-devtools.screenshot") - expect(JSON.parse(trace)["traceEvents"].any? { |object| object["name"] == "Screenshot" }).to eq(true) - end - end - - it "fails with provided error on any errors in stream output" do - execute_error = StandardError.new("error message") - expect do - browser.page.tracing.record(path: file_path) do - browser.go_to - expect(browser.page.tracing).to receive(:stream).with(kind_of(String)).once.and_raise(execute_error) - end - end.to raise_exception(execute_error, "error message") - expect(File.exist?(file_path)).to be(false) - end - - it "waits for promise fill with timeout when it provided" do - expect(browser.page.tracing).to receive(:subscribe_on_tracing_event).with(no_args) - trace = browser.page.tracing.record(timeout: 1) do - browser.go_to - end - expect(trace).to be_nil - end - end end end diff --git a/spec/page/tracing_spec.rb b/spec/page/tracing_spec.rb new file mode 100644 index 00000000..5d252066 --- /dev/null +++ b/spec/page/tracing_spec.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Ferrum + describe Page::Tracing do + let(:file_path) { "#{PROJECT_ROOT}/spec/tmp/trace.json" } + let(:file_path2) { "#{PROJECT_ROOT}/spec/tmp/trace2.json" } + let(:file_path3) { "#{PROJECT_ROOT}/spec/tmp/trace3.json" } + let(:content) { JSON.parse(File.read(file_path)) } + let(:trace_config) { JSON.parse(content["metadata"]["trace-config"]) } + + it "outputs a trace" do + page.tracing.record(path: file_path) { page.go_to } + + expect(File.exist?(file_path)).to be(true) + ensure + FileUtils.rm_f(file_path) + end + + it "runs with custom options" do + page.tracing.record( + path: file_path, + trace_config: { + includedCategories: ["disabled-by-default-devtools.timeline"], + excludedCategories: ["*"] + } + ) { page.go_to } + + expect(File.exist?(file_path)).to be(true) + expect(trace_config["excluded_categories"]).to eq(["*"]) + expect(trace_config["included_categories"]).to eq(["disabled-by-default-devtools.timeline"]) + expect(content["traceEvents"].any? { |o| o["cat"] == "toplevel" }).to eq(false) + ensure + FileUtils.rm_f(file_path) + end + + it "runs with default categories" do + page.tracing.record(path: file_path) { page.go_to } + + expect(File.exist?(file_path)).to be(true) + expect(trace_config["excluded_categories"]).to eq(["*"]) + expect(trace_config["included_categories"]) + .to match_array(%w[devtools.timeline v8.execute disabled-by-default-devtools.timeline + disabled-by-default-devtools.timeline.frame toplevel blink.console + blink.user_timing latencyInfo disabled-by-default-devtools.timeline.stack + disabled-by-default-v8.cpu_profiler disabled-by-default-v8.cpu_profiler.hires]) + expect(content["traceEvents"].any? { |o| o["cat"] == "toplevel" }).to eq(true) + ensure + FileUtils.rm_f(file_path) + end + + it "throws an exception if tracing is on two pages" do + page.tracing.record(path: file_path) do + page.go_to + + expect do + another = browser.create_page + another.tracing.record(path: file_path2) { another.go_to } + end.to raise_exception(Ferrum::BrowserError, "Tracing has already been started (possibly in another tab).") + expect(File.exist?(file_path2)).to be(false) + end + + expect(File.exist?(file_path)).to be(true) + end + + it "handles tracing complete event once" do + expect(page.tracing).to receive(:stream_handle).exactly(3).times.and_call_original + + page.tracing.record(path: file_path) { page.go_to } + expect(File.exist?(file_path)).to be(true) + + page.tracing.record(path: file_path2) { page.go_to } + expect(File.exist?(file_path2)).to be(true) + + page.tracing.record(path: file_path3) { page.go_to } + expect(File.exist?(file_path3)).to be(true) + ensure + FileUtils.rm_f(file_path) + FileUtils.rm_f(file_path2) + FileUtils.rm_f(file_path3) + end + + it "returns base64 encoded string" do + trace = page.tracing.record(encoding: :base64) { page.go_to } + + decoded = Base64.decode64(trace) + content = JSON.parse(decoded) + expect(content["traceEvents"].any?).to eq(true) + end + + it "returns buffer with no encoding" do + trace = page.tracing.record { page.go_to } + + content = JSON.parse(trace) + expect(content["traceEvents"].any?).to eq(true) + end + + context "screenshots enabled" do + it "fills file with screenshot data" do + page.tracing.record(path: file_path, screenshots: true) { page.go_to("/ferrum/grid") } + + expect(File.exist?(file_path)).to be(true) + expect(trace_config["included_categories"]).to include("disabled-by-default-devtools.screenshot") + expect(content["traceEvents"].any? { |o| o["name"] == "Screenshot" }).to eq(true) + ensure + FileUtils.rm_f(file_path) + end + + it "returns a buffer with screenshot data" do + trace = page.tracing.record(screenshots: true) { page.go_to("/ferrum/grid") } + + expect(File.exist?(file_path)).to be(false) + content = JSON.parse(trace) + trace_config = JSON.parse(content["metadata"]["trace-config"]) + expect(trace_config["included_categories"]).to include("disabled-by-default-devtools.screenshot") + expect(content["traceEvents"].any? { |o| o["name"] == "Screenshot" }).to eq(true) + end + end + + it "waits for promise fill with timeout when it provided" do + expect(page.tracing).to receive(:subscribe_tracing_complete).with(no_args) + trace = page.tracing.record(timeout: 1) { page.go_to } + expect(trace).to be_nil + end + end +end