diff --git a/CHANGELOG.md b/CHANGELOG.md
index 22c5b281..a68e3303 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,8 +17,15 @@
`browser.process.output`
- `browser.add_extension` adds an extension for pages and iframes attached afterwards [#663]
- `page.isolated_world(name)` and `frame.isolated_world(name)` evaluate JavaScript in a world hidden from the page
+- `Node#backend_node_id` and `Node#node_type`
### Changed
+- `evaluate` fetches the whole result in one CDP call instead of one per nested object, dates come back as ISO 8601
+ strings and a cyclic result is a `Hash`/`Array` with `CyclicObject` only where it refers back to itself, instead of
+ being replaced by `CyclicObject` entirely
+- Nodes returned by `evaluate`, `css` and the other finders no longer cost a `DOM.describeNode` call each,
+ `Node#description` is fetched on first use. `Node.new` takes the node as Chrome deep serializes it instead of
+ a target id and a description
- The logger cuts any string value over 1024 characters in CDP messages, not just screenshots.
`FERRUM_LOG_VALUE_LENGTH` sets the length, `0` keeps them whole. `FERRUM_LOGGING_SCREENSHOTS` is gone
- `ProcessTimeoutError` is raised as soon as Chrome exits instead of after `:process_timeout`
@@ -43,6 +50,8 @@
### Fixed
- `evaluate`, `execute` and friends raise `JavaScriptError` when JS throws something other than an `Error`, e.g.
`throw "smth"`, instead of returning it. Returning an `Error` object no longer crashes with `NoMethodError`
+- `evaluate` keeps `null` and `undefined` in arrays and returns `NaN`, `Infinity`, `-0` and `BigInt` instead of `nil`,
+ and no longer crashes on objects with getters
- `workers` skips detached workers, `target.worker` with no session raises `NoSuchTargetError` instead of a CDP error
- `attach_target` reuses a target's session and returns the target, instead of opening one more that reruns extensions
- Service workers registered by a page stayed attached and alive for the browser's lifetime, now Chrome can stop them
diff --git a/docs/13-javascript.md b/docs/13-javascript.md
index 362d095b..1c8d028f 100644
--- a/docs/13-javascript.md
+++ b/docs/13-javascript.md
@@ -16,6 +16,11 @@ Evaluate and return result for given JS expression
page.evaluate("[window.scrollX, window.scrollY]")
```
+The result is converted to Ruby: objects become `Hash`, arrays and node lists `Array`, DOM nodes `Ferrum::Node`,
+dates ISO 8601 strings, `NaN` and `Infinity` `Float`, `BigInt` `Integer`, `null` and `undefined` `nil`. Values
+without a Ruby counterpart, such as functions, `Map`, `Set` or `window`, become `{}`. A reference back to an
+enclosing object, e.g. `a.self = a`, becomes `Ferrum::CyclicObject`.
+
#### evaluate_async(expression, wait_time, \*args)
Evaluate asynchronous expression and return result
diff --git a/lib/ferrum/frame/runtime.rb b/lib/ferrum/frame/runtime.rb
index 993a92f2..b2cce4df 100644
--- a/lib/ferrum/frame/runtime.rb
+++ b/lib/ferrum/frame/runtime.rb
@@ -4,10 +4,9 @@
module Ferrum
#
- # Placeholder object substituted for a JavaScript value that couldn't be
- # fully serialized because it contains circular references. It exists
- # only so that {#inspect} can report the situation instead of the
- # evaluation raising or hanging.
+ # Placeholder substituted for a reference back to a JavaScript object that
+ # contains it, e.g. `a.self = a`, so that the result stays a finite Ruby
+ # Hash or Array.
#
class CyclicObject
include Singleton
@@ -24,12 +23,23 @@ class Frame
#
# Evaluates and executes JavaScript in a frame's execution context via
# `Runtime.callFunctionOn`, converting arguments and return values
- # between Ruby and JS, and resolving object/array/node results
- # (including cyclic ones, via {CyclicObject}) into Ruby equivalents.
+ # between Ruby and JS, and resolving object/array/node results into Ruby
+ # equivalents. A reference back to an enclosing object becomes
+ # {CyclicObject}.
#
module Runtime
INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
+ DEEP_SERIALIZATION = {
+ serialization: "deep",
+ additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" }
+ }.freeze
+ UNSERIALIZABLE_NUMBERS = {
+ "NaN" => Float::NAN,
+ "-0" => -0.0,
+ "Infinity" => Float::INFINITY,
+ "-Infinity" => -Float::INFINITY
+ }.freeze
#
# Evaluate and return result for given JS expression.
@@ -153,6 +163,7 @@ def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
params = options.dup
params = params.merge(objectId: resolve_node(on)) if on
+ params = params.merge(serializationOptions: DEEP_SERIALIZATION) if handle
if params[:executionContextId].nil? && params[:objectId].nil?
params = params.merge(executionContextId: execution_id!)
@@ -182,38 +193,37 @@ def handle_error(response)
end
end
- def handle_response(response, check_cyclic: true)
- case response["type"]
- when "boolean", "number", "string"
- response["value"]
- when "undefined"
- nil
- when "function"
- {}
- when "object"
- object_id = response["objectId"]
+ def handle_response(response)
+ deserialize(response["deepSerializedValue"])
+ end
- case response["subtype"]
- when "node"
- description = @page.command("DOM.describeNode", objectId: object_id)["node"]
- Node.new(self, @page.target_id, description, object_id: object_id)
- when "array"
- reduce_props(object_id, [], check_cyclic: check_cyclic) do |memo, key, value|
- next(memo) unless Integer(key, exception: false)
+ def deserialize(serialized, references = {}, path = [])
+ reference = serialized["weakLocalObjectReference"]
+ if reference
+ return cyclic_object if path.include?(reference)
- value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
- memo.insert(key.to_i, value)
- end.compact
- when "date"
- response["description"]
- when "null"
- nil
- else
- reduce_props(object_id, {}, check_cyclic: check_cyclic) do |memo, key, value|
- value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
- memo.merge(key => value)
- end
- end
+ serialized = references[reference] ||= serialized
+ path += [reference]
+ end
+
+ value = serialized["value"]
+ case serialized["type"]
+ when "undefined", "null", "symbol"
+ nil
+ when "boolean", "string", "date"
+ value
+ when "number"
+ UNSERIALIZABLE_NUMBERS.fetch(value, value)
+ when "bigint"
+ Integer(value)
+ when "array", "nodelist", "htmlcollection"
+ value.map { |v| deserialize(v, references, path) }
+ when "object"
+ value.to_h { |k, v| [k, deserialize(v, references, path)] }
+ when "node"
+ Node.new(self, value)
+ else
+ {}
end
end
@@ -237,52 +247,6 @@ def prepare_args(args)
end
end
- def reduce_props(object_id, to, check_cyclic: true)
- if check_cyclic && cyclic?(object_id).dig("result", "value")
- to.is_a?(Array) ? [cyclic_object] : cyclic_object
- else
- props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)
- props["result"].reduce(to) do |memo, prop|
- next(memo) unless prop["enumerable"]
-
- yield(memo, prop["name"], prop["value"])
- end
- end
- end
-
- def cyclic?(object_id)
- @page.command(
- "Runtime.callFunctionOn",
- objectId: object_id,
- returnByValue: true,
- functionDeclaration: <<~JS
- function() {
- if (Array.isArray(this) &&
- this.every(e => e instanceof Node)) {
- return false;
- }
-
- function detectCycle(obj, seen) {
- if (typeof obj === "object") {
- if (seen.indexOf(obj) !== -1) {
- return true;
- }
- for (let key in obj) {
- if (obj.hasOwnProperty(key) && detectCycle(obj[key], seen.concat([obj]))) {
- return true;
- }
- }
- }
-
- return false;
- }
-
- return detectCycle(this, []);
- }
- JS
- )
- end
-
def cyclic_object
CyclicObject.instance
end
diff --git a/lib/ferrum/node.rb b/lib/ferrum/node.rb
index f8ca9b45..755d5561 100644
--- a/lib/ferrum/node.rb
+++ b/lib/ferrum/node.rb
@@ -14,18 +14,58 @@ class Node
MOVING_WAIT_DELAY = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
MOVING_WAIT_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
- attr_reader :page, :target_id, :description, :tag_name
+ # @return [Page] the page the node belongs to.
+ attr_reader :page
- def initialize(frame, target_id, description, object_id: nil, node_id: nil)
+ # @return [String] the id of the page's target.
+ attr_reader :target_id
+
+ # @return [Integer] Chrome's id of the node, stable for the node's lifetime.
+ attr_reader :backend_node_id
+
+ # @return [Integer] the DOM `nodeType`, e.g. `1` for an element, `3` for a text node.
+ attr_reader :node_type
+
+ # @return [String, nil] the id of the frame an `iframe` or `frame` element contains, `nil` for other nodes.
+ attr_reader :frame_id
+
+ #
+ # @param [Frame, IsolatedWorld] frame
+ # The runtime the node was returned from.
+ #
+ # @param [Hash{String => Object}] properties
+ # The node's properties from Chrome's deep serialization.
+ #
+ def initialize(frame, properties)
@page = frame.page
- @target_id = target_id
- @description = description
- @tag_name = description["nodeName"].downcase
- @object_id = object_id
- @node_id = node_id
+ @target_id = @page.target_id
+ @backend_node_id = properties["backendNodeId"]
+ @node_type = properties["nodeType"]
+ @tag_name = properties["localName"]&.downcase
+ @frame_id = properties["frameId"]
@document_version = @page.document_version
end
+ #
+ # Chrome's description of the node from `DOM.describeNode`, fetched on
+ # first use.
+ #
+ # @return [Hash{String => Object}]
+ #
+ def description
+ @description ||= page.command("DOM.describeNode", backendNodeId: backend_node_id)["node"]
+ end
+
+ #
+ # The lowercased tag name of an element, or the node name of any other
+ # node, e.g. `"#text"`.
+ #
+ # @return [String]
+ #
+ def tag_name
+ @tag_name ||= description["nodeName"].downcase
+ end
+
# Frontend node id is resolved lazily, on first actual need (focus, click, scroll_into_view, etc.)
# We can try to subscribe to `DOM.childNodeRemoved` and `DOM.childNodeInserted` in the future
# to keep track of nodes.
@@ -34,14 +74,15 @@ def node_id
document_version = page.document_version
@node_id = begin
- id = page.command("DOM.requestNode", objectId: @object_id)["nodeId"]
- if id.zero?
- raise NodeNotFoundError, { "message" => "node is not trackable, the page is still loading a new document" }
- end
+ id = page.command("DOM.pushNodesByBackendIdsToFrontend",
+ backendNodeIds: [backend_node_id])["nodeIds"].first
+ raise NodeNotFoundError, { "message" => "node is not trackable" } if id.zero?
id
- rescue NoExecutionContextError
- raise NodeNotFoundError, { "message" => "node is not trackable" }
+ rescue BrowserError => e
+ raise unless e.message == "Document needs to be requested first"
+
+ raise NodeNotFoundError, { "message" => "node is not trackable, the page is still loading a new document" }
end
@document_version = document_version
@node_id
@@ -51,16 +92,7 @@ def node_id
#
# @return [Boolean]
def node?
- description["nodeType"] == 1 # nodeType: 3, nodeName: "#text" e.g.
- end
-
- #
- # The id of the frame this node belongs to.
- #
- # @return [String]
- #
- def frame_id
- description["frameId"]
+ node_type == 1 # nodeType: 3, nodeName: "#text" e.g.
end
#
@@ -240,7 +272,7 @@ def select_file(value)
page.command(
"DOM.setFileInputFiles",
slowmoable: true,
- backendNodeId: description["backendNodeId"],
+ backendNodeId: backend_node_id,
files: Array(value)
)
end
@@ -453,7 +485,7 @@ def ==(other)
# We compare backendNodeId because once nodeId is sent to frontend backend
# never returns same nodeId sending 0. In other words frontend is
# responsible for keeping track of node ids.
- target_id == other.target_id && description["backendNodeId"] == other.description["backendNodeId"]
+ target_id == other.target_id && backend_node_id == other.backend_node_id
end
#
@@ -462,7 +494,8 @@ def ==(other)
# @return [String]
#
def inspect
- %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @description=#{@description.inspect}>)
+ %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} ) +
+ %(@backend_node_id=#{@backend_node_id} @tag_name=#{@tag_name.inspect}>)
end
#
diff --git a/sig/ferrum/frame/runtime.rbs b/sig/ferrum/frame/runtime.rbs
index 5c93ba6c..21766bfd 100644
--- a/sig/ferrum/frame/runtime.rbs
+++ b/sig/ferrum/frame/runtime.rbs
@@ -11,6 +11,10 @@ module Ferrum
INTERMITTENT_SLEEP: untyped
+ DEEP_SERIALIZATION: ::Hash[Symbol, untyped]
+
+ UNSERIALIZABLE_NUMBERS: ::Hash[String, Float]
+
def evaluate: (untyped expression, *untyped args) -> untyped
def evaluate_async: (untyped expression, untyped wait, *untyped args) -> untyped
@@ -27,7 +31,9 @@ module Ferrum
def handle_error: (untyped response) -> (nil | untyped)
- def handle_response: (untyped response) -> untyped
+ def handle_response: (Hash[String, untyped] response) -> untyped
+
+ def deserialize: (Hash[String, untyped] serialized, ?Hash[Integer, Hash[String, untyped]] references, ?Array[Integer] path) -> untyped
def resolve_node: (Node node) -> String
@@ -35,11 +41,7 @@ module Ferrum
def prepare_args: (untyped args) -> untyped
- def reduce_props: (untyped object_id, untyped to) { (untyped, untyped, untyped) -> untyped } -> (::Array[untyped] | untyped | untyped)
-
- def cyclic?: (untyped object_id) -> untyped
-
- def cyclic_object: () -> untyped
+ def cyclic_object: () -> CyclicObject
end
end
end
diff --git a/sig/ferrum/node.rbs b/sig/ferrum/node.rbs
index a8e7cd17..4b18abcd 100644
--- a/sig/ferrum/node.rbs
+++ b/sig/ferrum/node.rbs
@@ -5,22 +5,31 @@ module Ferrum
MOVING_WAIT_ATTEMPTS: untyped
@node_id: untyped
- @object_id: untyped
@document_version: ::Integer
attr_reader page: untyped
attr_reader target_id: untyped
- attr_reader description: untyped
+ @backend_node_id: Integer
+ @node_type: Integer
+ @tag_name: String?
+ @frame_id: String?
+ @description: Hash[String, untyped]?
- attr_reader tag_name: untyped
+ attr_reader backend_node_id: Integer
- def initialize: (untyped frame, untyped target_id, untyped node_id, untyped description) -> void
+ attr_reader node_type: Integer
+
+ def initialize: (untyped frame, Hash[String, untyped] properties) -> void
+
+ def description: () -> Hash[String, untyped]
+
+ def tag_name: () -> String
def node?: () -> untyped
- def frame_id: () -> untyped
+ attr_reader frame_id: String?
def frame: () -> untyped
diff --git a/spec/browser_spec.rb b/spec/browser_spec.rb
index 8f4c046a..28b24fe6 100644
--- a/spec/browser_spec.rb
+++ b/spec/browser_spec.rb
@@ -238,7 +238,7 @@ def broken.puts(*) = raise(IOError, "closed stream")
browser.go_to(base_url("/deeply_nested"))
expect(browser.evaluate(%(document.getElementById("text"))).text).to eq("text")
- expect(logger.string).to include("DOM.describeNode")
+ expect(logger.string).to include("DOM.setChildNodes")
ensure
browser&.quit
end
diff --git a/spec/frame/runtime_spec.rb b/spec/frame/runtime_spec.rb
index 4b41718d..d9b90499 100644
--- a/spec/frame/runtime_spec.rb
+++ b/spec/frame/runtime_spec.rb
@@ -110,6 +110,36 @@
)
end
+ it "returns nodes of every type" do
+ browser.content = <<~HTML
+
+ text
+
+
+
+
+ HTML
+ nodes = browser.evaluate(<<~JS)
+ (() => {
+ const host = document.getElementById("host");
+ host.attachShadow({ mode: "open" });
+ const p = document.querySelector("p");
+ return [document, document.doctype, p, p.firstChild, p.lastChild, document.querySelector("iframe"),
+ document.querySelector("my-element"), document.querySelector("circle"), host.shadowRoot];
+ })()
+ JS
+
+ expect(nodes.map(&:tag_name)).to eq(
+ %w[#document html p #text #comment iframe my-element circle #document-fragment]
+ )
+ expect(nodes.map(&:node_type)).to eq([9, 10, 1, 3, 8, 1, 1, 1, 11])
+ expect(nodes.map(&:node?)).to eq([false, false, true, false, false, true, true, true, false])
+ expect(nodes[5].frame_id).to eq(browser.frames.reject(&:main?).first.id)
+ expect(nodes.values_at(0..4, 6..8).map(&:frame_id)).to all(be_nil)
+ expect(nodes[2].description).to include("nodeName" => "P", "attributes" => %w[class a id b])
+ expect(nodes[8].description).to include("shadowRootType" => "open")
+ end
+
it "returns values properly" do
expect(browser.evaluate("null")).to be_nil
expect(browser.evaluate("false")).to be false
@@ -148,7 +178,33 @@
expect(browser.evaluate(%(new Error("smth")))).to eq({})
end
+ it "keeps null and undefined in arrays" do
+ expect(browser.evaluate("[1, null, undefined, 3]")).to eq([1, nil, nil, 3])
+ end
+
+ it "returns numbers JSON can't represent" do
+ expect(browser.evaluate("NaN")).to be_nan
+ expect(browser.evaluate("1 / 0")).to eq(Float::INFINITY)
+ expect(browser.evaluate("-1 / 0")).to eq(-Float::INFINITY)
+ expect(browser.evaluate("-0").to_s).to eq("-0.0")
+ expect(browser.evaluate("{ big: 2n ** 64n }")).to eq("big" => 2**64)
+ end
+
+ it "returns dates as ISO strings" do
+ expect(browser.evaluate("[new Date(Date.UTC(2012, 0))]")).to eq(["2012-01-01T00:00:00.000Z"])
+ end
+
+ it "returns values of getters" do
+ expect(browser.evaluate("{ get a() { return 1 } }")).to eq("a" => 1)
+ end
+
+ it "returns objects without a Ruby equivalent as empty hashes" do
+ expect(browser.evaluate("[new Map([[1, 2]]), new Set([1]), /a/, Symbol('a')]")).to eq([{}, {}, {}, nil])
+ end
+
context "when cyclic structure is returned" do
+ let(:cyclic) { Ferrum::CyclicObject.instance }
+
context "ignores seen" do
let(:code) do
<<~JS
@@ -166,11 +222,11 @@
end
it "returns object" do
- expect(browser.evaluate(code % "a")).to eq(Ferrum::CyclicObject.instance)
+ expect(browser.evaluate(code % "a")).to eq("a" => cyclic, "b" => {}, "c" => { "a" => cyclic })
end
it "returns array" do
- expect(browser.evaluate(code % "[a]")).to eq([Ferrum::CyclicObject.instance])
+ expect(browser.evaluate(code % "[a]")).to eq([{ "a" => cyclic, "b" => {}, "c" => { "a" => cyclic } }])
end
end
@@ -178,21 +234,23 @@
expect(browser.evaluate("(function() { var a = {}; return [a, a] })()")).to eq([{}, {}])
end
- it "only checks for a cycle once for the whole result tree, not once per nested object" do
- frame = browser.main_frame
- original_cyclic = frame.method(:cyclic?)
- calls = 0
- allow(frame).to receive(:cyclic?) do |*args|
- calls += 1
- original_cyclic.call(*args)
- end
+ it "returns an array containing itself" do
+ expect(browser.evaluate("(function() { var a = [1]; a.push(a); return a })()")).to eq([1, cyclic])
+ end
+
+ it "returns window as an empty hash" do
+ expect(browser.evaluate("window")).to eq({})
+ end
+
+ it "fetches the whole result tree with one command" do
+ allow(browser.page).to receive(:command).and_call_original
result = browser.evaluate(<<~JS)
Array.from({ length: 50 }, (_, i) => ({ i: i, nested: { j: i * 2 } }))
JS
expect(result.size).to eq(50)
- expect(calls).to eq(1)
+ expect(browser.page).to have_received(:command).once
end
end
end