feat(runtime): convert evaluate results from Chrome's deep serialization in one call

CHANGELOG.md

@@ -17,8 +17,15 @@
   `browser.process.output`
 - `browser.add_extension` adds an extension for pages and iframes attached afterwards [#663]
 - `page.isolated_world(name)` and `frame.isolated_world(name)` evaluate JavaScript in a world hidden from the page
+- `Node#backend_node_id` and `Node#node_type`
 
 ### Changed
+- `evaluate` fetches the whole result in one CDP call instead of one per nested object, dates come back as ISO 8601
+  strings and a cyclic result is a `Hash`/`Array` with `CyclicObject` only where it refers back to itself, instead of
+  being replaced by `CyclicObject` entirely
+- Nodes returned by `evaluate`, `css` and the other finders no longer cost a `DOM.describeNode` call each,
+  `Node#description` is fetched on first use. `Node.new` takes the node as Chrome deep serializes it instead of
+  a target id and a description
 - The logger cuts any string value over 1024 characters in CDP messages, not just screenshots.
   `FERRUM_LOG_VALUE_LENGTH` sets the length, `0` keeps them whole. `FERRUM_LOGGING_SCREENSHOTS` is gone
 - `ProcessTimeoutError` is raised as soon as Chrome exits instead of after `:process_timeout`
@@ -43,6 +50,8 @@
 ### Fixed
 - `evaluate`, `execute` and friends raise `JavaScriptError` when JS throws something other than an `Error`, e.g.
   `throw "smth"`, instead of returning it. Returning an `Error` object no longer crashes with `NoMethodError`
+- `evaluate` keeps `null` and `undefined` in arrays and returns `NaN`, `Infinity`, `-0` and `BigInt` instead of `nil`,
+  and no longer crashes on objects with getters
 - `workers` skips detached workers, `target.worker` with no session raises `NoSuchTargetError` instead of a CDP error
 - `attach_target` reuses a target's session and returns the target, instead of opening one more that reruns extensions
 - Service workers registered by a page stayed attached and alive for the browser's lifetime, now Chrome can stop them

docs/13-javascript.md

@@ -16,6 +16,11 @@ Evaluate and return result for given JS expression
 page.evaluate("[window.scrollX, window.scrollY]")
 ```
 
+The result is converted to Ruby: objects become `Hash`, arrays and node lists `Array`, DOM nodes `Ferrum::Node`,
+dates ISO 8601 strings, `NaN` and `Infinity` `Float`, `BigInt` `Integer`, `null` and `undefined` `nil`. Values
+without a Ruby counterpart, such as functions, `Map`, `Set` or `window`, become `{}`. A reference back to an
+enclosing object, e.g. `a.self = a`, becomes `Ferrum::CyclicObject`.
+
 #### evaluate_async(expression, wait_time, \*args)
 
 Evaluate asynchronous expression and return result

lib/ferrum/frame/runtime.rb

@@ -4,10 +4,9 @@
 
 module Ferrum
   #
-  # Placeholder object substituted for a JavaScript value that couldn't be
-  # fully serialized because it contains circular references. It exists
-  # only so that {#inspect} can report the situation instead of the
-  # evaluation raising or hanging.
+  # Placeholder substituted for a reference back to a JavaScript object that
+  # contains it, e.g. `a.self = a`, so that the result stays a finite Ruby
+  # Hash or Array.
   #
   class CyclicObject
     include Singleton
@@ -24,12 +23,23 @@ class Frame
     #
     # Evaluates and executes JavaScript in a frame's execution context via
     # `Runtime.callFunctionOn`, converting arguments and return values
-    # between Ruby and JS, and resolving object/array/node results
-    # (including cyclic ones, via {CyclicObject}) into Ruby equivalents.
+    # between Ruby and JS, and resolving object/array/node results into Ruby
+    # equivalents. A reference back to an enclosing object becomes
+    # {CyclicObject}.
     #
     module Runtime
       INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
       INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
+      DEEP_SERIALIZATION = {
+        serialization: "deep",
+        additionalParameters: { maxNodeDepth: 0, includeShadowTree: "none" }
+      }.freeze
+      UNSERIALIZABLE_NUMBERS = {
+        "NaN" => Float::NAN,
+        "-0" => -0.0,
+        "Infinity" => Float::INFINITY,
+        "-Infinity" => -Float::INFINITY
+      }.freeze
 
       #
       # Evaluate and return result for given JS expression.
@@ -153,6 +163,7 @@ def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
           params = options.dup
 
           params = params.merge(objectId: resolve_node(on)) if on
+          params = params.merge(serializationOptions: DEEP_SERIALIZATION) if handle
 
           if params[:executionContextId].nil? && params[:objectId].nil?
             params = params.merge(executionContextId: execution_id!)
@@ -182,38 +193,37 @@ def handle_error(response)
         end
       end
 
-      def handle_response(response, check_cyclic: true)
-        case response["type"]
-        when "boolean", "number", "string"
-          response["value"]
-        when "undefined"
-          nil
-        when "function"
-          {}
-        when "object"
-          object_id = response["objectId"]
+      def handle_response(response)
+        deserialize(response["deepSerializedValue"])
+      end
 
-          case response["subtype"]
-          when "node"
-            description = @page.command("DOM.describeNode", objectId: object_id)["node"]
-            Node.new(self, @page.target_id, description, object_id: object_id)
-          when "array"
-            reduce_props(object_id, [], check_cyclic: check_cyclic) do |memo, key, value|
-              next(memo) unless Integer(key, exception: false)
+      def deserialize(serialized, references = {}, path = [])
+        reference = serialized["weakLocalObjectReference"]
+        if reference
+          return cyclic_object if path.include?(reference)
 
-              value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
-              memo.insert(key.to_i, value)
-            end.compact
-          when "date"
-            response["description"]
-          when "null"
-            nil
-          else
-            reduce_props(object_id, {}, check_cyclic: check_cyclic) do |memo, key, value|
-              value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
-              memo.merge(key => value)
-            end
-          end
+          serialized = references[reference] ||= serialized
+          path += [reference]
+        end
+
+        value = serialized["value"]
+        case serialized["type"]
+        when "undefined", "null", "symbol"
+          nil
+        when "boolean", "string", "date"
+          value
+        when "number"
+          UNSERIALIZABLE_NUMBERS.fetch(value, value)
+        when "bigint"
+          Integer(value)
+        when "array", "nodelist", "htmlcollection"
+          value.map { |v| deserialize(v, references, path) }
+        when "object"
+          value.to_h { |k, v| [k, deserialize(v, references, path)] }
+        when "node"
+          Node.new(self, value)
+        else
+          {}
         end
       end
 
@@ -237,52 +247,6 @@ def prepare_args(args)
         end
       end
 
-      def reduce_props(object_id, to, check_cyclic: true)
-        if check_cyclic && cyclic?(object_id).dig("result", "value")
-          to.is_a?(Array) ? [cyclic_object] : cyclic_object
-        else
-          props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)
-          props["result"].reduce(to) do |memo, prop|
-            next(memo) unless prop["enumerable"]
-
-            yield(memo, prop["name"], prop["value"])
-          end
-        end
-      end
-
-      def cyclic?(object_id)
-        @page.command(
-          "Runtime.callFunctionOn",
-          objectId: object_id,
-          returnByValue: true,
-          functionDeclaration: <<~JS
-            function() {
-              if (Array.isArray(this) &&
-                  this.every(e => e instanceof Node)) {
-                return false;
-              }
-
-              function detectCycle(obj, seen) {
-                if (typeof obj === "object") {
-                  if (seen.indexOf(obj) !== -1) {
-                    return true;
-                  }
-                  for (let key in obj) {
-                    if (obj.hasOwnProperty(key) && detectCycle(obj[key], seen.concat([obj]))) {
-                      return true;
-                    }
-                  }
-                }
-
-                return false;
-              }
-
-              return detectCycle(this, []);
-            }
-          JS
-        )
-      end
-
       def cyclic_object
         CyclicObject.instance
       end

lib/ferrum/node.rb

@@ -14,18 +14,58 @@ class Node
     MOVING_WAIT_DELAY = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
     MOVING_WAIT_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
 
-    attr_reader :page, :target_id, :description, :tag_name
+    # @return [Page] the page the node belongs to.
+    attr_reader :page
 
-    def initialize(frame, target_id, description, object_id: nil, node_id: nil)
+    # @return [String] the id of the page's target.
+    attr_reader :target_id
+
+    # @return [Integer] Chrome's id of the node, stable for the node's lifetime.
+    attr_reader :backend_node_id
+
+    # @return [Integer] the DOM `nodeType`, e.g. `1` for an element, `3` for a text node.
+    attr_reader :node_type
+
+    # @return [String, nil] the id of the frame an `iframe` or `frame` element contains, `nil` for other nodes.
+    attr_reader :frame_id
+
+    #
+    # @param [Frame, IsolatedWorld] frame
+    #   The runtime the node was returned from.
+    #
+    # @param [Hash{String => Object}] properties
+    #   The node's properties from Chrome's deep serialization.
+    #
+    def initialize(frame, properties)
       @page = frame.page
-      @target_id = target_id
-      @description = description
-      @tag_name = description["nodeName"].downcase
-      @object_id = object_id
-      @node_id = node_id
+      @target_id = @page.target_id
+      @backend_node_id = properties["backendNodeId"]
+      @node_type = properties["nodeType"]
+      @tag_name = properties["localName"]&.downcase
+      @frame_id = properties["frameId"]
       @document_version = @page.document_version
     end
 
+    #
+    # Chrome's description of the node from `DOM.describeNode`, fetched on
+    # first use.
+    #
+    # @return [Hash{String => Object}]
+    #
+    def description
+      @description ||= page.command("DOM.describeNode", backendNodeId: backend_node_id)["node"]
+    end
+
+    #
+    # The lowercased tag name of an element, or the node name of any other
+    # node, e.g. `"#text"`.
+    #
+    # @return [String]
+    #
+    def tag_name
+      @tag_name ||= description["nodeName"].downcase
+    end
+
     # Frontend node id is resolved lazily, on first actual need (focus, click, scroll_into_view, etc.)
     # We can try to subscribe to `DOM.childNodeRemoved` and `DOM.childNodeInserted` in the future
     # to keep track of nodes.
@@ -34,14 +74,15 @@ def node_id
 
       document_version = page.document_version
       @node_id = begin
-        id = page.command("DOM.requestNode", objectId: @object_id)["nodeId"]
-        if id.zero?
-          raise NodeNotFoundError, { "message" => "node is not trackable, the page is still loading a new document" }
-        end
+        id = page.command("DOM.pushNodesByBackendIdsToFrontend",
+                          backendNodeIds: [backend_node_id])["nodeIds"].first
+        raise NodeNotFoundError, { "message" => "node is not trackable" } if id.zero?
 
         id
-      rescue NoExecutionContextError
-        raise NodeNotFoundError, { "message" => "node is not trackable" }
+      rescue BrowserError => e
+        raise unless e.message == "Document needs to be requested first"
+
+        raise NodeNotFoundError, { "message" => "node is not trackable, the page is still loading a new document" }
       end
       @document_version = document_version
       @node_id
@@ -51,16 +92,7 @@ def node_id
     #
     # @return [Boolean]
     def node?
-      description["nodeType"] == 1 # nodeType: 3, nodeName: "#text" e.g.
-    end
-
-    #
-    # The id of the frame this node belongs to.
-    #
-    # @return [String]
-    #
-    def frame_id
-      description["frameId"]
+      node_type == 1 # nodeType: 3, nodeName: "#text" e.g.
     end
 
     #
@@ -240,7 +272,7 @@ def select_file(value)
       page.command(
         "DOM.setFileInputFiles",
         slowmoable: true,
-        backendNodeId: description["backendNodeId"],
+        backendNodeId: backend_node_id,
         files: Array(value)
       )
     end
@@ -453,7 +485,7 @@ def ==(other)
       # We compare backendNodeId because once nodeId is sent to frontend backend
       # never returns same nodeId sending 0. In other words frontend is
       # responsible for keeping track of node ids.
-      target_id == other.target_id && description["backendNodeId"] == other.description["backendNodeId"]
+      target_id == other.target_id && backend_node_id == other.backend_node_id
     end
 
     #
@@ -462,7 +494,8 @@ def ==(other)
     # @return [String]
     #
     def inspect
-      %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @description=#{@description.inspect}>)
+      %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} ) +
+        %(@backend_node_id=#{@backend_node_id} @tag_name=#{@tag_name.inspect}>)
     end
 
     #

sig/ferrum/frame/runtime.rbs

@@ -11,6 +11,10 @@ module Ferrum
 
       INTERMITTENT_SLEEP: untyped
 
+      DEEP_SERIALIZATION: ::Hash[Symbol, untyped]
+
+      UNSERIALIZABLE_NUMBERS: ::Hash[String, Float]
+
       def evaluate: (untyped expression, *untyped args) -> untyped
 
       def evaluate_async: (untyped expression, untyped wait, *untyped args) -> untyped
@@ -27,19 +31,17 @@ module Ferrum
 
       def handle_error: (untyped response) -> (nil | untyped)
 
-      def handle_response: (untyped response) -> untyped
+      def handle_response: (Hash[String, untyped] response) -> untyped
+
+      def deserialize: (Hash[String, untyped] serialized, ?Hash[Integer, Hash[String, untyped]] references, ?Array[Integer] path) -> untyped
 
       def resolve_node: (Node node) -> String
 
       def resolve_node_options: () -> ::Hash[Symbol, Integer]
 
       def prepare_args: (untyped args) -> untyped
 
-      def reduce_props: (untyped object_id, untyped to) { (untyped, untyped, untyped) -> untyped } -> (::Array[untyped] | untyped | untyped)
-
-      def cyclic?: (untyped object_id) -> untyped
-
-      def cyclic_object: () -> untyped
+      def cyclic_object: () -> CyclicObject
     end
   end
 end