From f87f8f5de59b6dc7677aab9c0b843c95014cf0fe Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Tue, 27 Jun 2023 10:08:03 +0100 Subject: [PATCH 1/8] test: :white_check_mark: add a step for optional attribute values --- test/browser/features/resource-load-spans.feature | 14 ++++++++++++-- test/browser/features/steps/browser-steps.rb | 11 +++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/browser/features/resource-load-spans.feature b/test/browser/features/resource-load-spans.feature index 05be4a909..e47ccba0f 100644 --- a/test/browser/features/resource-load-spans.feature +++ b/test/browser/features/resource-load-spans.feature @@ -12,19 +12,24 @@ Feature: Resource Load Spans Then the trace payload field "resourceSpans.0.scopeSpans.0.spans.2.name" equals "[Custom]/resource-load-spans" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2.spanId" is stored as the value "parent_span_id" - # App bundle + # App bundle resource load And the trace payload field "resourceSpans.0.scopeSpans.0.spans.0.name" matches the regex "^\[ResourceLoad\]http:\/\/.*:[0-9]{4}\/resource-load-spans\/dist\/bundle\.js$" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.0.parentSpanId" equals the stored value "parent_span_id" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.0" string attribute "bugsnag.span.category" equals "resource_load" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.0" string attribute "http.flavor" equals "1.1" - # Image + # Image resource load And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1.name" matches the regex "^\[ResourceLoad\]http:\/\/.*:[0-9]{4}\/favicon.png$" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1.parentSpanId" equals the stored value "parent_span_id" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" string attribute "bugsnag.span.category" equals "resource_load" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" string attribute "http.url" matches the regex "^http:\/\/.*:[0-9]{4}\/favicon\.png\?height=100&width=100$" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" string attribute "http.flavor" equals "1.1" + # Image status code and body size have very patchy browser coverage + And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 + And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 + And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + @skip_on_npm_build @requires_resource_load_spans Scenario: Resource load spans are automatically instrumented (CDN build) @@ -55,3 +60,8 @@ Feature: Resource Load Spans And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2" string attribute "bugsnag.span.category" equals "resource_load" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2" string attribute "http.url" matches the regex "^http:\/\/.*:[0-9]{4}\/favicon\.png\?height=100&width=100$" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2" string attribute "http.flavor" equals "1.1" + + # Image status code and body size have very patchy browser coverage + And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 + And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 + And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 diff --git a/test/browser/features/steps/browser-steps.rb b/test/browser/features/steps/browser-steps.rb index ae8a254be..e271f1b85 100644 --- a/test/browser/features/steps/browser-steps.rb +++ b/test/browser/features/steps/browser-steps.rb @@ -207,6 +207,17 @@ end end +def check_attribute_equal_if_present(field, attribute, attr_type, expected) + actual = get_attribute_value field, attribute, attr_type + if actual != nil + Maze.check.equal(expected, actual) + end +end + +Then('if present, the trace payload field {string} integer attribute {string} equals {int}') do |field, attribute, expected| + check_attribute_equal_if_present field, attribute, 'intValue', expected +end + module Maze module Driver class Browser From aaba0a97e4b8d11b0bb4fe75521ce5213f66cd67 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Tue, 27 Jun 2023 17:08:19 +0100 Subject: [PATCH 2/8] test: :recycle: add ability to check any step for browser version before asserting on it --- test/browser/features/lib/browser.rb | 22 +++++++++++++++++++ .../features/resource-load-spans.feature | 9 ++++---- test/browser/features/steps/browser-steps.rb | 13 +++++++++-- .../support/requires-performance-body-size.rb | 4 ---- .../requires-performance-response-status.rb | 4 ---- 5 files changed, 38 insertions(+), 14 deletions(-) delete mode 100644 test/browser/features/support/requires-performance-body-size.rb delete mode 100644 test/browser/features/support/requires-performance-response-status.rb diff --git a/test/browser/features/lib/browser.rb b/test/browser/features/lib/browser.rb index 929d871c3..3b0fd4a4c 100644 --- a/test/browser/features/lib/browser.rb +++ b/test/browser/features/lib/browser.rb @@ -15,6 +15,14 @@ def initialize(browser_spec) end end + def name + @name + end + + def version + @version + end + # is this a mobile device? # we assume that android devices are always using the latest version of chrome def mobile? @@ -63,6 +71,20 @@ def supported_web_vitals end end + # | browser | version | + # | chrome | 54 | + # | firefox | 45 | + # | edge | 17 | + # | safari | 16.4 | + def supports_performance_encoded_body_size? + case @name + when "safari" + @version >= 17 # we test on 16.3 - not sure what to do here? + else + true + end + end + def supports_performance_response_status? case @name when "chrome", "edge" diff --git a/test/browser/features/resource-load-spans.feature b/test/browser/features/resource-load-spans.feature index e47ccba0f..7665d7ccb 100644 --- a/test/browser/features/resource-load-spans.feature +++ b/test/browser/features/resource-load-spans.feature @@ -61,7 +61,8 @@ Feature: Resource Load Spans And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2" string attribute "http.url" matches the regex "^http:\/\/.*:[0-9]{4}\/favicon\.png\?height=100&width=100$" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2" string attribute "http.flavor" equals "1.1" - # Image status code and body size have very patchy browser coverage - And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 - And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 - And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + # Image status code and body size have patchy browser coverage + And on the browser Chrome 109: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 + + And on the browsers Chrome 73, Safari 13, Firefox 67: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 + And on the browsers Chrome 73, Safari 13, Firefox 67: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 diff --git a/test/browser/features/steps/browser-steps.rb b/test/browser/features/steps/browser-steps.rb index e271f1b85..2a797cddc 100644 --- a/test/browser/features/steps/browser-steps.rb +++ b/test/browser/features/steps/browser-steps.rb @@ -214,8 +214,17 @@ def check_attribute_equal_if_present(field, attribute, attr_type, expected) end end -Then('if present, the trace payload field {string} integer attribute {string} equals {int}') do |field, attribute, expected| - check_attribute_equal_if_present field, attribute, 'intValue', expected +Then(/^on the (?:browser|browsers) (.*): (.+)/) do |spec, step_text| + current_platform = $browser.name + current_version = $browser.version + + browsers = spec.split(",") + + for browser in browsers do + name, version = browser.split(" ") + step(step_text) if current_platform.casecmp(name).zero? && current_version >= version.to_i + end + end module Maze diff --git a/test/browser/features/support/requires-performance-body-size.rb b/test/browser/features/support/requires-performance-body-size.rb deleted file mode 100644 index 92b44ed7d..000000000 --- a/test/browser/features/support/requires-performance-body-size.rb +++ /dev/null @@ -1,4 +0,0 @@ -Before("@requires_performance_body_size") do - skip_this_scenario unless $browser.supports_performance_body_size? -end - \ No newline at end of file diff --git a/test/browser/features/support/requires-performance-response-status.rb b/test/browser/features/support/requires-performance-response-status.rb deleted file mode 100644 index ea9787365..000000000 --- a/test/browser/features/support/requires-performance-response-status.rb +++ /dev/null @@ -1,4 +0,0 @@ -Before("@requires_performance_response_status") do - skip_this_scenario unless $browser.supports_performance_response_status? -end - \ No newline at end of file From 61545497b6e8345e5c6d0a000a5d51495aa1a706 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Wed, 28 Jun 2023 09:39:21 +0100 Subject: [PATCH 3/8] test: :white_check_mark: update expected browser versions --- test/browser/features/resource-load-spans.feature | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/browser/features/resource-load-spans.feature b/test/browser/features/resource-load-spans.feature index 7665d7ccb..4978e9e2d 100644 --- a/test/browser/features/resource-load-spans.feature +++ b/test/browser/features/resource-load-spans.feature @@ -64,5 +64,6 @@ Feature: Resource Load Spans # Image status code and body size have patchy browser coverage And on the browser Chrome 109: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 - And on the browsers Chrome 73, Safari 13, Firefox 67: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 - And on the browsers Chrome 73, Safari 13, Firefox 67: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + # Actually Safari 16.4 but our test devices currently use 16.3, this can be dropped to 16 when the devices update + And on the browsers Chrome 54, Android 54, Safari 17, iOS 17, Firefox 45, Edge 17: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 + And on the browsers Chrome 54, Android 54, Safari 17, iOS 17, Firefox 45, Edge 17: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 From 273d50b4c9554f1f75e88abb7b9f9846c82c13fa Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 29 Jun 2023 14:15:20 +0100 Subject: [PATCH 4/8] remove unused definition --- test/browser/features/lib/browser.rb | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/test/browser/features/lib/browser.rb b/test/browser/features/lib/browser.rb index 3b0fd4a4c..aa71d1031 100644 --- a/test/browser/features/lib/browser.rb +++ b/test/browser/features/lib/browser.rb @@ -15,13 +15,7 @@ def initialize(browser_spec) end end - def name - @name - end - - def version - @version - end + attr_reader :name, :version # is this a mobile device? # we assume that android devices are always using the latest version of chrome @@ -71,20 +65,6 @@ def supported_web_vitals end end - # | browser | version | - # | chrome | 54 | - # | firefox | 45 | - # | edge | 17 | - # | safari | 16.4 | - def supports_performance_encoded_body_size? - case @name - when "safari" - @version >= 17 # we test on 16.3 - not sure what to do here? - else - true - end - end - def supports_performance_response_status? case @name when "chrome", "edge" From 7180f48d5de1addd2443f791e2b584ea3f3122c2 Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Mon, 10 Jul 2023 15:59:23 +0100 Subject: [PATCH 5/8] test: :white_check_mark: rewrite scoped test step --- .../features/resource-load-spans.feature | 12 ++++++-- test/browser/features/steps/browser-steps.rb | 30 +++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/test/browser/features/resource-load-spans.feature b/test/browser/features/resource-load-spans.feature index 4978e9e2d..d3c5ae67a 100644 --- a/test/browser/features/resource-load-spans.feature +++ b/test/browser/features/resource-load-spans.feature @@ -62,8 +62,14 @@ Feature: Resource Load Spans And the trace payload field "resourceSpans.0.scopeSpans.0.spans.2" string attribute "http.flavor" equals "1.1" # Image status code and body size have patchy browser coverage - And on the browser Chrome 109: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 + And on Chrome versions >= 109: + """ + the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 + """ # Actually Safari 16.4 but our test devices currently use 16.3, this can be dropped to 16 when the devices update - And on the browsers Chrome 54, Android 54, Safari 17, iOS 17, Firefox 45, Edge 17: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 - And on the browsers Chrome 54, Android 54, Safari 17, iOS 17, Firefox 45, Edge 17: the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + And on Chrome versions >= 54, Android versions >= 54, Safari versions >= 17, iOS versions >= 17, Firefox versions >= 45, Edge versions >= 17: + """ + the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 + the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + """ diff --git a/test/browser/features/steps/browser-steps.rb b/test/browser/features/steps/browser-steps.rb index 2a797cddc..b040811f6 100644 --- a/test/browser/features/steps/browser-steps.rb +++ b/test/browser/features/steps/browser-steps.rb @@ -214,17 +214,29 @@ def check_attribute_equal_if_present(field, attribute, attr_type, expected) end end -Then(/^on the (?:browser|browsers) (.*): (.+)/) do |spec, step_text| - current_platform = $browser.name - current_version = $browser.version - - browsers = spec.split(",") +Then(/^on ((?:[A-z]+ versions (?:>=?|<=?|==) [0-9.]+(?:, )?)+):$/) do |browser_specs, steps_to_run| + spec_matcher = /^([A-z]+) versions (>=?|<=?|==) ([0-9.]+)$/ - for browser in browsers do - name, version = browser.split(" ") - step(step_text) if current_platform.casecmp(name).zero? && current_version >= version.to_i - end + browser_specs.split(", ").each do |browser_spec| + browser_spec.scan(spec_matcher) do |name, operator, version| + should_run_steps = $browser.name.casecmp?(name) && $browser.version.send(operator, version.to_i) + + # make sure this step is debuggable! + $logger.debug("#{$browser.name} == #{name} && v#{$browser.version} #{operator} #{version}? #{should_run_steps}") + + if should_run_steps + steps_to_run.each_line(chomp: true) do |step_to_run| + step(step_to_run) + end + else + indent = " " * 4 + # e.g. "a step\nanother step\n" -> " 1) a step\n 2) another step" + steps_indented = steps_to_run.each_line.map.with_index(1) { |step, i| "#{indent}#{i}) #{step.chomp}" }.join("\n") + $logger.info("Skipping steps on #{$browser.name} v#{$browser.version}:\n#{steps_indented}") + end + end + end end module Maze From 81d2cc022b4651877d9bb6074a841d646e46952a Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Thu, 13 Jul 2023 11:18:37 +0100 Subject: [PATCH 6/8] update assertions with patchy coverage --- .../browser/features/resource-load-spans.feature | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/browser/features/resource-load-spans.feature b/test/browser/features/resource-load-spans.feature index d3c5ae67a..247376d65 100644 --- a/test/browser/features/resource-load-spans.feature +++ b/test/browser/features/resource-load-spans.feature @@ -25,10 +25,18 @@ Feature: Resource Load Spans And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" string attribute "http.url" matches the regex "^http:\/\/.*:[0-9]{4}\/favicon\.png\?height=100&width=100$" And the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" string attribute "http.flavor" equals "1.1" - # Image status code and body size have very patchy browser coverage - And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 - And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 - And if present, the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + # Image status code and body size have patchy browser coverage + And on Chrome versions >= 109: + """ + the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.status_code" equals 200 + """ + + # Actually Safari 16.4 but our test devices currently use 16.3, this can be dropped to 16 when the devices update + And on Chrome versions >= 54, Android versions >= 54, Safari versions >= 17, iOS versions >= 17, Firefox versions >= 45, Edge versions >= 17: + """ + the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length" equals 2202 + the trace payload field "resourceSpans.0.scopeSpans.0.spans.1" integer attribute "http.response_content_length_uncompressed" equals 2202 + """ @skip_on_npm_build @requires_resource_load_spans From 585ccbb12383ab33fa2cddf2d61bc9fb24f8325a Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 14 Jul 2023 11:31:48 +0100 Subject: [PATCH 7/8] test: :lock: use more secure regex --- test/browser/features/steps/browser-steps.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/browser/features/steps/browser-steps.rb b/test/browser/features/steps/browser-steps.rb index b040811f6..7fd88aa36 100644 --- a/test/browser/features/steps/browser-steps.rb +++ b/test/browser/features/steps/browser-steps.rb @@ -214,8 +214,8 @@ def check_attribute_equal_if_present(field, attribute, attr_type, expected) end end -Then(/^on ((?:[A-z]+ versions (?:>=?|<=?|==) [0-9.]+(?:, )?)+):$/) do |browser_specs, steps_to_run| - spec_matcher = /^([A-z]+) versions (>=?|<=?|==) ([0-9.]+)$/ +Then(/^on ((?:[A-Za-z]+ versions (?:>=?|<=?|==) [0-9.]+(?:, )?)+):$/) do |browser_specs, steps_to_run| + spec_matcher = /^([A-Za-z]+) versions (>=?|<=?|==) ([0-9.]+)$/ browser_specs.split(", ").each do |browser_spec| browser_spec.scan(spec_matcher) do |name, operator, version| From 6661fb40c3f84fe51377626ae61f8ceac692f87b Mon Sep 17 00:00:00 2001 From: Ben Wilson Date: Fri, 14 Jul 2023 13:41:45 +0100 Subject: [PATCH 8/8] test: :white_check_mark: update should run on step --- test/browser/features/steps/browser-steps.rb | 27 ++++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/test/browser/features/steps/browser-steps.rb b/test/browser/features/steps/browser-steps.rb index 7fd88aa36..a5a192fc7 100644 --- a/test/browser/features/steps/browser-steps.rb +++ b/test/browser/features/steps/browser-steps.rb @@ -216,27 +216,32 @@ def check_attribute_equal_if_present(field, attribute, attr_type, expected) Then(/^on ((?:[A-Za-z]+ versions (?:>=?|<=?|==) [0-9.]+(?:, )?)+):$/) do |browser_specs, steps_to_run| spec_matcher = /^([A-Za-z]+) versions (>=?|<=?|==) ([0-9.]+)$/ + run_steps = false browser_specs.split(", ").each do |browser_spec| browser_spec.scan(spec_matcher) do |name, operator, version| should_run_steps = $browser.name.casecmp?(name) && $browser.version.send(operator, version.to_i) # make sure this step is debuggable! - $logger.debug("#{$browser.name} == #{name} && v#{$browser.version} #{operator} #{version}? #{should_run_steps}") - + $logger.debug("#{$browser.name} == #{name} && v#{$browser.version} #{operator} #{version}? #{should_run_steps}") + if should_run_steps - steps_to_run.each_line(chomp: true) do |step_to_run| - step(step_to_run) - end - else - indent = " " * 4 - # e.g. "a step\nanother step\n" -> " 1) a step\n 2) another step" - steps_indented = steps_to_run.each_line.map.with_index(1) { |step, i| "#{indent}#{i}) #{step.chomp}" }.join("\n") - - $logger.info("Skipping steps on #{$browser.name} v#{$browser.version}:\n#{steps_indented}") + run_steps = true end end end + + if run_steps + steps_to_run.each_line(chomp: true) do |step_to_run| + step(step_to_run) + end + else + indent = " " * 4 + # e.g. "a step\nanother step\n" -> " 1) a step\n 2) another step" + steps_indented = steps_to_run.each_line.map.with_index(1) { |step, i| "#{indent}#{i}) #{step.chomp}" }.join("\n") + + $logger.info("Skipping steps on #{$browser.name} v#{$browser.version}:\n#{steps_indented}") + end end module Maze