From d492151d0b5f520eb7f2b9b6505ef05fdb0d28b2 Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Fri, 23 Oct 2026 15:20:29 +1300 Subject: [PATCH 1/2] Stage generated files so an interrupted run leaves them intact The generator wrote template output over the committed structs and ran RuboCop on them in place. Task stops every step when one fails, so a failure elsewhere in `task ci` during formatting left the committed structs unformatted. The generators now write into a staging directory under tmp/, RuboCop formats the files there (with a copy of the config, whose paths are relative to it), and each file moves into place with one rename. The docs tests and typecheck import docs/generated, which isn't committed and was deleted and rewritten on every run, so they now wait for generation. scripts/generate_sorbet_log_structs.rb was an older copy of the generator that nothing ran. --- Taskfile.docs.yml | 6 +- scripts/generate_sorbet_log_structs.rb | 24 -------- scripts/generate_structs.rb | 54 +++++++++++++----- test/workflows/codegen_staging_test.rb | 70 +++++++++++++++++++++++ test/workflows/generation_test.rb | 52 +++++++++++++---- tools/codegen/log_fields_generator.rb | 11 ++-- tools/codegen/sorbet_generator.rb | 45 ++++++++------- tools/codegen/staging.rb | 77 ++++++++++++++++++++++++++ tools/codegen/ts_generator.rb | 41 +++++++------- 9 files changed, 278 insertions(+), 102 deletions(-) delete mode 100644 scripts/generate_sorbet_log_structs.rb create mode 100644 test/workflows/codegen_staging_test.rb create mode 100644 tools/codegen/staging.rb diff --git a/Taskfile.docs.yml b/Taskfile.docs.yml index fee656ff..a2f096bb 100644 --- a/Taskfile.docs.yml +++ b/Taskfile.docs.yml @@ -11,7 +11,8 @@ tasks: typecheck: dir: docs desc: Run TypeScript type checking - deps: [deps] + # The pages import docs/generated, which isn't committed + deps: [deps, ':ruby:generate'] cmds: - pnpm exec tsc --noEmit @@ -32,6 +33,7 @@ tasks: test: dir: docs desc: Run frontend tests - deps: [deps] + # The tests import docs/generated, which isn't committed + deps: [deps, ':ruby:generate'] cmds: - npm test diff --git a/scripts/generate_sorbet_log_structs.rb b/scripts/generate_sorbet_log_structs.rb deleted file mode 100644 index 95386ee3..00000000 --- a/scripts/generate_sorbet_log_structs.rb +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env ruby -# typed: strict -# frozen_string_literal: true - -# Ensure RubyGems and Bundler are set up so gem requires work in CI -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) -require "rubygems" -require "bundler/setup" - -require "sorbet-runtime" - -$LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) -require "log_struct/enums" - -require_relative "../tools/codegen/sorbet_generator" -require_relative "../tools/codegen/log_fields_generator" - -root = File.expand_path("..", __dir__) - -LogStruct::Codegen::SorbetGenerator.generate_all(root) - -system("bundle exec rubocop -A lib/log_struct/log > /dev/null 2>&1") - -LogStruct::Codegen::LogFieldsGenerator.generate(root) diff --git a/scripts/generate_structs.rb b/scripts/generate_structs.rb index fd4b7a12..da4d82f1 100755 --- a/scripts/generate_structs.rb +++ b/scripts/generate_structs.rb @@ -10,9 +10,11 @@ require "sorbet-runtime" require "fileutils" require "tempfile" +require "tmpdir" $LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) require "log_struct/enums" +require_relative "../tools/codegen/staging" require_relative "../tools/codegen/sorbet_generator" require_relative "../tools/codegen/ts_generator" require_relative "../tools/codegen/log_fields_generator" @@ -23,32 +25,36 @@ module Scripts sig { void } def self.run root = T.let(File.expand_path("..", __dir__), String) + # In the repository, so each generated file moves into place with a rename + staging_parent = File.join(root, "tmp") + FileUtils.mkdir_p(staging_parent) + Dir.mktmpdir("codegen-", staging_parent) do |dir| + generate(root, LogStruct::Codegen::Staging.new(root, dir)) + end + end + sig { params(root: String, staging: LogStruct::Codegen::Staging).void } + def self.generate(root, staging) puts "Generating Sorbet Ruby structs..." - LogStruct::Codegen::SorbetGenerator.generate_all(root) + ruby_files = LogStruct::Codegen::SorbetGenerator.generate_all(root, staging) puts "Formatting generated Ruby code..." - Tempfile.create("logstruct-codegen-rubocop") do |output| - unless system("bundle", "exec", "rubocop", "-A", "lib/log_struct/log", out: output, err: [:child, :out]) - output.rewind - warn(output.read) - raise "Failed to format generated Ruby log structs" - end - end + format_ruby(root, staging, ruby_files) + # The JSON export loads these structs + staging.install(LogStruct::Codegen::SorbetGenerator::OUT_DIR) puts "Generating log fields JSON..." - LogStruct::Codegen::LogFieldsGenerator.generate(root) + LogStruct::Codegen::LogFieldsGenerator.generate(staging) + staging.install(File.dirname(LogStruct::Codegen::LogFieldsGenerator::OUT_FILE)) puts "Generating TypeScript artifacts..." - LogStruct::Codegen::TSGenerator.generate_all(root) + LogStruct::Codegen::TSGenerator.generate_all(root, staging) puts "Exporting Sorbet JSON..." require "sorbet/typescript" - - # Ensure the freshly generated Sorbet structs are loaded before exporting require "log_struct/log" - output_dir = File.join(root, "docs", "generated", "logstruct") + output_dir = LogStruct::Codegen::TSGenerator::OUT_DIR exporter = Sorbet::Typescript::Exporter.new( enum_namespaces: ["LogStruct"], @@ -56,9 +62,27 @@ def self.run ) exporter.export( - enums_json: File.join(output_dir, "sorbet-enums.json"), - structs_json: File.join(output_dir, "sorbet-log-structs.json") + enums_json: staging.path(File.join(output_dir, "sorbet-enums.json")), + structs_json: staging.path(File.join(output_dir, "sorbet-log-structs.json")) ) + staging.install(output_dir, prune: true) + end + + # RuboCop's config names files by their paths relative to it (plain.rb may have an untyped prop), so a copy at + # the staging root applies to each staged file as it does to the file it replaces + sig { params(root: String, staging: LogStruct::Codegen::Staging, files: T::Array[String]).void } + def self.format_ruby(root, staging, files) + FileUtils.cp(File.join(root, ".rubocop.yml"), staging.path(".rubocop.yml")) + Tempfile.create("logstruct-codegen-rubocop") do |output| + # The files are named, because RuboCop leaves out what's under tmp/ when it searches a directory. Sorbet + # can't check a splat of an Array whose size it doesn't know. + command = ["bundle", "exec", "rubocop", "-A", *files] + unless system(*T.unsafe(command), chdir: staging.dir, out: output, err: [:child, :out]) + output.rewind + warn(output.read) + raise "Failed to format generated Ruby log structs" + end + end end end diff --git a/test/workflows/codegen_staging_test.rb b/test/workflows/codegen_staging_test.rb new file mode 100644 index 00000000..ca5fe32f --- /dev/null +++ b/test/workflows/codegen_staging_test.rb @@ -0,0 +1,70 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" +require_relative "../../tools/codegen/staging" + +module LogStruct + class CodegenStagingTest < ActiveSupport::TestCase + setup do + @root = Dir.mktmpdir + @staging = Codegen::Staging.new(@root, File.join(@root, "tmp", "codegen")) + end + + teardown do + FileUtils.rm_rf(@root) + end + + test "staged files stay out of the repository until they're installed" do + write_file("lib/log/plain.rb", "committed") + + @staging.write("lib/log/plain.rb", "generated") + + assert_equal "committed", File.read(File.join(@root, "lib/log/plain.rb")) + end + + test "install replaces the files it staged and keeps the others" do + write_file("lib/log/plain.rb", "committed") + write_file("lib/log/interfaces/common_fields.rb", "written by hand") + @staging.write("lib/log/plain.rb", "generated") + @staging.write("lib/log/puma/start.rb", "new source") + + @staging.install("lib/log") + + assert_equal "generated", File.read(File.join(@root, "lib/log/plain.rb")) + assert_equal "new source", File.read(File.join(@root, "lib/log/puma/start.rb")) + assert_equal "written by hand", File.read(File.join(@root, "lib/log/interfaces/common_fields.rb")) + assert_empty Dir.glob("**/*.rb", base: @staging.path("lib/log")) + end + + test "install with prune removes files and directories that are no longer generated" do + write_file("docs/generated/index.ts", "old") + write_file("docs/generated/sources/removed/index.ts", "old") + write_file("docs/other.ts", "outside") + @staging.write("docs/generated/index.ts", "new") + + @staging.install("docs/generated", prune: true) + + assert_equal ["index.ts"], Dir.children(File.join(@root, "docs/generated")) + assert_equal "new", File.read(File.join(@root, "docs/generated/index.ts")) + assert_equal "outside", File.read(File.join(@root, "docs/other.ts")) + end + + test "install with nothing staged leaves the repository alone" do + write_file("schemas/meta/log-fields.json", "committed") + + @staging.install("schemas/meta", prune: true) + + assert_equal "committed", File.read(File.join(@root, "schemas/meta/log-fields.json")) + end + + private + + def write_file(relative_path, content) + path = File.join(@root, relative_path) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, content) + end + end +end diff --git a/test/workflows/generation_test.rb b/test/workflows/generation_test.rb index f3487b36..8d847a61 100644 --- a/test/workflows/generation_test.rb +++ b/test/workflows/generation_test.rb @@ -2,12 +2,15 @@ # frozen_string_literal: true require "test_helper" +require "digest" require "open3" +require "tmpdir" module LogStruct class GenerationTest < ActiveSupport::TestCase - # Skip writes in the child process so failure checks cannot change generated files. - SETUP = <<~RUBY + ROOT = File.expand_path("../..", __dir__) + + LOAD_GENERATORS = <<~RUBY require "bundler/setup" require "sorbet-runtime" $LOAD_PATH.unshift(File.expand_path("lib")) @@ -15,13 +18,17 @@ class GenerationTest < ActiveSupport::TestCase require_relative "tools/codegen/sorbet_generator" require_relative "tools/codegen/ts_generator" require_relative "tools/codegen/log_fields_generator" - LogStruct::Codegen::SorbetGenerator.define_singleton_method(:generate_all) { |root| } - LogStruct::Codegen::TSGenerator.define_singleton_method(:generate_all) { |root| } - LogStruct::Codegen::LogFieldsGenerator.define_singleton_method(:generate) { |root| } + RUBY + + # Skip writes in the child process so failure checks cannot change generated files. + STUB_GENERATORS = <<~RUBY + LogStruct::Codegen::SorbetGenerator.define_singleton_method(:generate_all) { |root, staging| ["lib/log_struct/log/stub.rb"] } + LogStruct::Codegen::TSGenerator.define_singleton_method(:generate_all) { |root, staging| } + LogStruct::Codegen::LogFieldsGenerator.define_singleton_method(:generate) { |staging| } RUBY test "a failed formatter stops generation" do - output, status = run_generator("Kernel.define_method(:system) { |*args| false }") + output, status = run_generator(STUB_GENERATORS + "Kernel.define_method(:system) { |*args, **opts| false }") refute_predicate status, :success?, output assert_includes output, "Failed to format generated Ruby log structs" @@ -29,8 +36,8 @@ class GenerationTest < ActiveSupport::TestCase end test "a failed JSON export fails generation" do - output, status = run_generator(<<~RUBY) - Kernel.define_method(:system) { |*args| true } + output, status = run_generator(STUB_GENERATORS + <<~RUBY) + Kernel.define_method(:system) { |*args, **opts| true } require "sorbet/typescript" Sorbet::Typescript::Exporter.define_method(:export) { |**args| raise "intentional export failure" } RUBY @@ -39,11 +46,34 @@ class GenerationTest < ActiveSupport::TestCase assert_includes output, "intentional export failure" end + # `task ci` interrupts the generator when a step running beside it fails + test "an interrupted formatter leaves the generated files as they were" do + Dir.mktmpdir do |copy| + %w[lib schemas scripts tools .rubocop.yml].each { |path| FileUtils.cp_r(File.join(ROOT, path), copy) } + before = file_digests(copy) + + output, status = run_generator("Kernel.define_method(:system) { |*args, **opts| raise Interrupt }", chdir: copy) + + refute_predicate status, :success?, output + assert_includes output, "Formatting generated Ruby code..." + assert_includes output, "Interrupt" + assert_equal before, file_digests(copy) + assert_empty Dir.children(File.join(copy, "tmp")) + end + end + private - def run_generator(injection) - script = SETUP + injection + "\nload 'scripts/generate_structs.rb'\n" - Open3.capture2e(RbConfig.ruby, "-e", script, chdir: File.expand_path("../..", __dir__)) + def run_generator(injection, chdir: ROOT) + script = LOAD_GENERATORS + injection + "\nload 'scripts/generate_structs.rb'\n" + env = {"BUNDLE_GEMFILE" => File.join(ROOT, "Gemfile")} + Open3.capture2e(env, RbConfig.ruby, "-e", script, chdir: chdir) + end + + def file_digests(dir) + Dir.glob("{lib,schemas}/**/*", base: dir).select { |path| File.file?(File.join(dir, path)) }.to_h do |path| + [path, Digest::SHA256.file(File.join(dir, path)).hexdigest] + end end end end diff --git a/tools/codegen/log_fields_generator.rb b/tools/codegen/log_fields_generator.rb index 5467f9a6..8b86e48f 100644 --- a/tools/codegen/log_fields_generator.rb +++ b/tools/codegen/log_fields_generator.rb @@ -9,16 +9,17 @@ rescue LoadError require_relative "../../lib/log_struct/enums/log_field" end +require_relative "staging" module LogStruct module Codegen module LogFieldsGenerator extend T::Sig - sig { params(root: String).void } - def self.generate(root) - out_file = File.join(root, "schemas", "meta", "log-fields.json") + OUT_FILE = "schemas/meta/log-fields.json" + sig { params(staging: Staging).void } + def self.generate(staging) names = enum_constant_names data = { "$schema" => "http://json-schema.org/draft-07/schema#", @@ -32,8 +33,8 @@ def self.generate(root) } } - File.write(out_file, JSON.pretty_generate(data) + "\n") - puts "Generated: #{out_file} (#{names.size} fields)" + staging.write(OUT_FILE, JSON.pretty_generate(data) + "\n") + puts "Generated: #{OUT_FILE} (#{names.size} fields)" end sig { returns(T::Array[String]) } diff --git a/tools/codegen/sorbet_generator.rb b/tools/codegen/sorbet_generator.rb index 848d8f65..a9937f2f 100644 --- a/tools/codegen/sorbet_generator.rb +++ b/tools/codegen/sorbet_generator.rb @@ -2,12 +2,12 @@ # typed: strict # frozen_string_literal: true -require "fileutils" require "erb" require "sorbet-runtime" require_relative "models" require_relative "schema_loader" +require_relative "staging" require_relative "templates" module LogStruct @@ -15,50 +15,49 @@ module Codegen module SorbetGenerator extend T::Sig - sig { params(root: String).void } - def self.generate_all(root) - out_base_dir = File.join(root, "lib", "log_struct", "log") - FileUtils.mkdir_p(out_base_dir) + OUT_DIR = "lib/log_struct/log" + # Returns the generated files' paths in the repository + sig { params(root: String, staging: Staging).returns(T::Array[String]) } + def self.generate_all(root, staging) payloads = SchemaLoader.payloads(root) if payloads.empty? warn("No schemas found in #{File.join(root, "schemas", "log_sources")}") - return + return [] end - payloads.each do |payload| - generate_source(root, out_base_dir, payload) + payloads.flat_map do |payload| + generate_source(root, staging, payload) end end - sig { params(root: String, out_base_dir: String, payload: SourcePayload).void } - def self.generate_source(root, out_base_dir, payload) + sig { params(root: String, staging: Staging, payload: SourcePayload).returns(T::Array[String]) } + def self.generate_source(root, staging, payload) source_snake = payload.source_snake - source_dir = File.join(out_base_dir, source_snake) if payload.events.size == 1 evt = T.must(payload.events.first) - single_out = File.join(out_base_dir, "#{source_snake}.rb") single_content = render_template(root, "event.rb.erb", {payload: payload, event: evt, single_event: true}) - File.write(single_out, single_content) - puts "Generated: #{single_out}" + [write(staging, "#{source_snake}.rb", single_content)] else - FileUtils.mkdir_p(source_dir) - - payload.events.each do |evt| - ev_out = File.join(source_dir, "#{LogStruct::Codegen.snake_case(evt.name)}.rb") + files = payload.events.map do |evt| ev_content = render_template(root, "event.rb.erb", {payload: payload, event: evt, single_event: false}) - File.write(ev_out, ev_content) - puts "Generated: #{ev_out}" + write(staging, File.join(source_snake, "#{LogStruct::Codegen.snake_case(evt.name)}.rb"), ev_content) end - parent_out = File.join(out_base_dir, "#{source_snake}.rb") parent_content = render_template(root, "source_parent.rb.erb", {payload: payload}) - File.write(parent_out, parent_content) - puts "Generated: #{parent_out}" + files << write(staging, "#{source_snake}.rb", parent_content) end end + sig { params(staging: Staging, file: String, content: String).returns(String) } + def self.write(staging, file, content) + relative_path = File.join(OUT_DIR, file) + staging.write(relative_path, content) + puts "Generated: #{relative_path}" + relative_path + end + sig { params(root: String, template_name: String, locals: T::Hash[Symbol, T.untyped]).returns(String) } def self.render_template(root, template_name, locals) payload = T.cast(locals[:payload], SourcePayload) diff --git a/tools/codegen/staging.rb b/tools/codegen/staging.rb new file mode 100644 index 00000000..2ecd74d6 --- /dev/null +++ b/tools/codegen/staging.rb @@ -0,0 +1,77 @@ +#!/usr/bin/env ruby +# typed: strict +# frozen_string_literal: true + +require "fileutils" +require "sorbet-runtime" + +module LogStruct + module Codegen + # The generators write into a staging directory that mirrors the repository, and their files move into the + # repository once they're complete and formatted. Formatting takes seconds, and `task ci` stops the generator + # when a step running beside it fails: files formatted where they were committed were left unformatted. + class Staging + extend T::Sig + + sig { returns(String) } + attr_reader :dir + + sig { params(root: String, dir: String).void } + def initialize(root, dir) + @root = root + @dir = dir + end + + # The staged path of the file that will be at relative_path in the repository + sig { params(relative_path: String).returns(String) } + def path(relative_path) + File.join(@dir, relative_path) + end + + sig { params(relative_path: String, content: String).void } + def write(relative_path, content) + staged = path(relative_path) + FileUtils.mkdir_p(File.dirname(staged)) + File.write(staged, content) + end + + # Moves the staged files under relative_dir into the repository, each with one rename, so Sorbet, tsc or a + # test reading them during generation gets the old file or the new one. With prune, files the generators + # no longer write are deleted. + sig { params(relative_dir: String, prune: T::Boolean).void } + def install(relative_dir, prune: false) + staged_dir = path(relative_dir) + # Nothing was generated here, which prune would read as every file being removed + return unless File.directory?(staged_dir) + + dest_dir = File.join(@root, relative_dir) + installed = files_under(staged_dir) + installed.each do |file| + dest = File.join(dest_dir, file) + FileUtils.mkdir_p(File.dirname(dest)) + File.rename(File.join(staged_dir, file), dest) + end + remove_other_files(dest_dir, installed.to_set) if prune + end + + private + + sig { params(dir: String).returns(T::Array[String]) } + def files_under(dir) + Dir.glob("**/*", base: dir).select { |file| File.file?(File.join(dir, file)) } + end + + sig { params(dir: String, kept: T::Set[String]).void } + def remove_other_files(dir, kept) + files_under(dir).each do |file| + File.delete(File.join(dir, file)) unless kept.include?(file) + end + # A removed log source's directory, after the directories inside it + Dir.glob("**/", base: dir).sort.reverse_each do |subdir| + path = File.join(dir, subdir) + Dir.rmdir(path) if Dir.empty?(path) + end + end + end + end +end diff --git a/tools/codegen/ts_generator.rb b/tools/codegen/ts_generator.rb index aed34866..e42f42cb 100644 --- a/tools/codegen/ts_generator.rb +++ b/tools/codegen/ts_generator.rb @@ -2,12 +2,12 @@ # typed: strict # frozen_string_literal: true -require "fileutils" require "sorbet-runtime" require_relative "models" require_relative "schema_loader" require_relative "helpers" +require_relative "staging" require "erb" module LogStruct @@ -15,29 +15,27 @@ module Codegen module TSGenerator extend T::Sig - sig { params(root: String).void } - def self.generate_all(root) - out_dir = File.join(root, "docs", "generated", "logstruct") - FileUtils.rm_rf(out_dir) - FileUtils.mkdir_p(out_dir) + OUT_DIR = "docs/generated/logstruct" - generate_enums(root, out_dir) + sig { params(root: String, staging: Staging).void } + def self.generate_all(root, staging) + generate_enums(root, staging) SchemaLoader.payloads(root).each do |payload| - generate_source_ts(root, out_dir, payload) + generate_source_ts(root, staging, payload) end - generate_barrel(root, out_dir) + generate_barrel(root, staging) end - sig { params(root: String, out_dir: String).void } - def self.generate_enums(root, out_dir) + sig { params(root: String, staging: Staging).void } + def self.generate_enums(root, staging) template_path = File.join(root, "tools", "codegen", "templates", "typescript", "enums.ts.erb") erb = ERB.new(File.read(template_path), trim_mode: "-") ctx = Object.new ctx.extend(LogStruct::Codegen::Helpers) content = erb.result(ctx.instance_eval { binding }) - File.write(File.join(out_dir, "enums.ts"), content) + staging.write(File.join(OUT_DIR, "enums.ts"), content) end sig { params(payload: SourcePayload).returns(String) } @@ -52,10 +50,9 @@ def self.snake(name) .downcase end - sig { params(root: String, out_dir: String, payload: SourcePayload).void } - def self.generate_source_ts(root, out_dir, payload) - dir = File.join(out_dir, "sources", payload.source_snake) - FileUtils.mkdir_p(dir) + sig { params(root: String, staging: Staging, payload: SourcePayload).void } + def self.generate_source_ts(root, staging, payload) + dir = File.join(OUT_DIR, "sources", payload.source_snake) payload.events.each do |evt| out = File.join(dir, "#{snake(evt.name)}.ts") @@ -63,26 +60,26 @@ def self.generate_source_ts(root, out_dir, payload) erb = ERB.new(File.read(template_path), trim_mode: "-") ctx = LogStruct::Codegen::TemplateBinding.new(payload, evt, root, {}) content = erb.result(ctx.get_binding) - File.write(out, content) + staging.write(out, content) end template_path = File.join(root, "tools", "codegen", "templates", "typescript", "source_index.ts.erb") erb = ERB.new(File.read(template_path), trim_mode: "-") ctx = LogStruct::Codegen::TemplateBinding.new(payload, nil, root, {}) content = erb.result(ctx.get_binding) - File.write(File.join(dir, "index.ts"), content) + staging.write(File.join(dir, "index.ts"), content) end - sig { params(root: String, out_dir: String).void } - def self.generate_barrel(root, out_dir) - roots = Dir[File.join(out_dir, "sources", "*")].map { |p| File.basename(p) } + sig { params(root: String, staging: Staging).void } + def self.generate_barrel(root, staging) + roots = Dir[staging.path(File.join(OUT_DIR, "sources", "*"))].map { |p| File.basename(p) } template_path = File.join(root, "tools", "codegen", "templates", "typescript", "root_index.ts.erb") erb = ERB.new(File.read(template_path), trim_mode: "-") ctx = Object.new ctx.extend(LogStruct::Codegen::Helpers) ctx.define_singleton_method(:roots) { roots } content = erb.result(ctx.instance_eval { binding }) - File.write(File.join(out_dir, "index.ts"), content) + staging.write(File.join(OUT_DIR, "index.ts"), content) end end end From 9e0f92986c4b21e3c79e22bc99d05dabe9c07486 Mon Sep 17 00:00:00 2001 From: Nathan Broadbent Date: Fri, 23 Oct 2026 15:20:43 +1300 Subject: [PATCH 2/2] Run the dependency audits in GitHub CI bundler-audit and pnpm audit ran in `task ci` but not in GitHub CI, so the Release workflow, which waits for CI, didn't wait for them. A Security Audit job runs them through scripts/security_audit.sh, which `task security` runs too. --- .github/workflows/test.yml | 28 ++++++++++++++++++++++++++++ AGENTS.md | 1 + Taskfile.yml | 3 +-- scripts/security_audit.sh | 8 ++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100755 scripts/security_audit.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a2d675a..efa723e9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,6 +97,34 @@ jobs: - name: jscpd (Duplicate Code Detection) run: pnpm exec jscpd . + audit: + name: Security Audit + # The Release workflow waits for this workflow, so a release can't ship a dependency with a known advisory + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Ruby + uses: ruby/setup-ruby@651ff0d62c5c79bed6d53ec518387aae20e72c1a # v1.329.0 + with: + ruby-version: '.ruby-version' + bundler-cache: true + + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 22 + + - name: Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + with: + version: 10.18.3 + + - name: Check locked Ruby and docs dependencies for known advisories + run: scripts/security_audit.sh + docs: name: 'Next.js Site Tests' runs-on: ubuntu-22.04 diff --git a/AGENTS.md b/AGENTS.md index f32f6a15..d0497af0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,7 @@ For template file changes (tests, rake tasks, etc.), just run `scripts/rails_tes - Run all linting: `task lint` - Auto-fix all lint issues: `task lint:fix` - Spellcheck: `task spellcheck` +- Check locked dependencies for known advisories: `task security` (also a CI job, so releases wait for it) #### Ruby Commands diff --git a/Taskfile.yml b/Taskfile.yml index 4c23ed6f..95083c84 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -69,8 +69,7 @@ tasks: security: desc: Check locked Ruby and docs dependencies for known advisories cmds: - - bundle exec bundler-audit check --update - - cd docs && pnpm audit + - scripts/security_audit.sh fix: desc: Auto-fix lint issues then run the full validation workflow (replaces scripts/all_write.sh) diff --git a/scripts/security_audit.sh b/scripts/security_audit.sh new file mode 100755 index 00000000..71a76ce1 --- /dev/null +++ b/scripts/security_audit.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +# Known advisories in the locked Ruby and docs dependencies +bundle exec bundler-audit check --update +cd docs && pnpm audit