diff --git a/CHANGELOG.md b/CHANGELOG.md index 671b61b0..b0bb710d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0] - 2026-10-23 + +### Changed + +- **Breaking**: CarrierWave lines log `file_id` as an HMAC of the file's identifier (keyed like email hashes, see `hash_salt`), and no longer log `filename`, `store_path` or `ext`. CarrierWave names stored files after the uploaded file by default, and an uploaded file's name often names a person. `LogField::Filename`, `LogField::StorePath` and `LogField::Extension` are removed. +- **Breaking**: A filtered Hash's summary has its `_keys_count` but not its first 10 keys (`_keys`): a Hash's keys can be secrets themselves, like tokens or email addresses used as keys. +- **Breaking**: `additional_data` can't set any field of its line's own type, whether or not the line has it (an error line's `msg`, `backtrace` or `error_message`), or `req_id`. Like `src`, `evt`, `lvl` and `ts`, such a key is left out and handled as a LogStruct error (raised in `local_environments`). It used to replace the field. Other types' field names, like `name` on a plain line, can still be used. +- **Filtering**: Without a `hash_salt`, values hashed before the Rails app has its `secret_key_base` use a random key for each process, instead of a fixed key that was public in LogStruct's source. Set `hash_salt` to keep these hashes the same across processes. +- **Development**: `task ci` checks the locked Ruby and docs dependencies for known advisories, with `bundler-audit` and `pnpm audit`. + +### Fixed + +- **Formatting**: Hash keys longer than `max_string_bytes` are left out with their values, and counted in `_oversized_keys`. A request's parameter names could make a line any size. +- **Formatting**: `ActionController::Parameters` are written like a Hash, within the size limits, instead of being converted whole first. Lograge's `params` and Ahoy's `properties` are converted at their first level only, and only as many keys as the line keeps; `_more_keys` and `_oversized_keys` count the rest. +- **Filtering**: A Rails filter block with three arguments gets a copy of the parameters that stops at 10,000 values, 20 levels of nesting or a value that contains itself. Past those, every key the block would check is filtered. The parameters used to be copied whole. +- **Errors**: An error raised while formatting, serializing or filtering a value is reported with its class and backtrace but not its message, which can quote the value. Errors in Rails filter blocks and LogStruct's filter matchers aren't reported with the key, which can be the secret. An object whose `as_json` raises is logged as `[SERIALIZATION_ERROR]` instead of its `to_s`, and its `inspect` isn't reported. +- **Formatting**: A Hash key that isn't a String or Symbol is written as its string, scrubbed, so an email address in its `to_s` is scrubbed. +- **Filtering**: A value longer than `max_string_bytes` is hashed by its first `max_string_bytes` bytes and its length. +- **Development**: Update Next.js to 16.4.1 for the docs site, which fixes three advisories in features the static site doesn't use, and the test-only Dalli to 3.2.13. + ## [0.2.2] - 2026-10-21 ### Changed diff --git a/Gemfile b/Gemfile index 27c5e046..aca235c9 100644 --- a/Gemfile +++ b/Gemfile @@ -34,7 +34,8 @@ gem "sorbet-typescript" # Essential testing gems that don't have Ruby version restrictions group :test do - gem "dalli" + gem "dalli", "~> 3.2.13" + gem "bundler-audit", require: false gem "minitest", "~> 5.20" gem "minitest-reporters", "~> 1.6" gem "simplecov", "~> 0.22", require: false diff --git a/Gemfile.lock b/Gemfile.lock index da51faf2..a06716b8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - logstruct (0.2.2) + logstruct (0.3.0) actionpack (>= 7.1) activesupport (>= 7.1) globalid (>= 1.0) @@ -105,6 +105,9 @@ GEM bugsnag (6.27.1) concurrent-ruby (~> 1.0) builder (3.3.0) + bundler-audit (0.9.3) + bundler (>= 1.2.0) + thor (~> 1.0) carrierwave (3.1.4) activemodel (>= 6.0.0) activesupport (>= 6.0.0) @@ -119,7 +122,8 @@ GEM connection_pool (2.5.0) content_disposition (1.0.0) crass (1.0.7) - dalli (3.2.8) + dalli (3.2.13) + logger date (3.4.1) debug (1.10.0) irb (~> 1.10) @@ -460,9 +464,10 @@ DEPENDENCIES amazing_print bigdecimal bugsnag (~> 6.26) + bundler-audit carrierwave (~> 3.0) climate_control (~> 1.2) - dalli + dalli (~> 3.2.13) debug dotenv-rails (~> 3.1) drb diff --git a/Taskfile.yml b/Taskfile.yml index 231bf6fe..4c23ed6f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -64,6 +64,13 @@ tasks: - docs:test - test - ruby:tapioca:verify + - security + + security: + desc: Check locked Ruby and docs dependencies for known advisories + cmds: + - bundle exec bundler-audit check --update + - cd docs && pnpm audit fix: desc: Auto-fix lint issues then run the full validation workflow (replaces scripts/all_write.sh) diff --git a/docs/app/docs/filtering-sensitive-data/page.tsx b/docs/app/docs/filtering-sensitive-data/page.tsx index 99317b77..2d1575a2 100644 --- a/docs/app/docs/filtering-sensitive-data/page.tsx +++ b/docs/app/docs/filtering-sensitive-data/page.tsx @@ -40,8 +40,7 @@ const FILTERED_VALUES_LINE = `{ "credentials": { "_filtered": { "_class": "Hash", - "_keys_count": 2, - "_keys": ["client_id", "client_secret"] + "_keys_count": 2 } }, "src": "app", @@ -463,8 +462,8 @@ export default function FilteringSensitiveDataPage() { ).
  • - Hash: its class, how many keys it has (_keys_count) and its - first 10 keys (_keys), scrubbed like any string. An empty Hash is{' '} + Hash: its class and how many keys it has (_keys_count). + Child keys are omitted because they may contain secrets. An empty Hash is{' '} {'{"_class": "Hash", "_empty": true}'}.
  • @@ -577,7 +576,10 @@ export default function FilteringSensitiveDataPage() { when the block returns a truthy value or changes the String it was given. A block that takes three arguments gets the parameters around the key, with indifferent access: the value of the line's top-level field (a request line's params), or the line - itself for a top-level key. + itself for a top-level key. They're all there, including what the size limits leave + out of the line. Parameters with more than 10,000 values, nested more than 20 levels deep + or containing themselves aren't copied for the block: every key it would check is + filtered.
  • The partial matches of Symbols and Strings don't apply to the names of @@ -630,8 +632,10 @@ export default function FilteringSensitiveDataPage() {

    - Outside a Rails app, without hash_salt, LogStruct uses a fixed key that is - public in its source code. + Outside a Rails app, without hash_salt, LogStruct uses a random process key. + Set hash_salt if hashes must be stable across processes or restarts. Values + longer than max_string_bytes are hashed using their bounded prefix and total + byte length.

    Hashes are pseudonyms, not anonymous data: anyone who has the key and a list of email @@ -814,8 +818,8 @@ export default function FilteringSensitiveDataPage() { (its key or location), not the name it was uploaded with, which often names a person. Shrine's upload options are logged with the file names taken out of a{' '} content_disposition. A location your uploader builds from a file's name - (and CarrierWave's paths, which use the uploaded name by default) is logged as it is: - scrub it with a string_scrubbing_handler. + is logged as it is: scrub it with a string_scrubbing_handler. CarrierWave + logs an HMAC of the file ID and omits the file name and storage path.
  • Mail delivery job arguments: a deliver_later job ( diff --git a/docs/app/docs/log-fields/example-lines.json b/docs/app/docs/log-fields/example-lines.json index 37c4a953..db373235 100644 --- a/docs/app/docs/log-fields/example-lines.json +++ b/docs/app/docs/log-fields/example-lines.json @@ -387,14 +387,11 @@ "lvl": "info", "ts": "2026-10-17T09:30:15.123+00:00", "storage": "file", - "file_id": "avatar.png", + "file_id": "a4570aeb17304689", "uploader": "AvatarUploader", "model": "User", "mount_point": "avatar", "version": "", - "store_path": "uploads/user/avatar/1873/avatar.png", - "ext": "png", - "filename": "avatar.png", "mime_type": "image/png", "size": 48213, "duration_ms": 12.7, @@ -407,14 +404,11 @@ "lvl": "info", "ts": "2026-10-17T09:30:15.123+00:00", "storage": "file", - "file_id": "avatar.png", + "file_id": "a4570aeb17304689", "uploader": "AvatarUploader", "model": "User", "mount_point": "avatar", "version": "", - "store_path": "uploads/user/avatar/1873/avatar.png", - "ext": "png", - "filename": "avatar.png", "mime_type": "image/png", "size": 48213, "prog": "Rails", @@ -426,13 +420,11 @@ "lvl": "info", "ts": "2026-10-17T09:30:15.123+00:00", "storage": "file", - "file_id": "avatar.png", + "file_id": "a4570aeb17304689", "uploader": "AvatarUploader", "model": "User", "mount_point": "avatar", "version": "", - "store_path": "uploads/user/avatar/1873/avatar.png", - "ext": "png", "prog": "Rails", "req_id": "6f1c9a2e-7b4d-4e8a-9c61-2d5b8f0e4a17" }, @@ -791,8 +783,7 @@ "credentials": { "_filtered": { "_class": "Hash", - "_keys_count": 2, - "_keys": ["client_id", "client_secret"] + "_keys_count": 2 } }, "note": "Call Linda on [PHONE]", diff --git a/docs/app/docs/log-fields/page.tsx b/docs/app/docs/log-fields/page.tsx index c83fffde..64a558d2 100644 --- a/docs/app/docs/log-fields/page.tsx +++ b/docs/app/docs/log-fields/page.tsx @@ -192,13 +192,17 @@ export default function LogFieldsPage() { {json(FORMATS.additional_data)} @@ -328,8 +332,8 @@ export default function LogFieldsPage() { it's capitalized.
  • - A Hash: _class, _keys_count and its first 10 keys in{' '} - _keys. An empty Hash is {'{"_class": "Hash", "_empty": true}'}. + A Hash: _class and _keys_count. An empty Hash is{' '} + {'{"_class": "Hash", "_empty": true}'}.
  • An Array: _class and _count. An empty Array is{' '} @@ -344,7 +348,7 @@ export default function LogFieldsPage() {
  • Values nested more than 20 levels deep, and values that contain themselves, are written as - a summary without _keys. + a summary.

    • @@ -380,7 +384,9 @@ export default function LogFieldsPage() {

  • max_string_bytes (default 32768, 32 KB): a longer string is cut to that many bytes after it's scrubbed, and ends with{' '} - "... (cut from N bytes)". + "... (cut from N bytes)". A longer Hash key and its value are + omitted; + _oversized_keys counts omitted keys.
    • diff --git a/docs/app/docs/soc2-compliance/page.tsx b/docs/app/docs/soc2-compliance/page.tsx index 3d48b11b..625527ce 100644 --- a/docs/app/docs/soc2-compliance/page.tsx +++ b/docs/app/docs/soc2-compliance/page.tsx @@ -85,8 +85,8 @@ export default function SOC2CompliancePage() { keys, cookies, sessions, authorization headers, card numbers, social security numbers and the like), or a key in Rails' filter_parameters or your{' '} filter_keys, is replaced by a summary: its class, plus a Hash's key - count and first 10 keys or an Array's item count. email and{' '} - email_address values get a hash of the address too. + count or an Array's item count. email and email_address{' '} + values get a hash of the address too.
  • Strings. In every string, LogStruct replaces email addresses with{' '} @@ -113,11 +113,7 @@ export default function SOC2CompliancePage() { "credentials": { "_filtered": { "_class": "Hash", - "_keys_count": 2, - "_keys": [ - "username", - "password" - ] + "_keys_count": 2 } }, "reset_url": "https://app.example.com/reset?token=[FILTERED]&step=2", diff --git a/docs/app/docs/sorbet-types/page.tsx b/docs/app/docs/sorbet-types/page.tsx index b7f0a21a..9caee1c5 100644 --- a/docs/app/docs/sorbet-types/page.tsx +++ b/docs/app/docs/sorbet-types/page.tsx @@ -208,9 +208,10 @@ bundle exec srb tc`} LogStruct::Log::Interfaces::AdditionalDataField and{' '} LogStruct::Log::Shared::MergeAdditionalDataFields, for an optional{' '} additional_data Hash whose keys go at the top level of the line. A key named{' '} - src, evt, lvl or ts is a LogStruct - error (logstruct_errors): raised in development and test, and logged and left - out in production. + src, evt, lvl, ts or{' '} + req_id is a LogStruct error (logstruct_errors): raised in + development and test, and logged and left out in production. Your own fields win over + additional data when your serialize writes them after merging it.

    • diff --git a/docs/lib/log-generation/sample-data.ts b/docs/lib/log-generation/sample-data.ts index 7443f38a..8af002ea 100644 --- a/docs/lib/log-generation/sample-data.ts +++ b/docs/lib/log-generation/sample-data.ts @@ -193,7 +193,6 @@ export const SampleHelpers = { queue: (gen: RandomGen) => gen.sample(['default', 'mailers', 'critical', 'low']), email: (gen: RandomGen) => `user${gen.randomInt(1, 999)}@example.com`, emailArray: (gen: RandomGen) => [`user${gen.randomInt(1, 999)}@example.com`], - filename: (gen: RandomGen) => gen.sample(['file.txt', 'image.png', 'data.json']), mime: (gen: RandomGen) => gen.sample(['text/plain', 'image/png', 'application/json']), url: (_gen: RandomGen) => 'https://example.com/file', ip: (gen: RandomGen) => gen.sample(['127.0.0.1', '10.0.0.1', '192.168.1.1']), @@ -291,8 +290,6 @@ export const SampleByLogField: Readonly< [LogField.Uploader]: (_gen: RandomGen) => 'AvatarUploader', [LogField.Model]: (_gen: RandomGen) => 'User', [LogField.MountPoint]: (_gen: RandomGen) => 'avatar', - [LogField.StorePath]: (_gen: RandomGen) => 'uploads/store/file.txt', - [LogField.Extension]: (_gen: RandomGen) => '.jpg', [LogField.Sql]: SampleHelpers.sql, [LogField.Name]: SampleHelpers.name, [LogField.RowCount]: (gen: RandomGen) => gen.randomInt(0, 100), @@ -307,7 +304,6 @@ export const SampleByLogField: Readonly< [LogField.Adapter]: (_gen: RandomGen) => 'PostgreSQLAdapter', [LogField.ResourceClass]: (_gen: RandomGen) => 'User', [LogField.AhoyEvent]: (_gen: RandomGen) => 'signup', - [LogField.Filename]: SampleHelpers.filename, [LogField.Properties]: (_gen: RandomGen) => ({ plan: 'pro' }), [LogField.Data]: (_gen: RandomGen) => ({}), [LogField.DownloadOptions]: (_gen: RandomGen) => ({ filename: 'file.txt' }), diff --git a/docs/package.json b/docs/package.json index 5f630edf..1348e37a 100644 --- a/docs/package.json +++ b/docs/package.json @@ -30,7 +30,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^0.544.0", - "next": "16.4.0", + "next": "16.4.1", "next-themes": "^0.4.6", "react": "^19.3.0", "react-dom": "^19.3.0", diff --git a/docs/pnpm-lock.yaml b/docs/pnpm-lock.yaml index a0dabbd1..93fa3de4 100644 --- a/docs/pnpm-lock.yaml +++ b/docs/pnpm-lock.yaml @@ -42,8 +42,8 @@ importers: specifier: ^0.544.0 version: 0.544.0(react@19.3.0) next: - specifier: 16.4.0 - version: 16.4.0(@babel/core@7.29.7)(@playwright/test@1.64.0)(@types/node@22.20.5)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) + specifier: 16.4.1 + version: 16.4.1(@babel/core@7.29.7)(@playwright/test@1.64.0)(@types/node@22.20.5)(react-dom@19.3.0(react@19.3.0))(react@19.3.0) next-themes: specifier: ^0.4.6 version: 0.4.6(react-dom@19.3.0(react@19.3.0))(react@19.3.0) @@ -58,7 +58,7 @@ importers: version: 16.1.2(react@19.3.0) recharts: specifier: ^3.10.1 - version: 3.10.1(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react-is@19.3.0)(react@19.3.0)(redux@5.0.1) + version: 3.10.1(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react-is@19.3.1)(react@19.3.0)(redux@5.0.1) tailwind-merge: specifier: ^3.7.0 version: 3.7.0 @@ -699,57 +699,57 @@ packages: '@emnapi/core': ^1.7.1 || ^2.0.0-alpha.4 '@emnapi/runtime': ^1.7.1 || ^2.0.0-alpha.4 - '@next/env@16.4.0': - resolution: {integrity: sha512-pyLlT5oafGlX8X7xjgGNXz5dNIDSxREm+qD6i9VLSZiZFopA/XWl8NM/KPmLx/8EG/++c49iid3gNWSQW1+2dQ==} + '@next/env@16.4.1': + resolution: {integrity: sha512-5sUUrrSvs/2/2LnLoesybMGx12EtPB46fdZ6RhAw0XDGSPxU2LFyQBB7yuEiOFFJPKmBY6+q2jTr1/AMbnp0/g==} - '@next/swc-darwin-arm64@16.4.0': - resolution: {integrity: sha512-yyZ+9MgViAKLLb5l1gPBGjRwnq3teNLb5Zx6JEw8LzNSTjlsnp+baUHFxTgl7PpjCmKf48VQfyipG1eBflb48Q==} + '@next/swc-darwin-arm64@16.4.1': + resolution: {integrity: sha512-U+hQ5Gj3fpzf4fveTIF3dpUFYhaSW8vHhdAnbiIomzXzg7boreYFgBqprepXFtW4sw+aTttiNmg0gew/iFNhbg==} engines: {node: '>= 10'} cpu: [arm64] os: [darwin] - '@next/swc-darwin-x64@16.4.0': - resolution: {integrity: sha512-RePEEejEj1Auf5s9qEcG0JbcDP3ceNUS/ofNX8AhxzgIjTVlZIHMatlp16YplcSMpsVo/ZMQHJTjbMYJznjGUg==} + '@next/swc-darwin-x64@16.4.1': + resolution: {integrity: sha512-/hqC3V1XLUjhMASf/WridKdO9FXoEgN3HBWmtKbj00MPgnltutfraWJ3bYTpCRIgEIpbjJyOVLpYgAhh/BkdcQ==} engines: {node: '>= 10'} cpu: [x64] os: [darwin] - '@next/swc-linux-arm64-gnu@16.4.0': - resolution: {integrity: sha512-HhsyCqd+NpmvDDVY/LqkAN24f+QVvXh3ie+PfcwKinWL2Mu826JymQxJXGjAIfmJ4VlHjp4Pe/nZtXtUoEMAqw==} + '@next/swc-linux-arm64-gnu@16.4.1': + resolution: {integrity: sha512-GHzncHbcaS9g95kcvTNtKQW3aCLoR+xc3Jm7qrkpk1R+D2BjMHIPyvQGRPKCdMX95BsH/421jBUK06q9tEwzjw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [glibc] - '@next/swc-linux-arm64-musl@16.4.0': - resolution: {integrity: sha512-sN9Pf+Zh+RpZAOmbR7d8KAA8T61CVhawtbEJ2Bk6QrRzx6Z/6haaF4UiwBzytM96GRRRWZh9/WALIEO3npMamA==} + '@next/swc-linux-arm64-musl@16.4.1': + resolution: {integrity: sha512-1SuJyDxW/PR9dHJat/ZRgcYLhuSZ+1psP4DlKb2tKIuXbjygqpLrJWRSq+onZQBv8LcmU67EIrmb6c3XTaJLjw==} engines: {node: '>= 10'} cpu: [arm64] os: [linux] libc: [musl] - '@next/swc-linux-x64-gnu@16.4.0': - resolution: {integrity: sha512-D5M5xLWQVd1CvR+L7BickM1weRfmM41x3L1NWcnKY/oyJJqr1qwaCgmu3dMAhLBOMdZwSuDCRV6DJRRpiUkbDQ==} + '@next/swc-linux-x64-gnu@16.4.1': + resolution: {integrity: sha512-Istng1hMGp9TGO4lUqeKLd70MPN/OOkucVTP2Nmj+gLkwBgmPqUw8CJYBCC7nbl/B/23bBCcjCI5Hsh6hazC1A==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [glibc] - '@next/swc-linux-x64-musl@16.4.0': - resolution: {integrity: sha512-8d5WLS6a9YalqpOgXdlbvonFu+sIgjA+CO6tGzetrpPVcUL56foa7B7664h41rAlbLi5fJwK4vWji3zf4QDy+w==} + '@next/swc-linux-x64-musl@16.4.1': + resolution: {integrity: sha512-aSC9wMNiZRRFVpPkPWBzRwLQxIGm/CiQcXyFF1XNcgodb+uMxdk7aSSGGpDXCbnvZdf4itjHxpOd5h7i7zaiYw==} engines: {node: '>= 10'} cpu: [x64] os: [linux] libc: [musl] - '@next/swc-win32-arm64-msvc@16.4.0': - resolution: {integrity: sha512-sQuOIiW3snb48UwrBcDMvFA/u03qtw2LeuSIxFIQ9AJuW9UG/4Ddsfo+eV73PnZwBhyncMtsKVh7PrzMol2/fA==} + '@next/swc-win32-arm64-msvc@16.4.1': + resolution: {integrity: sha512-t/myvwFDlypMUjRvbBf8qLPyuaB0iXYYdBtwMt6DM6f+60wtGr4bSYholdPqNeV5VUXZhrpMwfza829kmhS3iA==} engines: {node: '>= 10'} cpu: [arm64] os: [win32] - '@next/swc-win32-x64-msvc@16.4.0': - resolution: {integrity: sha512-6/33lCuxpdHUy27RegyGM/5uTpF7YZd3UR4D0gpkOA/tWY5YR7msbQjqPgCZmSmxrrTLGCXYA4A8vT2H5DPGTQ==} + '@next/swc-win32-x64-msvc@16.4.1': + resolution: {integrity: sha512-6ICqPo9awCkrtYNPhwkx1uU7rlFWrZRrIboDHS2LU+jv088UBYQtmJ25h6pCp27/+FZjF6BQ4Te4ErugyMeuBg==} engines: {node: '>= 10'} cpu: [x64] os: [win32] @@ -1647,6 +1647,11 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + baseline-browser-mapping@2.11.29: + resolution: {integrity: sha512-YeesxhUmR4h5y5V8KF8iBCqJb8YFrBJ++XBTeGcWIaQOYXGN19JM0FoiOjAhwuDoln8owMgXcWBmgHtYSkky2w==} + engines: {node: '>=6.0.0'} + hasBin: true + brace-expansion@2.1.7: resolution: {integrity: sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==} @@ -1681,6 +1686,9 @@ packages: caniuse-lite@1.0.30001815: resolution: {integrity: sha512-uDybUEyfgbyJJ93J1vikTylBX31AkdVYeVPeffMQpf8XN7hSfr0oZOn4VFOIMNvtU5+rHvC5Udg2TgtJKHm23Q==} + caniuse-lite@1.0.30001816: + resolution: {integrity: sha512-+8Yx7BXCoqk6IF8yTicjfUR6D/ldJzdlsodlEOiOSPmhK844WIgk67/zIQaGAOxNkZyyvBlrRFnLvqKIoqbAFw==} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -2417,8 +2425,8 @@ packages: react: ^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc react-dom: ^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc - next@16.4.0: - resolution: {integrity: sha512-N2C+9q89Z/oLrsepSRS4jRMtwmUZAoZuAHU0e9T6+sAB5DdhADmrnOW+zjYijlrYFmVfmQhGVGy9eMFoB7UgKg==} + next@16.4.1: + resolution: {integrity: sha512-uc657V62E34fOI6W5NtG9IzwsPcms0Z60grz2MJGtglgD+pTFnNt4Yw2PzaU622tfX4vhrDgLZzicE2YlhI20Q==} engines: {node: '>=20.9.0'} hasBin: true peerDependencies: @@ -2541,6 +2549,10 @@ packages: resolution: {integrity: sha512-49cGhUbXj8Qenv0iTMxA1cFBzxXoctpC9Ujd77t1WcbJIr6nF/eI7g/8MgxrYldFRuAXvja7xQRwavoW7kgrxQ==} engines: {node: ^10 || ^12 || >=14} + postcss@8.5.30: + resolution: {integrity: sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==} + engines: {node: ^10 || ^12 || >=14} + pretty-format@30.5.1: resolution: {integrity: sha512-byhRAPguVKMQIj4kjJwJ5lAskVhfuiSdiYl/aLTWpgkGEmic2jYhJh1yE9ih8Ox44Xg9ccCT11/S6QrzSJuNrg==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -2567,8 +2579,8 @@ packages: react-is@18.3.1: resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} - react-is@19.3.0: - resolution: {integrity: sha512-UpMYezM4v5/18F28aC66AEsjXIgE02kyEMH6yLdgLXu/UTfa1Ntwck/nNLrbqJsEXW7gPb0coNO9FQse9WTovA==} + react-is@19.3.1: + resolution: {integrity: sha512-RRieF6IyxQBkcK6pP8xOHprLTPW6QjRkbGdImtqNme+/hYVwCZkLQLdlZGOhUqXwkS9IEynaGvPJtv6XB/cz0Q==} react-redux@9.3.0: resolution: {integrity: sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==} @@ -3672,30 +3684,30 @@ snapshots: '@tybys/wasm-util': 0.10.4 optional: true - '@next/env@16.4.0': {} + '@next/env@16.4.1': {} - '@next/swc-darwin-arm64@16.4.0': + '@next/swc-darwin-arm64@16.4.1': optional: true - '@next/swc-darwin-x64@16.4.0': + '@next/swc-darwin-x64@16.4.1': optional: true - '@next/swc-linux-arm64-gnu@16.4.0': + '@next/swc-linux-arm64-gnu@16.4.1': optional: true - '@next/swc-linux-arm64-musl@16.4.0': + '@next/swc-linux-arm64-musl@16.4.1': optional: true - '@next/swc-linux-x64-gnu@16.4.0': + '@next/swc-linux-x64-gnu@16.4.1': optional: true - '@next/swc-linux-x64-musl@16.4.0': + '@next/swc-linux-x64-musl@16.4.1': optional: true - '@next/swc-win32-arm64-msvc@16.4.0': + '@next/swc-win32-arm64-msvc@16.4.1': optional: true - '@next/swc-win32-x64-msvc@16.4.0': + '@next/swc-win32-x64-msvc@16.4.1': optional: true '@parcel/watcher-android-arm64@2.6.0': @@ -4503,6 +4515,8 @@ snapshots: baseline-browser-mapping@2.11.28: {} + baseline-browser-mapping@2.11.29: {} + brace-expansion@2.1.7: dependencies: balanced-match: 1.0.2 @@ -4535,6 +4549,8 @@ snapshots: caniuse-lite@1.0.30001815: {} + caniuse-lite@1.0.30001816: {} + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -5412,25 +5428,25 @@ snapshots: react: 19.3.0 react-dom: 19.3.0(react@19.3.0) - next@16.4.0(@babel/core@7.29.7)(@playwright/test@1.64.0)(@types/node@22.20.5)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): + next@16.4.1(@babel/core@7.29.7)(@playwright/test@1.64.0)(@types/node@22.20.5)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: - '@next/env': 16.4.0 + '@next/env': 16.4.1 '@swc/helpers': 0.5.23 - baseline-browser-mapping: 2.11.28 - caniuse-lite: 1.0.30001815 - postcss: 8.5.29 + baseline-browser-mapping: 2.11.29 + caniuse-lite: 1.0.30001816 + postcss: 8.5.30 react: 19.3.0 react-dom: 19.3.0(react@19.3.0) styled-jsx: 5.1.6(@babel/core@7.29.7)(react@19.3.0) optionalDependencies: - '@next/swc-darwin-arm64': 16.4.0 - '@next/swc-darwin-x64': 16.4.0 - '@next/swc-linux-arm64-gnu': 16.4.0 - '@next/swc-linux-arm64-musl': 16.4.0 - '@next/swc-linux-x64-gnu': 16.4.0 - '@next/swc-linux-x64-musl': 16.4.0 - '@next/swc-win32-arm64-msvc': 16.4.0 - '@next/swc-win32-x64-msvc': 16.4.0 + '@next/swc-darwin-arm64': 16.4.1 + '@next/swc-darwin-x64': 16.4.1 + '@next/swc-linux-arm64-gnu': 16.4.1 + '@next/swc-linux-arm64-musl': 16.4.1 + '@next/swc-linux-x64-gnu': 16.4.1 + '@next/swc-linux-x64-musl': 16.4.1 + '@next/swc-win32-arm64-msvc': 16.4.1 + '@next/swc-win32-x64-msvc': 16.4.1 '@playwright/test': 1.64.0 sharp: 0.35.5(@types/node@22.20.5) transitivePeerDependencies: @@ -5531,10 +5547,16 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.2 + postcss@8.5.30: + dependencies: + nanoid: 3.3.20 + picocolors: 1.1.1 + source-map-js: 1.2.2 + pretty-format@30.5.1: dependencies: '@jest/react-is-18': react-is@18.3.1 - '@jest/react-is-19': react-is@19.3.0 + '@jest/react-is-19': react-is@19.3.1 '@jest/schemas': 30.5.0 ansi-styles: 5.2.0 @@ -5553,7 +5575,7 @@ snapshots: react-is@18.3.1: {} - react-is@19.3.0: {} + react-is@19.3.1: {} react-redux@9.3.0(@types/react@19.3.0)(react@19.3.0)(redux@5.0.1): dependencies: @@ -5603,7 +5625,7 @@ snapshots: react@19.3.0: {} - recharts@3.10.1(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react-is@19.3.0)(react@19.3.0)(redux@5.0.1): + recharts@3.10.1(@types/react@19.3.0)(react-dom@19.3.0(react@19.3.0))(react-is@19.3.1)(react@19.3.0)(redux@5.0.1): dependencies: '@reduxjs/toolkit': 2.13.0(react-redux@9.3.0(@types/react@19.3.0)(react@19.3.0)(redux@5.0.1))(react@19.3.0) clsx: 2.1.1 @@ -5613,7 +5635,7 @@ snapshots: immer: 11.1.21 react: 19.3.0 react-dom: 19.3.0(react@19.3.0) - react-is: 19.3.0 + react-is: 19.3.1 react-redux: 9.3.0(@types/react@19.3.0)(react@19.3.0)(redux@5.0.1) reselect: 5.2.0 tiny-invariant: 1.3.3 diff --git a/lib/log_struct/bounded_input.rb b/lib/log_struct/bounded_input.rb new file mode 100644 index 00000000..26b4f914 --- /dev/null +++ b/lib/log_struct/bounded_input.rb @@ -0,0 +1,101 @@ +# typed: strict +# frozen_string_literal: true + +require "action_controller" + +module LogStruct + # Copies of logged Hashes and Arrays that stop before an input too large to copy in full + module BoundedInput + extend T::Sig + + # A Rails filter block with three arguments gets every parameter, so a copy for it keeps what the formatter + # leaves out of the line. These limits only stop values no request sends: nesting deeper than the formatter + # writes, and more values than a form or JSON body has. A copy that reaches one is truncated, and the + # filters that read it fail closed. + MAX_DEPTH = 20 + MAX_NODES = 10_000 + + # The keys that count what a Hash's copy left out. The formatter writes them as it writes its own counts: + # Rails' default filter_parameters include :_key, which would filter them. + COUNT_KEYS = T.let(Set[:_more_keys, :_oversized_keys].freeze, T::Set[Symbol]) + + class State < T::Struct + prop :remaining_nodes, Integer, default: MAX_NODES + prop :truncated, T::Boolean, default: false + end + + sig do + params(value: T.untyped, state: State, depth: Integer, ancestors: T::Hash[T.untyped, T::Boolean]).returns(T.untyped) + end + def self.copy(value, state:, depth: 0, ancestors: {}.compare_by_identity) + if depth > MAX_DEPTH || ancestors.key?(value) || state.remaining_nodes <= 0 + state.truncated = true + return nil + end + state.remaining_nodes -= 1 + + case value + when Hash, ::ActionController::Parameters + ancestors[value] = true + begin + result = {} + value.each_pair do |key, child| + break if state.truncated + + result[key] = copy(child, state: state, depth: depth + 1, ancestors: ancestors) + end + result + ensure + ancestors.delete(value) + end + when Array + ancestors[value] = true + begin + result = [] + value.each do |child| + break if state.truncated + + result << copy(child, state: state, depth: depth + 1, ancestors: ancestors) + end + result + ensure + ancestors.delete(value) + end + else + value + end + end + + # How many keys a Hash or ActionController::Parameters has, without converting its values + sig { params(hash: T.untyped).returns(Integer) } + def self.key_count(hash) + hash.is_a?(Hash) ? hash.size : hash.keys.size + end + + # The first keys of a Hash or ActionController::Parameters as Symbols, for a log struct's Hash field, without + # converting the keys the formatter would leave out. The formatter writes max_hash_keys entries of a Hash, + # and the counts of the keys left out here are entries too: the last key's slot goes to _more_keys, and an + # omitted oversized key's slot to _oversized_keys, so the formatter writes both counts as they are. + sig { params(hash: T.untyped).returns(T::Hash[Symbol, T.untyped]) } + def self.symbolize_keys(hash) + limits = LogStruct.config.limits + total = key_count(hash) + examined = (total > limits.max_hash_keys) ? [limits.max_hash_keys - 1, 0].max : total + result = T.let({}, T::Hash[Symbol, T.untyped]) + oversized_keys = 0 + hash.each_pair.with_index do |(key, value), index| + break if index >= examined + + if key.to_s.bytesize > limits.max_string_bytes + oversized_keys += 1 + next + end + + result[key.to_s.to_sym] = value + end + result[:_more_keys] = total - examined if examined < total + result[:_oversized_keys] = oversized_keys if oversized_keys.positive? + result + end + end +end diff --git a/lib/log_struct/concerns/configuration.rb b/lib/log_struct/concerns/configuration.rb index 28e3bb5d..3b038be4 100644 --- a/lib/log_struct/concerns/configuration.rb +++ b/lib/log_struct/concerns/configuration.rb @@ -259,7 +259,7 @@ def build_path_block_matcher(filter) result = filter.call(key, value, root) !!result || (value.is_a?(String) && value != original_value) rescue => e - handle_filter_error(e, filter, original_key || "") + handle_filter_error(e, filter) true end ConfigStruct::PathFilterMatcher.new(callable: callable, label: filter.inspect) @@ -290,11 +290,11 @@ def build_callable_filter_matcher(filter) begin !!filter.call(key) rescue => e - handle_filter_error(e, filter, key) + handle_filter_error(e, filter) true end rescue => e - handle_filter_error(e, filter, key) + handle_filter_error(e, filter) true end ConfigStruct::FilterMatcher.new(callable: callable, label: filter.inspect) @@ -305,11 +305,11 @@ def arity_for_filter(filter) filter.respond_to?(:arity) ? filter.arity : 2 end - sig { params(error: StandardError, filter: T.untyped, key: String).void } - def handle_filter_error(error, filter, key) + # The key isn't reported: a key can be the secret (an email address or a token used as a key) + sig { params(error: StandardError, filter: T.untyped).void } + def handle_filter_error(error, filter) context = { filter: filter.class.name, - key: key, filter_label: begin filter.inspect rescue @@ -317,7 +317,7 @@ def handle_filter_error(error, filter, key) end } - LogStruct.handle_exception(error, source: Source::Internal, context: context) + LogStruct.handle_value_error(error, "filter_parameters entry failed", context: context) end end end diff --git a/lib/log_struct/concerns/error_handling.rb b/lib/log_struct/concerns/error_handling.rb index 9b42400f..93b5d0dd 100644 --- a/lib/log_struct/concerns/error_handling.rb +++ b/lib/log_struct/concerns/error_handling.rb @@ -89,6 +89,16 @@ def filtered_context(context) Thread.current[:logstruct_formatting_internal_error] = previous_internal_error end + # Handle an error raised while LogStruct read a value it was logging, without the error's message: the + # message can quote that value, as a NoMethodError quotes its receiver's inspect before Ruby 3.4. + # failure says what failed, and the reported error keeps the original's class and backtrace. + sig { params(error: Exception, failure: String, context: T.nilable(T::Hash[Symbol, T.untyped])).void } + def handle_value_error(error, failure, context: nil) + reported = RuntimeError.new("#{error.class}: #{failure}") + reported.set_backtrace(error.backtrace) + handle_exception(reported, source: Source::Internal, context: context) + end + # Handle an error according to the configured error handling mode (log, report, raise, etc). # Pass `log: false` when the caller has already logged the error in its own structured line. sig { params(error: StandardError, source: Source, context: T.nilable(T::Hash[Symbol, T.untyped]), log: T::Boolean).void } diff --git a/lib/log_struct/config_struct/filters.rb b/lib/log_struct/config_struct/filters.rb index 04d92711..83919908 100644 --- a/lib/log_struct/config_struct/filters.rb +++ b/lib/log_struct/config_struct/filters.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "active_support/hash_with_indifferent_access" +require_relative "../bounded_input" module LogStruct # A key's words, lowercase and joined by underscores, so that keys match however they're written: @@ -38,8 +39,8 @@ def matches?(key, value) # A filter that needs more than the key: Rails' entries with a dot (`card.number`) match a key by its # path, and Rails gives filter blocks with three arguments the parameters around the key. The path # starts at the line's top-level field (`["params", "card", "number"]`); `root` is that field's value, - # or the whole line for a top-level key, with indifferent access as Rails gives it. Building it copies - # every value inside it, so a matcher that doesn't read it sets uses_root to false and gets nil. + # or the whole line for a top-level key, with indifferent access as Rails gives it. A matcher + # that doesn't read the root sets uses_root to false and gets nil. class PathFilterMatcher < T::Struct extend T::Sig @@ -50,7 +51,10 @@ class PathFilterMatcher < T::Struct sig { params(path: T::Array[String], value: T.untyped, root: T.untyped).returns(T::Boolean) } def matches?(path, value, root) root = nil unless uses_root - callable.call(path, value, root.is_a?(LazyRoot) ? root.value : root) + root = root.value if root.is_a?(LazyRoot) + return true if root.equal?(LazyRoot::UNAVAILABLE) + + callable.call(path, value, root) end end @@ -58,6 +62,8 @@ def matches?(path, value, root) class LazyRoot extend T::Sig + UNAVAILABLE = T.let(Object.new.freeze, Object) + sig { params(value: T.untyped).void } def initialize(value) @source = value @@ -70,12 +76,20 @@ def initialize(value) def value return @value if @built - @built = true - @value = begin - @source.is_a?(Hash) ? ActiveSupport::HashWithIndifferentAccess.new(@source) : @source - rescue SystemStackError, StandardError - @source + state = BoundedInput::State.new + bounded = BoundedInput.copy(@source, state: state) + @value = if state.truncated + UNAVAILABLE + elsif bounded.is_a?(Hash) + ActiveSupport::HashWithIndifferentAccess.new(bounded) + else + bounded end + @built = true + @value + rescue SystemStackError, StandardError + @built = true + @value = UNAVAILABLE end end @@ -86,7 +100,7 @@ class Filters < T::Struct # A filtered value is replaced by a summary of it, with no length (which would say how long a # secret is): # - # { _filtered: { _class: "Hash", _keys_count: 3, _keys: [:key1, :key2, :key3] } } # first 10 keys + # { _filtered: { _class: "Hash", _keys_count: 3 } } # { _filtered: { _class: "Array", _count: 2 } } # { _filtered: { _class: "String" } } # plus its _hash for filter_keys_with_hashes # diff --git a/lib/log_struct/config_struct/limits.rb b/lib/log_struct/config_struct/limits.rb index fb8bbed4..018f7b90 100644 --- a/lib/log_struct/config_struct/limits.rb +++ b/lib/log_struct/config_struct/limits.rb @@ -21,7 +21,7 @@ class Limits < T::Struct # Default: 20 prop :max_backtrace_lines, Integer, default: 20 - # Strings longer than this many bytes are cut, and say how long they were + # Strings longer than this many bytes are cut; oversized Hash keys and their values are omitted # Default: 32768 (32 KB) prop :max_string_bytes, Integer, default: 32_768 end diff --git a/lib/log_struct/enums/log_field.rb b/lib/log_struct/enums/log_field.rb index 531edbd6..b3027c29 100644 --- a/lib/log_struct/enums/log_field.rb +++ b/lib/log_struct/enums/log_field.rb @@ -120,7 +120,6 @@ class LogField < T::Enum Storage = new(:storage) Operation = new(:op) FileId = new(:file_id) - Filename = new(:filename) MimeType = new(:mime_type) Size = new(:size) Metadata = new(:metadata) @@ -134,8 +133,6 @@ class LogField < T::Enum Model = new(:model) MountPoint = new(:mount_point) Version = new(:version) - StorePath = new(:store_path) - Extension = new(:ext) # SQL-specific fields Sql = new(:sql) diff --git a/lib/log_struct/formatter.rb b/lib/log_struct/formatter.rb index 3f0236d8..469dd351 100644 --- a/lib/log_struct/formatter.rb +++ b/lib/log_struct/formatter.rb @@ -10,6 +10,7 @@ require_relative "enums/event" require_relative "enums/log_field" require_relative "string_scrubber" +require_relative "bounded_input" require_relative "log" require_relative "param_filters" require_relative "multi_error_reporter" @@ -80,13 +81,13 @@ def process_values(arg, recursion_depth: 0, ancestors: {}.compare_by_identity, p return summarize_unprocessed_value(arg) if recursion_depth > MAX_RECURSION_DEPTH || ancestors.key?(arg) case arg - when Hash, Array + when Hash, Array, ::ActionController::Parameters ancestors[arg] = true begin - if arg.is_a?(Hash) - process_hash(arg, recursion_depth: recursion_depth, ancestors: ancestors, path: path, root: root) - else + if arg.is_a?(Array) process_array(arg, recursion_depth: recursion_depth, ancestors: ancestors, path: path, root: root) + else + process_hash(arg, recursion_depth: recursion_depth, ancestors: ancestors, path: path, root: root) end ensure ancestors.delete(arg) @@ -107,7 +108,7 @@ def process_values(arg, recursion_depth: 0, ancestors: {}.compare_by_identity, p scrub_string(String(object)) end rescue => e - LogStruct.handle_exception(e, source: Source::Internal) + LogStruct.handle_value_error(e, "GlobalID formatting failed") "[GLOBALID_ERROR]" end end @@ -125,26 +126,24 @@ def process_values(arg, recursion_depth: 0, ancestors: {}.compare_by_identity, p when Numeric, TrueClass, FalseClass, NilClass arg else - # Structs, ActionController::Parameters and other objects: filter and scrub what + # Structs and other objects: filter and scrub what # to_json would write process_values(arg.as_json, recursion_depth: recursion_depth + 1, ancestors: ancestors, path: path, root: root) end rescue SystemStackError, StandardError => e # A SystemStackError comes from an object whose as_json refers back to itself - e = RuntimeError.new("#{e.class}: #{e.message}") unless e.is_a?(StandardError) - # Report error through LogStruct's framework context = { processor_method: "process_values", value_type: arg.class.name, recursion_depth: recursion_depth } - LogStruct.handle_exception(e, source: Source::Internal, context: context) + LogStruct.handle_value_error(e, "value formatting failed", context: context) summarize_unprocessed_value(arg) end sig do params( - hash: T::Hash[T.untyped, T.untyped], + hash: T.untyped, recursion_depth: Integer, ancestors: T::Hash[T.untyped, T::Boolean], path: T.nilable(T::Array[String]), @@ -158,13 +157,30 @@ def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil) line_root = (path && recursion_depth == 0) ? ConfigStruct::LazyRoot.new(hash) : nil max_keys = LogStruct.config.limits.max_hash_keys - hash.each_with_index do |(key, value), index| + max_key_bytes = LogStruct.config.limits.max_string_bytes + oversized_keys = 0 + hash.each_pair.with_index do |(key, value), index| if index >= max_keys - result[:_more_keys] = hash.size - max_keys + result[:_more_keys] = BoundedInput.key_count(hash) - max_keys break end + if BoundedInput::COUNT_KEYS.include?(key) && value.is_a?(Integer) + result[key] = value + next + end + + if key.to_s.bytesize > max_key_bytes + oversized_keys += 1 + next + end + + # Checked again: each byte that isn't valid UTF-8 becomes a 3-byte U+FFFD key = valid_utf8_key(key) + if key.to_s.bytesize > max_key_bytes + oversized_keys += 1 + next + end logstruct_field = recursion_depth == 0 && LOG_FIELD_KEYS.include?(key) key_path = path && [*path, key.to_s] # A top-level key's root is the line; the keys below it get the top-level field's value @@ -173,7 +189,9 @@ def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil) # Check if this key should be filtered at any depth filtered = ParamFilters.should_filter_key?(key, value, logstruct_field: logstruct_field, path: key_path, root: key_root) # Keys are scrubbed too: an email address can be a key - result[StringScrubber.scrub_name(key)] = if !filtered + scrubbed_key = StringScrubber.scrub_name(key) + scrubbed_key = scrub_limited_string(scrubbed_key) if scrubbed_key.is_a?(String) && scrubbed_key.bytesize > max_key_bytes + result[scrubbed_key] = if !filtered process_values(value, recursion_depth: recursion_depth + 1, ancestors: ancestors, path: key_path, root: children_root) elsif value == RAILS_FILTERED_MASK # Rails' parameter filter has already replaced the value @@ -183,6 +201,8 @@ def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil) end end + result[:_oversized_keys] = oversized_keys if oversized_keys.positive? + result end @@ -205,7 +225,7 @@ def valid_utf8_key(key) fixed = valid_utf8(name) fixed.equal?(name) ? key : fixed.to_sym else - key + valid_utf8(key.to_s) end end @@ -295,15 +315,10 @@ def log_value_to_hash(log_value, time:) log_value.to_s end rescue => e - # Handle serialization errors - context = { - object_class: log_value.class.name, - object_inspect: log_value.inspect.truncate(100) - } - LogStruct.handle_exception(e, source: Source::Internal, context: context) - - # Fall back to the string representation to ensure we continue processing - log_value.to_s + LogStruct.handle_value_error(e, "value serialization failed", context: {object_class: log_value.class.name}) + + # An object's string representation may contain the same secret that broke serialization. + "[SERIALIZATION_ERROR]" end end diff --git a/lib/log_struct/hash_utils.rb b/lib/log_struct/hash_utils.rb index 79bf2034..6d80cdfc 100644 --- a/lib/log_struct/hash_utils.rb +++ b/lib/log_struct/hash_utils.rb @@ -2,13 +2,14 @@ # frozen_string_literal: true require "openssl" +require "securerandom" module LogStruct # Utility module for hashing sensitive data module HashUtils - # The key when there's neither a configured hash_salt nor a Rails app with a secret_key_base. - # It's public, so these hashes can be matched to a list of candidate values. - FALLBACK_KEY = "l0g5t0p" + # The key without a hash_salt, before the Rails app has its secret_key_base. A fixed key would be public, + # and anyone could match its hashes to a list of candidate emails; these hashes differ between processes. + FALLBACK_KEY = T.let(::SecureRandom.random_bytes(32).freeze, String) class << self extend T::Sig @@ -17,7 +18,13 @@ class << self sig { params(value: String).returns(String) } def hash_value(value) length = LogStruct.config.filters.hash_length - OpenSSL::HMAC.hexdigest("SHA256", hash_key, value)[0...length] || "error" + max_bytes = LogStruct.config.limits.max_string_bytes + input = if value.bytesize > max_bytes + value.byteslice(0, max_bytes).to_s.b + "\0".b + value.bytesize.to_s.b + else + value + end + OpenSSL::HMAC.hexdigest("SHA256", hash_key, input)[0...length] || "error" end # The configured hash_salt, or a key derived from the app's secret_key_base diff --git a/lib/log_struct/integrations/ahoy.rb b/lib/log_struct/integrations/ahoy.rb index d52c2787..ff627ab9 100644 --- a/lib/log_struct/integrations/ahoy.rb +++ b/lib/log_struct/integrations/ahoy.rb @@ -1,6 +1,8 @@ # typed: strict # frozen_string_literal: true +require_relative "../bounded_input" + module LogStruct module Integrations # Ahoy analytics integration. If Ahoy is present, prepend a small hook to @@ -45,10 +47,9 @@ def track(name, *args) # Event properties can come from the browser, so they can be anything; a non-Hash is logged as its class sig { params(properties: T.untyped).returns(T.nilable(T::Hash[Symbol, T.untyped])) } def self.properties_for_log(properties) - properties = properties.to_unsafe_h if properties.respond_to?(:to_unsafe_h) case properties when nil then nil - when Hash then properties.to_h { |key, value| [key.to_s.to_sym, value] } + when Hash, ::ActionController::Parameters then BoundedInput.symbolize_keys(properties) else {_class: properties.class.name} end end diff --git a/lib/log_struct/integrations/carrierwave.rb b/lib/log_struct/integrations/carrierwave.rb index 1e0ccfef..74d6a6d7 100644 --- a/lib/log_struct/integrations/carrierwave.rb +++ b/lib/log_struct/integrations/carrierwave.rb @@ -6,6 +6,7 @@ rescue LoadError # CarrierWave gem is not available, integration will be skipped end +require_relative "../hash_utils" module LogStruct module Integrations @@ -49,7 +50,6 @@ def store!(*args) # file_fields is a Hash Sorbet can't check against the struct's keyword arguments Log::CarrierWave::Upload.new( **T.unsafe(file_fields), - filename: file&.filename, mime_type: file&.content_type, size: file&.size, duration_ms: duration_ms @@ -72,7 +72,6 @@ def download!(uri, remote_headers = {}) log_file_operation do Log::CarrierWave::Download.new( **T.unsafe(file_fields), - filename: file&.filename, mime_type: file&.content_type, size: file&.size ) @@ -87,13 +86,11 @@ def download!(uri, remote_headers = {}) def file_fields { storage: storage.class.name.to_s.split("::").last.to_s.downcase.to_sym, - file_id: identifier.to_s, + file_id: HashUtils.hash_value(identifier.to_s), uploader: self.class.name, model: model&.class&.name, mount_point: mounted_as&.to_s, - version: version_name&.to_s, - store_path: store_path, - extension: file&.extension + version: version_name&.to_s } end diff --git a/lib/log_struct/integrations/lograge.rb b/lib/log_struct/integrations/lograge.rb index 12aea7fe..3665b1af 100644 --- a/lib/log_struct/integrations/lograge.rb +++ b/lib/log_struct/integrations/lograge.rb @@ -6,6 +6,7 @@ rescue LoadError # Lograge gem is not available, integration will be skipped end +require_relative "../bounded_input" module LogStruct module Integrations @@ -196,12 +197,12 @@ def apply_custom_options(event, options) result = custom_options_proc.call(event, options) return if !result.is_a?(Hash) || result.equal?(options) - options.merge!(result.to_h { |key, value| [key.to_s.to_sym, value] }) + options.merge!(BoundedInput.symbolize_keys(result)) end sig { params(data: T::Hash[T.any(Symbol, String), T.untyped]).returns(Log::Request) } def build_request_log(data) - normalized_data = normalize_lograge_data(data) + normalized_data = BoundedInput.symbolize_keys(data) # Coerce common fields to expected types status = ((s = normalized_data[:status]) && s.respond_to?(:to_i)) ? s.to_i : s @@ -210,7 +211,7 @@ def build_request_log(data) db = ((b = normalized_data[:db]) && b.respond_to?(:to_f)) ? b.to_f : b params = normalized_data[:params] - params = params.deep_symbolize_keys if params&.respond_to?(:deep_symbolize_keys) + params = BoundedInput.symbolize_keys(params) if params.is_a?(Hash) || params.is_a?(::ActionController::Parameters) additional_data = extract_additional_data(normalized_data) @@ -237,13 +238,6 @@ def build_request_log(data) ) end - sig { params(data: T::Hash[T.any(Symbol, String), T.untyped]).returns(T::Hash[Symbol, T.untyped]) } - def normalize_lograge_data(data) - data.each_with_object({}) do |(key, value), normalized| - normalized[key.to_s.to_sym] = value - end - end - sig { params(data: T::Hash[Symbol, T.untyped]).returns(T.nilable(T::Hash[Symbol, T.untyped])) } def extract_additional_data(data) extras = T.let({}, T::Hash[Symbol, T.untyped]) diff --git a/lib/log_struct/log/action_mailer/delivered.rb b/lib/log_struct/log/action_mailer/delivered.rb index 15806a57..a6dd9049 100644 --- a/lib/log_struct/log/action_mailer/delivered.rb +++ b/lib/log_struct/log/action_mailer/delivered.rb @@ -42,6 +42,16 @@ class Delivered < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::To, LogField::From, LogField::Subject, LogField::MessageId, LogField::MailerClass, LogField::MailerAction, LogField::AttachmentCount].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/action_mailer/delivery.rb b/lib/log_struct/log/action_mailer/delivery.rb index bf189225..57814a88 100644 --- a/lib/log_struct/log/action_mailer/delivery.rb +++ b/lib/log_struct/log/action_mailer/delivery.rb @@ -42,6 +42,16 @@ class Delivery < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::To, LogField::From, LogField::Subject, LogField::MessageId, LogField::MailerClass, LogField::MailerAction, LogField::AttachmentCount].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/action_mailer/error.rb b/lib/log_struct/log/action_mailer/error.rb index fe3aff17..17988c92 100644 --- a/lib/log_struct/log/action_mailer/error.rb +++ b/lib/log_struct/log/action_mailer/error.rb @@ -47,6 +47,16 @@ class Error < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::To, LogField::From, LogField::Subject, LogField::MessageId, LogField::MailerClass, LogField::MailerAction, LogField::AttachmentCount, LogField::ErrorClass, LogField::Message, LogField::Backtrace].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/carrierwave.rb b/lib/log_struct/log/carrierwave.rb index 99a49304..c66b602c 100644 --- a/lib/log_struct/log/carrierwave.rb +++ b/lib/log_struct/log/carrierwave.rb @@ -21,8 +21,6 @@ class BaseFields < T::Struct const :model, T.nilable(String), default: nil const :mount_point, T.nilable(String), default: nil const :version, T.nilable(String), default: nil - const :store_path, T.nilable(String), default: nil - const :extension, T.nilable(String), default: nil Kwargs = T.type_alias do { @@ -31,9 +29,7 @@ class BaseFields < T::Struct uploader: T.nilable(String), model: T.nilable(String), mount_point: T.nilable(String), - version: T.nilable(String), - store_path: T.nilable(String), - extension: T.nilable(String) + version: T.nilable(String) } end @@ -45,9 +41,7 @@ def to_kwargs uploader: uploader, model: model, mount_point: mount_point, - version: version, - store_path: store_path, - extension: extension + version: version } end end diff --git a/lib/log_struct/log/carrierwave/delete.rb b/lib/log_struct/log/carrierwave/delete.rb index 04ac0cdd..2fe6a189 100644 --- a/lib/log_struct/log/carrierwave/delete.rb +++ b/lib/log_struct/log/carrierwave/delete.rb @@ -35,8 +35,6 @@ class Delete < T::Struct const :model, T.nilable(String), default: nil const :mount_point, T.nilable(String), default: nil const :version, T.nilable(String), default: nil - const :store_path, T.nilable(String), default: nil - const :extension, T.nilable(String), default: nil # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -51,8 +49,6 @@ def to_h h[LogField::Model] = model unless model.nil? h[LogField::MountPoint] = mount_point unless mount_point.nil? h[LogField::Version] = version unless version.nil? - h[LogField::StorePath] = store_path unless store_path.nil? - h[LogField::Extension] = extension unless extension.nil? h end end diff --git a/lib/log_struct/log/carrierwave/download.rb b/lib/log_struct/log/carrierwave/download.rb index 276916f6..29813e15 100644 --- a/lib/log_struct/log/carrierwave/download.rb +++ b/lib/log_struct/log/carrierwave/download.rb @@ -35,11 +35,8 @@ class Download < T::Struct const :model, T.nilable(String), default: nil const :mount_point, T.nilable(String), default: nil const :version, T.nilable(String), default: nil - const :store_path, T.nilable(String), default: nil - const :extension, T.nilable(String), default: nil # Event-specific fields - const :filename, T.nilable(String), default: nil const :mime_type, T.nilable(String), default: nil const :size, T.nilable(Integer), default: nil @@ -56,9 +53,6 @@ def to_h h[LogField::Model] = model unless model.nil? h[LogField::MountPoint] = mount_point unless mount_point.nil? h[LogField::Version] = version unless version.nil? - h[LogField::StorePath] = store_path unless store_path.nil? - h[LogField::Extension] = extension unless extension.nil? - h[LogField::Filename] = filename unless filename.nil? h[LogField::MimeType] = mime_type unless mime_type.nil? h[LogField::Size] = size unless size.nil? h diff --git a/lib/log_struct/log/carrierwave/upload.rb b/lib/log_struct/log/carrierwave/upload.rb index 93d4a7d2..1dbf9294 100644 --- a/lib/log_struct/log/carrierwave/upload.rb +++ b/lib/log_struct/log/carrierwave/upload.rb @@ -35,11 +35,8 @@ class Upload < T::Struct const :model, T.nilable(String), default: nil const :mount_point, T.nilable(String), default: nil const :version, T.nilable(String), default: nil - const :store_path, T.nilable(String), default: nil - const :extension, T.nilable(String), default: nil # Event-specific fields - const :filename, T.nilable(String), default: nil const :mime_type, T.nilable(String), default: nil const :size, T.nilable(Integer), default: nil const :metadata, T.nilable(T::Hash[String, T.untyped]), default: nil @@ -58,9 +55,6 @@ def to_h h[LogField::Model] = model unless model.nil? h[LogField::MountPoint] = mount_point unless mount_point.nil? h[LogField::Version] = version unless version.nil? - h[LogField::StorePath] = store_path unless store_path.nil? - h[LogField::Extension] = extension unless extension.nil? - h[LogField::Filename] = filename unless filename.nil? h[LogField::MimeType] = mime_type unless mime_type.nil? h[LogField::Size] = size unless size.nil? h[LogField::Metadata] = metadata unless metadata.nil? diff --git a/lib/log_struct/log/error.rb b/lib/log_struct/log/error.rb index c64e4edc..955c28ff 100644 --- a/lib/log_struct/log/error.rb +++ b/lib/log_struct/log/error.rb @@ -48,6 +48,16 @@ class Error < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Path, LogField::HttpMethod, LogField::SourceIp, LogField::UserAgent, LogField::Referer, LogField::ErrorClass, LogField::Message, LogField::ErrorMessage, LogField::Backtrace, LogField::Status, LogField::CauseClass, LogField::CauseMessage].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Request fields (optional) include LogStruct::Log::Interfaces::RequestFields diff --git a/lib/log_struct/log/plain.rb b/lib/log_struct/log/plain.rb index 18008aa5..a4e038c4 100644 --- a/lib/log_struct/log/plain.rb +++ b/lib/log_struct/log/plain.rb @@ -36,6 +36,16 @@ class Plain < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Message].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/puma/error.rb b/lib/log_struct/log/puma/error.rb index eb10cbf9..306001e3 100644 --- a/lib/log_struct/log/puma/error.rb +++ b/lib/log_struct/log/puma/error.rb @@ -42,6 +42,16 @@ class Error < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::ErrorClass, LogField::Message, LogField::Backtrace, LogField::ErrorContext, LogField::HttpMethod, LogField::Path].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/puma/shutdown.rb b/lib/log_struct/log/puma/shutdown.rb index c7cc2f24..4f75b05a 100644 --- a/lib/log_struct/log/puma/shutdown.rb +++ b/lib/log_struct/log/puma/shutdown.rb @@ -37,6 +37,16 @@ class Shutdown < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::ProcessId].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/puma/start.rb b/lib/log_struct/log/puma/start.rb index 28bf3e0b..2a5e2256 100644 --- a/lib/log_struct/log/puma/start.rb +++ b/lib/log_struct/log/puma/start.rb @@ -45,6 +45,16 @@ class Start < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Mode, LogField::PumaVersion, LogField::PumaCodename, LogField::RubyVersion, LogField::MinThreads, LogField::MaxThreads, LogField::Environment, LogField::ProcessId, LogField::ListeningAddresses].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/request.rb b/lib/log_struct/log/request.rb index bbfdc303..7b84cf27 100644 --- a/lib/log_struct/log/request.rb +++ b/lib/log_struct/log/request.rb @@ -52,6 +52,16 @@ class Request < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Path, LogField::HttpMethod, LogField::SourceIp, LogField::UserAgent, LogField::Referer, LogField::Format, LogField::Controller, LogField::Action, LogField::Status, LogField::DurationMs, LogField::View, LogField::Database, LogField::Params, LogField::Host, LogField::ContentType, LogField::Accept].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Request fields (optional) include LogStruct::Log::Interfaces::RequestFields diff --git a/lib/log_struct/log/security/blocked_host.rb b/lib/log_struct/log/security/blocked_host.rb index 4f5dbb61..157f6297 100644 --- a/lib/log_struct/log/security/blocked_host.rb +++ b/lib/log_struct/log/security/blocked_host.rb @@ -46,6 +46,16 @@ class BlockedHost < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Path, LogField::HttpMethod, LogField::SourceIp, LogField::UserAgent, LogField::Referer, LogField::Message, LogField::BlockedHost, LogField::BlockedHosts, LogField::XForwardedFor].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Request fields (optional) include LogStruct::Log::Interfaces::RequestFields diff --git a/lib/log_struct/log/security/csrf_violation.rb b/lib/log_struct/log/security/csrf_violation.rb index 0d83b5a8..91cabd08 100644 --- a/lib/log_struct/log/security/csrf_violation.rb +++ b/lib/log_struct/log/security/csrf_violation.rb @@ -43,6 +43,16 @@ class CSRFViolation < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Path, LogField::HttpMethod, LogField::SourceIp, LogField::UserAgent, LogField::Referer, LogField::Message].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Request fields (optional) include LogStruct::Log::Interfaces::RequestFields diff --git a/lib/log_struct/log/security/ip_spoof.rb b/lib/log_struct/log/security/ip_spoof.rb index e7e18438..f8eec375 100644 --- a/lib/log_struct/log/security/ip_spoof.rb +++ b/lib/log_struct/log/security/ip_spoof.rb @@ -45,6 +45,16 @@ class IPSpoof < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Path, LogField::HttpMethod, LogField::SourceIp, LogField::UserAgent, LogField::Referer, LogField::Message, LogField::ClientIp, LogField::XForwardedFor].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Request fields (optional) include LogStruct::Log::Interfaces::RequestFields diff --git a/lib/log_struct/log/sidekiq/error.rb b/lib/log_struct/log/sidekiq/error.rb index 43e4ba34..3a1ae75f 100644 --- a/lib/log_struct/log/sidekiq/error.rb +++ b/lib/log_struct/log/sidekiq/error.rb @@ -42,6 +42,16 @@ class Error < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::ErrorClass, LogField::Message, LogField::Backtrace, LogField::ErrorContext, LogField::QueueName, LogField::RetryCount].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/sidekiq/log.rb b/lib/log_struct/log/sidekiq/log.rb index 453bb83f..b906c66f 100644 --- a/lib/log_struct/log/sidekiq/log.rb +++ b/lib/log_struct/log/sidekiq/log.rb @@ -40,6 +40,16 @@ class Log < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Message, LogField::Context, LogField::ProcessId, LogField::ThreadId].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/log/sql.rb b/lib/log_struct/log/sql.rb index cd10587d..cbf92414 100644 --- a/lib/log_struct/log/sql.rb +++ b/lib/log_struct/log/sql.rb @@ -47,6 +47,16 @@ class SQL < T::Struct const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + FIELD_KEYS = T.let( + [LogField::Message, LogField::Sql, LogField::Name, LogField::DurationMs, LogField::RowCount, LogField::Adapter, LogField::BindParams, LogField::DatabaseName, LogField::ConnectionPoolSize, LogField::ActiveConnections, LogField::OperationType, LogField::TableNames].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end + # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon diff --git a/lib/log_struct/param_filters.rb b/lib/log_struct/param_filters.rb index 6b0b22ce..18df5514 100644 --- a/lib/log_struct/param_filters.rb +++ b/lib/log_struct/param_filters.rb @@ -43,7 +43,7 @@ def should_filter_key?(key, value = nil, logstruct_field: false, path: nil, root matcher.matches?(normalized_key, value) rescue => e - handle_filter_matcher_error(e, matcher.label, normalized_key) + handle_filter_matcher_error(e, matcher.label) # An unsuccessful filter must never expose the value it was meant to protect. true end @@ -55,7 +55,7 @@ def should_filter_key?(key, value = nil, logstruct_field: false, path: nil, root filters.path_matchers.any? do |matcher| matcher.matches?(path, value, root) rescue => e - handle_filter_matcher_error(e, matcher.label, normalized_key) + handle_filter_matcher_error(e, matcher.label) true end end @@ -94,16 +94,14 @@ def summarize_string(string, include_hash) filtered_string end - # Summarize a filtered Hash: its first 10 keys and how many there are. No size, which would say - # how long the secrets in it are. + # A filtered Hash's child keys can themselves contain secrets. sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T::Hash[Symbol, T.untyped]) } def summarize_hash(hash) return {_class: "Hash", _empty: true} if hash.empty? { _class: Hash, - _keys_count: hash.size, - _keys: hash.each_key.take(10).map { |key| normalize_summary_key(StringScrubber.scrub_name(key)) } + _keys_count: hash.size } end @@ -120,26 +118,9 @@ def summarize_array(array) private - sig { params(key: T.any(String, Symbol, Integer, T.untyped)).returns(T.any(Symbol, String)) } - def normalize_summary_key(key) - if key.is_a?(Symbol) - key - elsif key.respond_to?(:to_sym) - key.to_sym - else - key.to_s - end - rescue - key.to_s - end - - sig { params(error: StandardError, label: String, key: String).void } - def handle_filter_matcher_error(error, label, key) - context = { - matcher: label, - key: key - } - LogStruct.handle_exception(error, source: Source::Internal, context: context) + sig { params(error: StandardError, label: String).void } + def handle_filter_matcher_error(error, label) + LogStruct.handle_value_error(error, "filter matcher failed", context: {matcher: label}) end end end diff --git a/lib/log_struct/shared/merge_additional_data_fields.rb b/lib/log_struct/shared/merge_additional_data_fields.rb index e865cef3..c0f74b0b 100644 --- a/lib/log_struct/shared/merge_additional_data_fields.rb +++ b/lib/log_struct/shared/merge_additional_data_fields.rb @@ -3,6 +3,7 @@ require_relative "interfaces/additional_data_field" require_relative "../enums/source" +require_relative "../enums/log_field" module LogStruct module Log @@ -14,17 +15,32 @@ module MergeAdditionalDataFields requires_ancestor { T::Struct } requires_ancestor { Interfaces::AdditionalDataField } - # Reserved keys that cannot be overwritten by additional_data. - # These are the core log structure fields that must not be modified. - RESERVED_KEYS = T.let(%i[src evt lvl ts].freeze, T::Array[Symbol]) + # Every line's own fields, and the request ID the formatter adds unless the line has one + RESERVED_KEYS = T.let( + [LogField::Source, LogField::Event, LogField::Level, LogField::Timestamp, LogField::RequestId].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + NO_FIELD_KEYS = T.let(Set.new.freeze, T::Set[Symbol]) + + # The keys of every field of the struct's type, set or not on this line, which additional_data can't + # set: it would replace the field, or stand in for one the type didn't write. LogStruct's log types list + # theirs. A custom type's serialize writes its own fields after merging additional_data, so they win. + sig { overridable.returns(T::Set[Symbol]) } + def field_keys + NO_FIELD_KEYS + end sig { params(hash: T::Hash[Symbol, T.untyped]).void } def merge_additional_data_fields(hash) ad = additional_data return unless ad - ad.each do |key, value| + ad.each_pair.with_index do |(key, value), index| + break if index >= LogStruct.config.limits.max_hash_keys + next if key.to_s.bytesize > LogStruct.config.limits.max_string_bytes + sym_key = key.to_sym - if RESERVED_KEYS.include?(sym_key) + if RESERVED_KEYS.include?(sym_key) || field_keys.include?(sym_key) LogStruct.handle_exception( ArgumentError.new("additional_data attempted to overwrite reserved key: #{sym_key}"), source: Source::Internal diff --git a/lib/log_struct/version.rb b/lib/log_struct/version.rb index a1269418..29f98cb9 100644 --- a/lib/log_struct/version.rb +++ b/lib/log_struct/version.rb @@ -2,5 +2,5 @@ # frozen_string_literal: true module LogStruct - VERSION = "0.2.2" + VERSION = "0.3.0" end diff --git a/schemas/log_sources/carrierwave.yml b/schemas/log_sources/carrierwave.yml index d9a56db4..e2847704 100644 --- a/schemas/log_sources/carrierwave.yml +++ b/schemas/log_sources/carrierwave.yml @@ -17,16 +17,11 @@ base: Model: String MountPoint: String Version: String - StorePath: String - Extension: String events: Upload: default_required: false fields: - Filename: - type: String - required: false MimeType: type: String required: false @@ -47,9 +42,6 @@ events: Download: default_required: false fields: - Filename: - type: String - required: false MimeType: type: String required: false diff --git a/schemas/meta/log-fields.json b/schemas/meta/log-fields.json index fdd9914f..6e5319b2 100644 --- a/schemas/meta/log-fields.json +++ b/schemas/meta/log-fields.json @@ -45,10 +45,8 @@ "ExecutionTime", "Executions", "Exist", - "Extension", "File", "FileId", - "Filename", "FinishedAt", "Format", "From", @@ -104,7 +102,6 @@ "StartedAt", "Status", "Storage", - "StorePath", "Subject", "TableNames", "ThreadId", diff --git a/sorbet/rbi/gems/bundler-audit@0.9.3.rbi b/sorbet/rbi/gems/bundler-audit@0.9.3.rbi new file mode 100644 index 00000000..7527df3a --- /dev/null +++ b/sorbet/rbi/gems/bundler-audit@0.9.3.rbi @@ -0,0 +1,315 @@ +# typed: true + +# DO NOT EDIT MANUALLY +# This is an autogenerated file for types exported from the `bundler-audit` gem. +# Please instead update this file by running `bin/tapioca gem bundler-audit`. + + +# source://bundler-audit//lib/bundler/audit/advisory.rb#22 +module Bundler::Audit; end + +# Represents an advisory loaded from the {Database}. +# +# source://bundler-audit//lib/bundler/audit/advisory.rb#26 +class Bundler::Audit::Advisory < ::Struct + # Compares two advisories. + # + # @param other [Advisory] + # @return [Boolean] + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#210 + def ==(other); end + + # Determines how critical the vulnerability is. + # + # @return [:none, :low, :medium, :high, :critical, nil] The criticality of the vulnerability based on the CVSS score. + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#138 + def criticality; end + + # The CVE identifier. + # + # @return [String, nil] + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#93 + def cve_id; end + + # The GHSA (GitHub Security Advisory) identifier + # + # @return [String, nil] + # @since 0.7.0 + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#113 + def ghsa_id; end + + # Return a compacted list of all ids + # + # @return [Array] + # @since 0.7.0 + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#124 + def identifiers; end + + # The OSVDB identifier. + # + # @return [String, nil] + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#102 + def osvdb_id; end + + # Checks whether the version is patched against the advisory. + # + # @param version [Gem::Version] The version to compare against {#patched_versions}. + # @return [Boolean] Specifies whether the version is patched against the advisory. + # @since 0.2.0 + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#184 + def patched?(version); end + + # Converts the advisory to a Hash. + # + # @return [Hash{Symbol => Object}] + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#219 + def to_h; end + + # Returns the value of attribute id + # + # @return [Object] the current value of id + def to_s; end + + # Checks whether the version is not affected by the advisory. + # + # @param version [Gem::Version] The version to compare against {#unaffected_versions}. + # @return [Boolean] Specifies whether the version is not affected by the advisory. + # @since 0.2.0 + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#167 + def unaffected?(version); end + + # Checks whether the version is vulnerable to the advisory. + # + # @param version [Gem::Version] The version to compare against {#patched_versions}. + # @return [Boolean] Specifies whether the version is vulnerable to the advisory or not. + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#199 + def vulnerable?(version); end + + class << self + # Loads the advisory from a YAML file. + # + # @api semipublic + # @param path [String] The path to the advisory YAML file. + # @return [Advisory] + # + # source://bundler-audit//lib/bundler/audit/advisory.rb#50 + def load(path); end + end +end + +# Represents the directory of advisories, grouped by gem name +# and CVE number. +# +# source://bundler-audit//lib/bundler/audit/database.rb#29 +class Bundler::Audit::Database + # Initializes the Advisory Database. + # + # @param path [String] The path to the advisory database. + # @raise [ArgumentError] The path was not a directory. + # @return [Database] a new instance of Database + # + # source://bundler-audit//lib/bundler/audit/database.rb#62 + def initialize(path = T.unsafe(nil)); end + + # Enumerates over every advisory in the database. + # + # @return [Enumerator] If no block is given, an Enumerator will be returned. + # @yield [advisory] If a block is given, it will be passed each advisory. + # @yieldparam advisory [Advisory] An advisory from the database. + # + # source://bundler-audit//lib/bundler/audit/database.rb#265 + def advisories(&block); end + + # Enumerates over advisories for the given gem. + # + # @param name [String] The gem name to lookup. + # @return [Enumerator] If no block is given, an Enumerator will be returned. + # @yield [advisory] If a block is given, each advisory for the given gem will be yielded. + # @yieldparam advisory [Advisory] An advisory for the given gem. + # + # source://bundler-audit//lib/bundler/audit/database.rb#288 + def advisories_for(name); end + + # Verifies whether the gem is effected by any advisories. + # + # @param gem [Gem::Specification] The gem to verify. + # @return [Enumerator] If no block is given, an Enumerator will be returned. + # @yield [advisory] If a block is given, it will be passed advisories that effect + # the gem. + # @yieldparam advisory [Advisory] An advisory that effects the specific version of the gem. + # + # source://bundler-audit//lib/bundler/audit/database.rb#312 + def check_gem(gem); end + + # The last commit ID of the repository. + # + # @return [String, nil] The commit hash or `nil` if the database is not a git repository. + # @since 0.9.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#228 + def commit_id; end + + # Determines if the database is a git repository. + # + # @return [Boolean] + # @since 0.8.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#180 + def git?; end + + # Inspects the database. + # + # @return [String] The inspected database. + # + # source://bundler-audit//lib/bundler/audit/database.rb#348 + def inspect; end + + # Determines the time when the database was last updated. + # + # @return [Time] + # @since 0.8.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#243 + def last_updated_at; end + + # The path to the advisory database. + # + # @return [String] + # + # source://bundler-audit//lib/bundler/audit/database.rb#51 + def path; end + + # The number of advisories within the database. + # + # @return [Integer] The number of advisories. + # + # source://bundler-audit//lib/bundler/audit/database.rb#328 + def size; end + + # Converts the database to a String. + # + # @return [String] The path to the database. + # + # source://bundler-audit//lib/bundler/audit/database.rb#338 + def to_s; end + + # Updates the ruby-advisory-db. + # + # @option options + # @param options [Hash] Additional options. + # @raise [UpdateFailed] Could not update the ruby-advisory-db git repository. + # @return [true, nil] * `true` - the ruby-advisory-db git repository was successfully + # updated. + # * `nil` - the ruby-advisory-db is not a git repository or the `git` + # command is not installed. + # @since 0.8.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#204 + def update!(options = T.unsafe(nil)); end + + protected + + # Enumerates over every advisory path in the database. + # + # @yield [path] The given block will be passed each advisory path. + # @yieldparam path [String] A path to an advisory `.yml` file. + # + # source://bundler-audit//lib/bundler/audit/database.rb#363 + def each_advisory_path(&block); end + + # Enumerates over the advisories for the given gem. + # + # @param name [String] The gem of the gem. + # @yield [path] The given block will be passed each advisory path. + # @yieldparam path [String] A path to an advisory `.yml` file. + # + # source://bundler-audit//lib/bundler/audit/database.rb#379 + def each_advisory_path_for(name, &block); end + + class << self + # Downloads the ruby-advisory-db. + # + # @note Requires network access. + # @option options + # @option options + # @param options [Hash] Additional options. + # @raise [DownloadFailed] Indicates that the download failed. + # @return [Database] The newly downloaded database. + # @since 0.8.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#117 + def download(options = T.unsafe(nil)); end + + # Tests whether the database exists. + # + # @param path [String] The given path of the database to check. + # @return [Boolean] + # @since 0.8.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#90 + def exists?(path = T.unsafe(nil)); end + + # The default path for the database. + # + # @return [String] The path to the database directory. + # + # source://bundler-audit//lib/bundler/audit/database.rb#76 + def path; end + + # Updates the ruby-advisory-db. + # + # @deprecated Use {#update!} instead. + # @note Requires network access. + # @option options + # @param options [Hash] Additional options. + # @raise [ArgumentError] Invalid options were given. + # @return [Boolean] Specifies whether the update was successful. + # @since 0.3.0 + # + # source://bundler-audit//lib/bundler/audit/database.rb#157 + def update!(options = T.unsafe(nil)); end + end +end + +# Default path to the ruby-advisory-db. +# +# @since 0.8.0 +# +# source://bundler-audit//lib/bundler/audit/database.rb#46 +Bundler::Audit::Database::DEFAULT_PATH = T.let(T.unsafe(nil), String) + +# source://bundler-audit//lib/bundler/audit/database.rb#31 +class Bundler::Audit::Database::DownloadFailed < ::RuntimeError; end + +# Git URL of the ruby-advisory-db. +# +# source://bundler-audit//lib/bundler/audit/database.rb#38 +Bundler::Audit::Database::URL = T.let(T.unsafe(nil), String) + +# Path to the user's copy of the ruby-advisory-db. +# +# source://bundler-audit//lib/bundler/audit/database.rb#41 +Bundler::Audit::Database::USER_PATH = T.let(T.unsafe(nil), String) + +# source://bundler-audit//lib/bundler/audit/database.rb#34 +class Bundler::Audit::Database::UpdateFailed < ::RuntimeError; end + +# bundler-audit version +# +# source://bundler-audit//lib/bundler/audit/version.rb#21 +Bundler::Audit::VERSION = T.let(T.unsafe(nil), String) + +module Bundler::MatchRemoteMetadata + include ::Bundler::FetchMetadata + include ::Bundler::MatchMetadata +end diff --git a/sorbet/rbi/gems/dalli@3.2.8.rbi b/sorbet/rbi/gems/dalli@3.2.13.rbi similarity index 85% rename from sorbet/rbi/gems/dalli@3.2.8.rbi rename to sorbet/rbi/gems/dalli@3.2.13.rbi index 0ddd9cdf..12e0b3e1 100644 --- a/sorbet/rbi/gems/dalli@3.2.8.rbi +++ b/sorbet/rbi/gems/dalli@3.2.13.rbi @@ -77,7 +77,7 @@ class Dalli::Client # Make sure memcache servers are alive, or raise an Dalli::RingError # - # source://dalli//lib/dalli/client.rb#342 + # source://dalli//lib/dalli/client.rb#348 def alive!; end # Append value to the value already stored on the server for 'key'. @@ -86,7 +86,7 @@ class Dalli::Client # source://dalli//lib/dalli/client.rb#246 def append(key, value); end - # source://dalli//lib/dalli/client.rb#361 + # source://dalli//lib/dalli/client.rb#367 def cache_nils; end # compare and swap values using optimistic locking. @@ -116,11 +116,13 @@ class Dalli::Client # Close our connection to each server. # If you perform another operation after this, the connections will be re-established. # - # source://dalli//lib/dalli/client.rb#349 + # source://dalli//lib/dalli/client.rb#355 def close; end # Decr subtracts the given amount from the counter on the memcached server. # Amt must be a positive integer value. + # Default, if given, must be an Integer (or a String of decimal digits); + # anything else raises ArgumentError. # # memcached counters are unsigned and cannot hold negative values. Calling # decr on a counter which is 0 will just return 0. @@ -135,7 +137,7 @@ class Dalli::Client # # If the value already exists, it must have been set with raw: true # - # source://dalli//lib/dalli/client.rb#292 + # source://dalli//lib/dalli/client.rb#297 def decr(key, amt = T.unsafe(nil), ttl = T.unsafe(nil), default = T.unsafe(nil)); end # source://dalli//lib/dalli/client.rb#239 @@ -162,13 +164,13 @@ class Dalli::Client # Flush the memcached server, at 'delay' seconds in the future. # Delay defaults to zero seconds, which means an immediate flush. # - # source://dalli//lib/dalli/client.rb#302 + # source://dalli//lib/dalli/client.rb#308 def flush(delay = T.unsafe(nil)); end # Flush the memcached server, at 'delay' seconds in the future. # Delay defaults to zero seconds, which means an immediate flush. # - # source://dalli//lib/dalli/client.rb#302 + # source://dalli//lib/dalli/client.rb#308 def flush_all(delay = T.unsafe(nil)); end # Gat (get and touch) fetch an item and simultaneously update its expiration time. @@ -210,6 +212,8 @@ class Dalli::Client # Incr adds the given amount to the counter on the memcached server. # Amt must be a positive integer value. + # Default, if given, must be an Integer (or a String of decimal digits); + # anything else raises ArgumentError. # # If default is nil, the counter must already exist or the operation # will fail and will return nil. Otherwise this method will return @@ -221,7 +225,7 @@ class Dalli::Client # # If the value already exists, it must have been set with raw: true # - # source://dalli//lib/dalli/client.rb#270 + # source://dalli//lib/dalli/client.rb#272 def incr(key, amt = T.unsafe(nil), ttl = T.unsafe(nil), default = T.unsafe(nil)); end # Turn on quiet aka noreply support for a number of @@ -246,7 +250,7 @@ class Dalli::Client # @return [Boolean] # - # source://dalli//lib/dalli/client.rb#357 + # source://dalli//lib/dalli/client.rb#363 def not_found?(val); end # Prepend value to the value already stored on the server for 'key'. @@ -291,12 +295,12 @@ class Dalli::Client # Close our connection to each server. # If you perform another operation after this, the connections will be re-established. # - # source://dalli//lib/dalli/client.rb#349 + # source://dalli//lib/dalli/client.rb#355 def reset; end # Reset stats for each server. # - # source://dalli//lib/dalli/client.rb#324 + # source://dalli//lib/dalli/client.rb#330 def reset_stats; end # source://dalli//lib/dalli/client.rb#200 @@ -312,7 +316,7 @@ class Dalli::Client # You can optionally pass a type including :items, :slabs or :settings to get specific stats # Returns a hash like { 'hostname:port' => { 'stat1' => 'value1', ... }, 'hostname2:port' => { ... } } # - # source://dalli//lib/dalli/client.rb#313 + # source://dalli//lib/dalli/client.rb#319 def stats(type = T.unsafe(nil)); end # Touch updates expiration time for a given key. @@ -324,7 +328,7 @@ class Dalli::Client # Version of the memcache servers. # - # source://dalli//lib/dalli/client.rb#332 + # source://dalli//lib/dalli/client.rb#338 def version; end # Stub method so a bare Dalli client can pretend to be a connection pool. @@ -332,20 +336,20 @@ class Dalli::Client # @yield [_self] # @yieldparam _self [Dalli::Client] the object that the method was called on # - # source://dalli//lib/dalli/client.rb#366 + # source://dalli//lib/dalli/client.rb#372 def with; end private - # source://dalli//lib/dalli/client.rb#376 + # source://dalli//lib/dalli/client.rb#390 def cas_core(key, always_set, ttl = T.unsafe(nil), req_options = T.unsafe(nil)); end # @raise [ArgumentError] # - # source://dalli//lib/dalli/client.rb#372 + # source://dalli//lib/dalli/client.rb#378 def check_positive!(amt); end - # source://dalli//lib/dalli/client.rb#433 + # source://dalli//lib/dalli/client.rb#448 def normalize_options(opts); end # Chokepoint method for memcached methods with a key argument. @@ -357,29 +361,37 @@ class Dalli::Client # a particular memcached instance becomes unreachable, or the # operational times out. # - # source://dalli//lib/dalli/client.rb#417 + # source://dalli//lib/dalli/client.rb#432 def perform(*all_args); end - # source://dalli//lib/dalli/client.rb#440 + # source://dalli//lib/dalli/client.rb#455 def pipelined_getter; end - # source://dalli//lib/dalli/client.rb#398 + # source://dalli//lib/dalli/client.rb#413 def protocol_implementation; end - # source://dalli//lib/dalli/client.rb#394 + # source://dalli//lib/dalli/client.rb#409 def ring; end # Uses the argument TTL or the client-wide default. Ensures # that the value is an integer # - # source://dalli//lib/dalli/client.rb#388 + # source://dalli//lib/dalli/client.rb#403 def ttl_or_default(ttl); end + + # Numeric arguments that become meta protocol flags (GHSA-6wmv-xq9m-fmp7). + # Checked before the request starts so a bad value raises a clean + # ArgumentError, instead of Protocol::Base#request logging it as + # unexpected and closing the connection. + # + # source://dalli//lib/dalli/client.rb#386 + def validate_integer!(name, value); end end -# source://dalli//lib/dalli/client.rb#307 +# source://dalli//lib/dalli/client.rb#313 Dalli::Client::ALLOWED_STAT_KEYS = T.let(T.unsafe(nil), Array) -# source://dalli//lib/dalli/client.rb#355 +# source://dalli//lib/dalli/client.rb#361 Dalli::Client::CACHE_NILS = T.let(T.unsafe(nil), Hash) # Default compressor used by Dalli, that uses @@ -388,14 +400,51 @@ Dalli::Client::CACHE_NILS = T.let(T.unsafe(nil), Hash) # source://dalli//lib/dalli/compressor.rb#11 class Dalli::Compressor class << self + # @raise [Dalli::UnmarshalError] + # + # source://dalli//lib/dalli/compressor.rb#59 + def append_within_limit(out, chunk, max_bytes); end + # source://dalli//lib/dalli/compressor.rb#12 def compress(data); end - # source://dalli//lib/dalli/compressor.rb#16 - def decompress(data); end + # max_bytes caps the decompressed size; inflating stops and raises as soon + # as the output passes it, so a small compressed value can't expand into + # gigabytes of memory. + # + # source://dalli//lib/dalli/compressor.rb#26 + def decompress(data, max_bytes: T.unsafe(nil)); end + + # Both yielded and returned output count toward the limit + # + # source://dalli//lib/dalli/compressor.rb#54 + def inflate_slice(inflater, slice, out, max_bytes); end + + # source://dalli//lib/dalli/compressor.rb#38 + def inflate_within_limit(inflater, data, max_bytes); end + + # False when even the largest possible expansion stays within max_bytes, + # so the value can be inflated in one step + # + # @return [Boolean] + # + # source://dalli//lib/dalli/compressor.rb#34 + def may_exceed?(data, max_bytes); end end end +# Compressed input is fed to zlib this many bytes at a time. MRI's zlib +# yields the output in small chunks; one that returns it instead still +# produces at most MAX_EXPANSION times this per step. +# +# source://dalli//lib/dalli/compressor.rb#21 +Dalli::Compressor::INFLATE_SLICE_BYTES = T.let(T.unsafe(nil), Integer) + +# DEFLATE can't expand data by more than about 1032:1 +# +# source://dalli//lib/dalli/compressor.rb#17 +Dalli::Compressor::MAX_EXPANSION = T.let(T.unsafe(nil), Integer) + # generic error # # source://dalli//lib/dalli.rb#10 @@ -405,14 +454,37 @@ class Dalli::DalliError < ::RuntimeError; end # Gzip. Gzip adds a checksum to each compressed # entry. # -# source://dalli//lib/dalli/compressor.rb#26 +# source://dalli//lib/dalli/compressor.rb#72 class Dalli::GzipCompressor class << self - # source://dalli//lib/dalli/compressor.rb#27 + # source://dalli//lib/dalli/compressor.rb#73 def compress(data); end - # source://dalli//lib/dalli/compressor.rb#35 - def decompress(data); end + # source://dalli//lib/dalli/compressor.rb#81 + def decompress(data, max_bytes: T.unsafe(nil)); end + end +end + +# A serializer that stores values as JSON and never creates Ruby objects +# from what it reads. +# +# Passing the JSON module itself as the serializer (serializer: JSON) reads +# values back with JSON.load, which on json gem versions before 3.0 honors +# "json_class" and instantiates other classes when the json gem's additions +# are loaded. This serializer reads with JSON.parse, so a value written to +# memcached by someone else can only come back as plain hashes, arrays, +# strings, numbers, booleans and nil. +# +# Dalli::Client.new(servers, serializer: Dalli::JSONSerializer) +# +# source://dalli//lib/dalli/json_serializer.rb#19 +module Dalli::JSONSerializer + class << self + # source://dalli//lib/dalli/json_serializer.rb#20 + def dump(value); end + + # source://dalli//lib/dalli/json_serializer.rb#24 + def load(data); end end end @@ -428,19 +500,19 @@ class Dalli::KeyManager # source://dalli//lib/dalli/key_manager.rb#31 def initialize(client_options); end - # source://dalli//lib/dalli/key_manager.rb#73 + # source://dalli//lib/dalli/key_manager.rb#76 def digest_class; end - # source://dalli//lib/dalli/key_manager.rb#97 + # source://dalli//lib/dalli/key_manager.rb#100 def evaluate_namespace; end # Returns the key with the namespace prefixed, if a namespace is # defined. Otherwise just returns the key # - # source://dalli//lib/dalli/key_manager.rb#61 + # source://dalli//lib/dalli/key_manager.rb#64 def key_with_namespace(key); end - # source://dalli//lib/dalli/key_manager.rb#67 + # source://dalli//lib/dalli/key_manager.rb#70 def key_without_namespace(key); end # Returns the value of attribute namespace. @@ -448,25 +520,27 @@ class Dalli::KeyManager # source://dalli//lib/dalli/key_manager.rb#29 def namespace; end - # source://dalli//lib/dalli/key_manager.rb#89 + # source://dalli//lib/dalli/key_manager.rb#92 def namespace_from_options; end - # source://dalli//lib/dalli/key_manager.rb#77 + # source://dalli//lib/dalli/key_manager.rb#80 def namespace_regexp; end - # source://dalli//lib/dalli/key_manager.rb#113 + # source://dalli//lib/dalli/key_manager.rb#139 def prefix_length(digest); end # Produces a truncated key, if the raw key is longer than the maximum allowed # length. The truncated key is produced by generating a hex digest # of the key, and appending that to a truncated section of the key. # - # source://dalli//lib/dalli/key_manager.rb#108 + # @raise [ArgumentError] + # + # source://dalli//lib/dalli/key_manager.rb#111 def truncated_key(key); end # @raise [ArgumentError] # - # source://dalli//lib/dalli/key_manager.rb#83 + # source://dalli//lib/dalli/key_manager.rb#86 def validate_digest_class_option(opts); end # Validates the key, and transforms as needed. @@ -481,8 +555,17 @@ class Dalli::KeyManager # # @raise [ArgumentError] # - # source://dalli//lib/dalli/key_manager.rb#50 + # source://dalli//lib/dalli/key_manager.rb#53 def validate_key(key); end + + # memcached's key limit applies to the key as sent: in bytes, and, with + # the meta protocol, after base64 encoding when the key needs it. A key + # within the limit in characters can still be too long. With the meta + # protocol memcached rejects it with a CLIENT_ERROR that must never reach + # a pipelined get; with the binary protocol it drops the connection. + # + # source://dalli//lib/dalli/key_manager.rb#132 + def wire_length(key); end end # source://dalli//lib/dalli/key_manager.rb#23 @@ -642,7 +725,7 @@ class Dalli::Protocol::Base # # @return [Boolean] # - # source://dalli//lib/dalli/protocol/base.rb#57 + # source://dalli//lib/dalli/protocol/base.rb#63 def alive?; end # source://forwardable/1.3.3/forwardable.rb#231 @@ -666,12 +749,12 @@ class Dalli::Protocol::Base # source://forwardable/1.3.3/forwardable.rb#231 def hostname(*args, **_arg1, &block); end - # source://dalli//lib/dalli/protocol/base.rb#65 + # source://dalli//lib/dalli/protocol/base.rb#71 def lock!; end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/base.rb#147 + # source://dalli//lib/dalli/protocol/base.rb#156 def multi?; end # source://forwardable/1.3.3/forwardable.rb#231 @@ -689,7 +772,7 @@ class Dalli::Protocol::Base # source://dalli//lib/dalli/protocol/base.rb#17 def options=(_arg0); end - # source://dalli//lib/dalli/protocol/base.rb#139 + # source://dalli//lib/dalli/protocol/base.rb#148 def password; end # Abort current pipelined get. Generally used to signal an external @@ -698,14 +781,14 @@ class Dalli::Protocol::Base # # Returns nothing. # - # source://dalli//lib/dalli/protocol/base.rb#118 + # source://dalli//lib/dalli/protocol/base.rb#124 def pipeline_abort; end # Did the last call to #pipeline_response_setup complete successfully? # # @return [Boolean] # - # source://dalli//lib/dalli/protocol/base.rb#131 + # source://dalli//lib/dalli/protocol/base.rb#140 def pipeline_complete?; end # Attempt to receive and parse as many key/value pairs as possible @@ -715,7 +798,7 @@ class Dalli::Protocol::Base # # Returns a Hash of kv pairs received. # - # source://dalli//lib/dalli/protocol/base.rb#86 + # source://dalli//lib/dalli/protocol/base.rb#92 def pipeline_next_responses; end # Start reading key/value pairs from this connection. This is usually called @@ -724,7 +807,7 @@ class Dalli::Protocol::Base # # Returns nothing. # - # source://dalli//lib/dalli/protocol/base.rb#74 + # source://dalli//lib/dalli/protocol/base.rb#80 def pipeline_response_setup; end # source://forwardable/1.3.3/forwardable.rb#231 @@ -732,23 +815,31 @@ class Dalli::Protocol::Base # @return [Boolean] # - # source://dalli//lib/dalli/protocol/base.rb#147 + # source://dalli//lib/dalli/protocol/base.rb#156 def quiet?; end # source://forwardable/1.3.3/forwardable.rb#231 def raise_down_error(*args, **_arg1, &block); end + # True when this request passed raw: true. A stored value is then + # returned as stored, never deserialized or decompressed. + # + # @return [Boolean] + # + # source://dalli//lib/dalli/protocol/base.rb#32 + def raw_request?(options); end + # source://forwardable/1.3.3/forwardable.rb#231 def reconnect_down_server?(*args, **_arg1, &block); end # Chokepoint method for error handling and ensuring liveness # - # source://dalli//lib/dalli/protocol/base.rb#31 + # source://dalli//lib/dalli/protocol/base.rb#37 def request(opkey, *args); end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/base.rb#143 + # source://dalli//lib/dalli/protocol/base.rb#152 def require_auth?; end # source://forwardable/1.3.3/forwardable.rb#231 @@ -763,13 +854,13 @@ class Dalli::Protocol::Base # source://forwardable/1.3.3/forwardable.rb#231 def socket_type(*args, **_arg1, &block); end - # source://dalli//lib/dalli/protocol/base.rb#67 + # source://dalli//lib/dalli/protocol/base.rb#73 def unlock!; end # source://forwardable/1.3.3/forwardable.rb#231 def up!(*args, **_arg1, &block); end - # source://dalli//lib/dalli/protocol/base.rb#135 + # source://dalli//lib/dalli/protocol/base.rb#144 def username; end # Returns the value of attribute weight. @@ -791,10 +882,10 @@ class Dalli::Protocol::Base # @return [Boolean] # - # source://dalli//lib/dalli/protocol/base.rb#200 + # source://dalli//lib/dalli/protocol/base.rb#209 def cache_nils?(opts); end - # source://dalli//lib/dalli/protocol/base.rb#206 + # source://dalli//lib/dalli/protocol/base.rb#215 def connect; end # The socket connection to the underlying server is initialized as a side @@ -807,46 +898,46 @@ class Dalli::Protocol::Base # Since this is invoked exclusively in verify_state!, we don't need to worry about # thread safety. Using it elsewhere may require revisiting that assumption. # - # source://dalli//lib/dalli/protocol/base.rb#192 + # source://dalli//lib/dalli/protocol/base.rb#201 def ensure_connected!; end # Called after the noop response is received at the end of a set # of pipelined gets # - # source://dalli//lib/dalli/protocol/base.rb#228 + # source://dalli//lib/dalli/protocol/base.rb#237 def finish_pipeline; end - # source://dalli//lib/dalli/protocol/base.rb#239 + # source://dalli//lib/dalli/protocol/base.rb#248 def log_marshal_err(key, err); end - # source://dalli//lib/dalli/protocol/base.rb#244 + # source://dalli//lib/dalli/protocol/base.rb#253 def log_unexpected_err(err); end - # source://dalli//lib/dalli/protocol/base.rb#213 + # source://dalli//lib/dalli/protocol/base.rb#222 def pipelined_get(keys); end - # source://dalli//lib/dalli/protocol/base.rb#235 + # source://dalli//lib/dalli/protocol/base.rb#244 def reconnect_on_pipeline_complete!; end - # source://dalli//lib/dalli/protocol/base.rb#222 + # source://dalli//lib/dalli/protocol/base.rb#231 def response_buffer; end # @raise [Dalli::NotPermittedMultiOpError] # - # source://dalli//lib/dalli/protocol/base.rb#157 + # source://dalli//lib/dalli/protocol/base.rb#166 def verify_allowed_quiet!(opkey); end - # source://dalli//lib/dalli/protocol/base.rb#178 + # source://dalli//lib/dalli/protocol/base.rb#187 def verify_pipelined_state(_opkey); end # Checks to see if we can execute the specified operation. Checks # whether the connection is in use, and whether the command is allowed # - # source://dalli//lib/dalli/protocol/base.rb#167 + # source://dalli//lib/dalli/protocol/base.rb#176 def verify_state(opkey); end end -# source://dalli//lib/dalli/protocol/base.rb#156 +# source://dalli//lib/dalli/protocol/base.rb#165 Dalli::Protocol::Base::ALLOWED_QUIET_OPS = T.let(T.unsafe(nil), Array) # Access point for a single Memcached server, accessed via Memcached's binary @@ -872,7 +963,7 @@ class Dalli::Protocol::Binary < ::Dalli::Protocol::Base # and this isn't it. Maybe rename? Maybe eliminate? # # source://dalli//lib/dalli/protocol/binary.rb#47 - def cas(key); end + def cas(key, options = T.unsafe(nil)); end # Arithmetic Commands # @@ -1085,50 +1176,51 @@ class Dalli::Protocol::Binary::ResponseProcessor # source://dalli//lib/dalli/protocol/binary/response_processor.rb#34 def initialize(io_source, value_marshaller); end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#184 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#187 def auth_response(buf = T.unsafe(nil)); end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#150 + # @raise [Dalli::DalliError] + # + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#220 + def check_body_size!(size); end + + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#153 def consume_all_responses_until_noop; end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#192 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#195 def contains_header?(buf); end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#103 - def data_cas_response; end + # raw: true returns the value as stored; see #get + # + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#106 + def data_cas_response(raw: T.unsafe(nil)); end # Returns the new value for the key, if found and updated # - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#115 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#118 def decr_incr; end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#95 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#97 def delete; end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#138 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#141 def flush; end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#158 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#161 def generic_response; end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#69 - def get(cache_nils: T.unsafe(nil)); end - - # This method returns an array of values used in a pipelined - # getk process. The first value is the number of bytes by - # which to advance the pointer in the buffer. If the - # complete response is found in the buffer, this will - # be the response size. Otherwise it is zero. - # - # The remaining three values in the array are the ResponseHeader, - # key, and value. + # raw: true returns the value as stored, without deserializing or + # decompressing it, whatever flags it was stored with # - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#212 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#71 + def get(cache_nils: T.unsafe(nil), raw: T.unsafe(nil)); end + + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#226 def getk_response_from_buffer(buf); end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#170 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#173 def no_body_response; end # @raise [Dalli::DalliError] @@ -1145,20 +1237,20 @@ class Dalli::Protocol::Binary::ResponseProcessor # source://dalli//lib/dalli/protocol/binary/response_processor.rb#43 def read_response; end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#142 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#145 def reset; end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#198 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#201 def response_header_from_buffer(buf); end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#120 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#123 def stats; end # Response for a storage operation. Returns the cas on success. False # if the value wasn't stored. And raises an error on all other error # codes from memcached. # - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#86 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#88 def storage_response; end # source://dalli//lib/dalli/protocol/binary/response_processor.rb#49 @@ -1166,13 +1258,29 @@ class Dalli::Protocol::Binary::ResponseProcessor # @raise [Dalli::NetworkError] # - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#178 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#181 def validate_auth_format(extra_len, count); end - # source://dalli//lib/dalli/protocol/binary/response_processor.rb#146 + # source://dalli//lib/dalli/protocol/binary/response_processor.rb#149 def version; end end +# This method returns an array of values used in a pipelined +# getk process. The first value is the number of bytes by +# which to advance the pointer in the buffer. If the +# complete response is found in the buffer, this will +# be the response size. Otherwise it is zero. +# +# The remaining three values in the array are the ResponseHeader, +# key, and value. +# +# memcached can't store an item larger than 1 GiB (its -I maximum). A +# pipelined reply claiming an impossible size would otherwise have the +# buffer wait for (and accumulate) that many bytes. +# +# source://dalli//lib/dalli/protocol/binary/response_processor.rb#218 +Dalli::Protocol::Binary::ResponseProcessor::MAX_BODY_BYTES = T.let(T.unsafe(nil), Integer) + # Response codes taken from: # https://github.com/memcached/memcached/wiki/BinaryProtocolRevamped#response-status # @@ -1211,47 +1319,59 @@ class Dalli::Protocol::ConnectionManager # source://dalli//lib/dalli/protocol/connection_manager.rb#32 def initialize(hostname, port, socket_type, client_options); end - # source://dalli//lib/dalli/protocol/connection_manager.rb#145 + # source://dalli//lib/dalli/protocol/connection_manager.rb#160 def abort_request!; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#112 + # @raise [Dalli::DalliError] + # + # source://dalli//lib/dalli/protocol/connection_manager.rb#177 + def check_read_size!(count); end + + # source://dalli//lib/dalli/protocol/connection_manager.rb#115 def close; end # @raise [Dalli::NetworkError] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#222 + # source://dalli//lib/dalli/protocol/connection_manager.rb#250 def close_on_fork; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#106 + # A forked child shares the parent's connection, so it closes only its + # own file descriptor. Closing the TLS socket itself would send + # close_notify and end the parent's TLS session. + # + # source://dalli//lib/dalli/protocol/connection_manager.rb#131 + def close_socket; end + + # source://dalli//lib/dalli/protocol/connection_manager.rb#109 def confirm_in_progress!; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#101 + # source://dalli//lib/dalli/protocol/connection_manager.rb#104 def confirm_ready!; end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#125 + # source://dalli//lib/dalli/protocol/connection_manager.rb#135 def connected?; end # Marks the server instance as down. Updates the down_at state # and raises an Dalli::NetworkError that includes the underlying # error in the message. Calls close to clean up socket state # - # source://dalli//lib/dalli/protocol/connection_manager.rb#84 + # source://dalli//lib/dalli/protocol/connection_manager.rb#85 def down!; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#179 + # source://dalli//lib/dalli/protocol/connection_manager.rb#206 def error_on_request!(err_or_string); end - # source://dalli//lib/dalli/protocol/connection_manager.rb#52 + # source://dalli//lib/dalli/protocol/connection_manager.rb#53 def establish_connection; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#139 + # source://dalli//lib/dalli/protocol/connection_manager.rb#149 def finish_request!; end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#231 + # source://dalli//lib/dalli/protocol/connection_manager.rb#259 def fork_detected?; end # Returns the value of attribute hostname. @@ -1266,22 +1386,22 @@ class Dalli::Protocol::ConnectionManager # source://dalli//lib/dalli/protocol/connection_manager.rb#29 def hostname=(_arg0); end - # source://dalli//lib/dalli/protocol/connection_manager.rb#235 + # source://dalli//lib/dalli/protocol/connection_manager.rb#267 def log_down_detected; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#247 + # source://dalli//lib/dalli/protocol/connection_manager.rb#279 def log_up_detected; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#214 + # source://dalli//lib/dalli/protocol/connection_manager.rb#242 def log_warn_message(err_or_string); end - # source://dalli//lib/dalli/protocol/connection_manager.rb#175 + # source://dalli//lib/dalli/protocol/connection_manager.rb#202 def max_allowed_failures; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#206 + # source://dalli//lib/dalli/protocol/connection_manager.rb#234 def memcached_socket; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#44 + # source://dalli//lib/dalli/protocol/connection_manager.rb#45 def name; end # Returns the value of attribute options. @@ -1310,37 +1430,40 @@ class Dalli::Protocol::ConnectionManager # @raise [Dalli::NetworkError] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#93 + # source://dalli//lib/dalli/protocol/connection_manager.rb#96 def raise_down_error; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#157 + # source://dalli//lib/dalli/protocol/connection_manager.rb#183 def read(count); end - # source://dalli//lib/dalli/protocol/connection_manager.rb#149 + # source://dalli//lib/dalli/protocol/connection_manager.rb#164 def read_line; end # Non-blocking read. Here to support the operation # of the get_multi operation # - # source://dalli//lib/dalli/protocol/connection_manager.rb#171 + # source://dalli//lib/dalli/protocol/connection_manager.rb#198 def read_nonblock; end # @raise [Dalli::NetworkError] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#192 + # source://dalli//lib/dalli/protocol/connection_manager.rb#219 def reconnect!(message); end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#63 + # source://dalli//lib/dalli/protocol/connection_manager.rb#64 def reconnect_down_server?; end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/connection_manager.rb#129 + # source://dalli//lib/dalli/protocol/connection_manager.rb#139 def request_in_progress?; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#198 + # Called on connect. Deliberately leaves @fail_count alone; see + # finish_request!. + # + # source://dalli//lib/dalli/protocol/connection_manager.rb#227 def reset_down_info; end # Returns the value of attribute sock. @@ -1348,7 +1471,7 @@ class Dalli::Protocol::ConnectionManager # source://dalli//lib/dalli/protocol/connection_manager.rb#30 def sock; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#97 + # source://dalli//lib/dalli/protocol/connection_manager.rb#100 def socket_timeout; end # Returns the value of attribute socket_type. @@ -1363,19 +1486,26 @@ class Dalli::Protocol::ConnectionManager # source://dalli//lib/dalli/protocol/connection_manager.rb#29 def socket_type=(_arg0); end - # source://dalli//lib/dalli/protocol/connection_manager.rb#133 + # source://dalli//lib/dalli/protocol/connection_manager.rb#143 def start_request!; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#76 + # source://dalli//lib/dalli/protocol/connection_manager.rb#77 def up!; end - # source://dalli//lib/dalli/protocol/connection_manager.rb#163 + # source://dalli//lib/dalli/protocol/connection_manager.rb#190 def write(bytes); end end # source://dalli//lib/dalli/protocol/connection_manager.rb#16 Dalli::Protocol::ConnectionManager::DEFAULTS = T.let(T.unsafe(nil), Hash) +# memcached can't store an item larger than 1 GiB (its -I maximum), so a +# reply claiming more (or a negative size) is malformed or hostile. +# Reading allocates the full count up front, so check before reading. +# +# source://dalli//lib/dalli/protocol/connection_manager.rb#175 +Dalli::Protocol::ConnectionManager::MAX_READ_BYTES = T.let(T.unsafe(nil), Integer) + # Access point for a single Memcached server, accessed via Memcached's meta # protocol. Contains logic for managing connection state to the server (retries, etc), # formatting requests to the server, and unpacking responses. @@ -1402,7 +1532,7 @@ class Dalli::Protocol::Meta < ::Dalli::Protocol::Base # and this isn't it. Maybe rename? Maybe eliminate? # # source://dalli//lib/dalli/protocol/meta.rb#56 - def cas(key); end + def cas(key, options = T.unsafe(nil)); end # Arithmetic Commands # @@ -1501,12 +1631,22 @@ Dalli::Protocol::Meta::KeyRegularizer::WHITESPACE = T.let(T.unsafe(nil), Regexp) # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#10 class Dalli::Protocol::Meta::RequestFormatter class << self - # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#107 + # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#119 def cas_string(cas); end # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#77 def flush(delay: T.unsafe(nil), quiet: T.unsafe(nil)); end + # Numeric flag values are written straight into the command line, so a + # value that isn't an integer -- e.g. a String carrying CRLF -- would let + # the caller inject further memcached commands (GHSA-6wmv-xq9m-fmp7). + # Converting to Integer means only digits ever reach the wire. Strings + # are parsed as base 10 so "010" means 10, as memcached would read it, + # rather than octal 8. + # + # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#113 + def integer_flag(name, value); end + # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#52 def meta_arithmetic(key:, delta:, initial:, incr: T.unsafe(nil), cas: T.unsafe(nil), ttl: T.unsafe(nil), base64: T.unsafe(nil), quiet: T.unsafe(nil)); end @@ -1530,7 +1670,7 @@ class Dalli::Protocol::Meta::RequestFormatter # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#91 def mode_to_token(mode); end - # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#112 + # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#124 def parse_to_64_bit_int(val, default); end # source://dalli//lib/dalli/protocol/meta/request_formatter.rb#84 @@ -1552,35 +1692,40 @@ class Dalli::Protocol::Meta::ResponseProcessor # source://dalli//lib/dalli/protocol/meta/response_processor.rb#25 def initialize(io_source, value_marshaller); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#175 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#208 def bitflags_from_tokens(tokens); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#189 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#222 def body_len_from_tokens(tokens); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#179 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#212 def cas_from_tokens(tokens); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#107 + # @raise [Dalli::DalliError] + # + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#182 + def check_value_size!(size); end + + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#112 def consume_all_responses_until_mn; end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#160 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#193 def contains_header?(buf); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#72 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#77 def decr_incr; end # @raise [Dalli::DalliError] # - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#168 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#201 def error_on_unexpected!(expected_codes); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#90 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#95 def flush; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#122 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#127 def full_response_from_buffer(tokens, body, resp_size); end # This method returns an array of values used in a pipelined @@ -1592,52 +1737,69 @@ class Dalli::Protocol::Meta::ResponseProcessor # The remaining three values in the array are the ResponseHeader, # key, and value. # - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#137 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#146 def getk_response_from_buffer(buf); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#164 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#197 def header_from_buffer(buf); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#183 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#216 def key_from_tokens(tokens); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#67 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#72 def meta_delete; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#30 - def meta_get_with_value(cache_nils: T.unsafe(nil)); end + # raw: true returns the value as stored, without deserializing or + # decompressing it, whatever flags it was stored with + # + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#32 + def meta_get_with_value(cache_nils: T.unsafe(nil), raw: T.unsafe(nil)); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#38 - def meta_get_with_value_and_cas; end + # raw: true returns the value as stored; see #meta_get_with_value + # + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#42 + def meta_get_with_value_and_cas(raw: T.unsafe(nil)); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#48 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#53 def meta_get_without_value; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#60 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#65 def meta_set_append_prepend; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#53 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#58 def meta_set_with_cas; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#204 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#241 def next_line_to_tokens; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#200 + # A zero-size reply has no body unless it's a VA (a hit on an empty value) + # + # @return [Boolean] + # + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#189 + def no_body?(tokens, body_len); end + + # The size given right after "VA", or 0 when it isn't a number + # + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#178 + def positional_size(tokens); end + + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#237 def read_line; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#96 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#101 def reset; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#80 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#85 def stats; end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#114 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#119 def tokens_from_header_buffer(buf); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#193 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#230 def value_from_tokens(tokens, flag); end - # source://dalli//lib/dalli/protocol/meta/response_processor.rb#102 + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#107 def version; end end @@ -1653,6 +1815,13 @@ Dalli::Protocol::Meta::ResponseProcessor::EX = T.let(T.unsafe(nil), String) # source://dalli//lib/dalli/protocol/meta/response_processor.rb#15 Dalli::Protocol::Meta::ResponseProcessor::HD = T.let(T.unsafe(nil), String) +# memcached can't store an item larger than 1 GiB (its -I maximum). A +# pipelined reply claiming an impossible size would otherwise have the +# buffer wait for (and accumulate) that many bytes. +# +# source://dalli//lib/dalli/protocol/meta/response_processor.rb#175 +Dalli::Protocol::Meta::ResponseProcessor::MAX_VALUE_BYTES = T.let(T.unsafe(nil), Integer) + # source://dalli//lib/dalli/protocol/meta/response_processor.rb#16 Dalli::Protocol::Meta::ResponseProcessor::MN = T.let(T.unsafe(nil), String) @@ -1835,12 +2004,12 @@ Dalli::Protocol::TtlSanitizer::MAX_ACCEPTABLE_EXPIRATION_INTERVAL = T.let(T.unsa class Dalli::Protocol::ValueCompressor # @return [ValueCompressor] a new instance of ValueCompressor # - # source://dalli//lib/dalli/protocol/value_compressor.rb#27 + # source://dalli//lib/dalli/protocol/value_compressor.rb#33 def initialize(client_options); end # @return [Boolean] # - # source://dalli//lib/dalli/protocol/value_compressor.rb#60 + # source://dalli//lib/dalli/protocol/value_compressor.rb#87 def compress_by_default?; end # Checks whether we should apply compression when serializing a value @@ -1851,19 +2020,24 @@ class Dalli::Protocol::ValueCompressor # # @return [Boolean] # - # source://dalli//lib/dalli/protocol/value_compressor.rb#77 + # source://dalli//lib/dalli/protocol/value_compressor.rb#104 def compress_value?(value, req_options); end - # source://dalli//lib/dalli/protocol/value_compressor.rb#68 + # source://dalli//lib/dalli/protocol/value_compressor.rb#95 def compression_min_size; end - # source://dalli//lib/dalli/protocol/value_compressor.rb#64 + # source://dalli//lib/dalli/protocol/value_compressor.rb#91 def compressor; end - # source://dalli//lib/dalli/protocol/value_compressor.rb#49 + # @return [Boolean] + # + # source://dalli//lib/dalli/protocol/value_compressor.rb#74 + def compressor_accepts_max_bytes?; end + + # source://dalli//lib/dalli/protocol/value_compressor.rb#55 def retrieve(value, bitflags); end - # source://dalli//lib/dalli/protocol/value_compressor.rb#41 + # source://dalli//lib/dalli/protocol/value_compressor.rb#47 def store(value, req_options, bitflags); end end @@ -1873,10 +2047,13 @@ Dalli::Protocol::ValueCompressor::DEFAULTS = T.let(T.unsafe(nil), Hash) # https://www.hjp.at/zettel/m/memcached_flags.rxml # Looks like most clients use bit 1 to indicate gzip compression. # -# source://dalli//lib/dalli/protocol/value_compressor.rb#25 +# source://dalli//lib/dalli/protocol/value_compressor.rb#31 Dalli::Protocol::ValueCompressor::FLAG_COMPRESSED = T.let(T.unsafe(nil), Integer) -# source://dalli//lib/dalli/protocol/value_compressor.rb#21 +# source://dalli//lib/dalli/protocol/value_compressor.rb#26 +Dalli::Protocol::ValueCompressor::KEYWORD_PARAMETER_TYPES = T.let(T.unsafe(nil), Array) + +# source://dalli//lib/dalli/protocol/value_compressor.rb#24 Dalli::Protocol::ValueCompressor::OPTIONS = T.let(T.unsafe(nil), Array) # Dalli::Protocol::ValueMarshaller compartmentalizes the logic for marshalling @@ -2331,14 +2508,15 @@ module Process extend ::ConnectionPool::ForkTracker extend ::RedisClient::PIDCache::CoreExt extend ::Dalli::PIDCache::CoreExt - extend ::ActiveSupport::ForkTracker::CoreExt extend ::FFI::ModernForkTracking + extend ::ActiveSupport::ForkTracker::CoreExt + extend ::Bundler::ConnectionPool::ForkTracker end # source://dalli//lib/rack/session/dalli.rb#8 module Rack class << self - # source://rack/3.1.16/lib/rack/version.rb#18 + # source://rack/3.2.7/lib/rack/version.rb#14 def release; end end end diff --git a/sorbet/rbi/shims/bundler_audit.rbi b/sorbet/rbi/shims/bundler_audit.rbi new file mode 100644 index 00000000..46cce3c7 --- /dev/null +++ b/sorbet/rbi/shims/bundler_audit.rbi @@ -0,0 +1,8 @@ +# typed: true + +module Bundler::FetchMetadata; end +module Bundler::MatchMetadata; end + +module Bundler::ConnectionPool + module ForkTracker; end +end diff --git a/test/code_examples/filtering_test.rb b/test/code_examples/filtering_test.rb index ad7ffe08..65dcad38 100644 --- a/test/code_examples/filtering_test.rb +++ b/test/code_examples/filtering_test.rb @@ -70,14 +70,12 @@ def test_other_summaries assert_equal({_filtered: {_class: "Hash", _empty: true}}, formatter.process_values({credentials: {}})[:credentials]) assert_equal({_filtered: {_class: "Array", _empty: true}}, formatter.process_values({tokens: []})[:tokens]) assert_equal({_filtered: {_class: NilClass}}, formatter.process_values({password: nil})[:password]) - # Only the first 10 keys of a filtered Hash, scrubbed like any string summary = formatter.process_values({credentials: (1..12).to_h { |i| [:"key#{i}", i] }}) assert_equal 12, summary.dig(:credentials, :_filtered, :_keys_count) - assert_equal 10, summary.dig(:credentials, :_filtered, :_keys).size email_key = formatter.process_values({credentials: {"linda@example.com" => 1}}) - assert_equal [:"[EMAIL:#{StringScrubber.value_hash("linda@example.com")}]"], email_key.dig(:credentials, :_filtered, :_keys) + assert_equal({_class: Hash, _keys_count: 1}, email_key.dig(:credentials, :_filtered)) # A value Rails' parameter filter has already replaced stays as it is, with no hash assert_equal({email: "[FILTERED]", password: "[FILTERED]"}, formatter.process_values({email: "[FILTERED]", password: "[FILTERED]"})) # At any depth, and in Arrays diff --git a/test/code_examples/guides_test.rb b/test/code_examples/guides_test.rb index 4cbcdf76..b38f1855 100644 --- a/test/code_examples/guides_test.rb +++ b/test/code_examples/guides_test.rb @@ -219,7 +219,7 @@ def test_soc2_filtered_line assert_equal({"_class" => "String", "_hash" => email_hash}, line.dig("email", "_filtered")) # One hash for the address, whatever its case, under a key or inside a string assert_equal "Reset link sent to [EMAIL:#{email_hash}]", line["note"] - assert_equal({"_class" => "Hash", "_keys_count" => 2, "_keys" => %w[username password]}, line.dig("credentials", "_filtered")) + assert_equal({"_class" => "Hash", "_keys_count" => 2}, line.dig("credentials", "_filtered")) assert_equal "https://app.example.com/reset?token=[FILTERED]&step=2", line["reset_url"] assert_equal "[CREDIT_CARD]", line["card"] # Nothing matches a name, so it's logged as it is diff --git a/test/code_examples/log_fields_test.rb b/test/code_examples/log_fields_test.rb index 4227d990..fa6d7168 100644 --- a/test/code_examples/log_fields_test.rb +++ b/test/code_examples/log_fields_test.rb @@ -160,17 +160,24 @@ def test_additional_data assert_equal %w[src evt lvl ts msg account_id rows prog], line.keys - # Its keys replace the line's own fields, except the reserved ones - replaced = LogStruct::Log::Plain.new(message: "Imported", additional_data: {msg: "Replaced"}).serialize - - assert_equal "Replaced", replaced[:msg] - %i[src evt lvl ts].each do |key| + # The line's own fields and the request ID are reserved + %i[src evt lvl ts req_id msg].each do |key| error = assert_raises(ArgumentError) do LogStruct::Log::Plain.new(message: "Imported", additional_data: {key => "x"}).serialize end assert_equal "additional_data attempted to overwrite reserved key: #{key}", error.message end + # So are the type's fields this line doesn't have: an Error's error_message and backtrace + %i[error_message backtrace].each do |key| + error = assert_raises(ArgumentError) do + LogStruct::Log::Error.new(source: Source::App, error_class: RuntimeError, message: "Failed", additional_data: {key => "x"}).serialize + end + + assert_equal "additional_data attempted to overwrite reserved key: #{key}", error.message + end + # Another type's field names are free to use + assert_equal "Invoices", LogStruct::Log::Plain.new(message: "Imported", additional_data: {name: "Invoices"}).serialize[:name] end def test_filtered_summaries @@ -195,7 +202,7 @@ def test_filtered_summaries assert_equal({"_class" => "String", "_hash" => StringScrubber.value_hash("linda.martinez@example.com")}, line.dig("email", "_filtered")) assert_equal({"_class" => "String"}, line.dig("password", "_filtered")) assert_equal({"_class" => "Array", "_count" => 2}, line.dig("api_tokens", "_filtered")) - assert_equal({"_class" => "Hash", "_keys_count" => 2, "_keys" => %w[client_id client_secret]}, line.dig("credentials", "_filtered")) + assert_equal({"_class" => "Hash", "_keys_count" => 2}, line.dig("credentials", "_filtered")) assert_equal "Call Linda on [PHONE]", line["note"] end @@ -415,15 +422,13 @@ def example_lines stored_file = { storage: :file, - file_id: "avatar.png", + file_id: HashUtils.hash_value("avatar.png"), uploader: "AvatarUploader", model: "User", mount_point: "avatar", - version: "", - store_path: "uploads/user/avatar/1873/avatar.png", - extension: "png" + version: "" } - upload = {**stored_file, filename: "avatar.png", mime_type: "image/png", size: 48_213} + upload = {**stored_file, mime_type: "image/png", size: 48_213} example(Log::CarrierWave::Upload) { in_request { rails.info(Log::CarrierWave::Upload.new(**upload, duration_ms: 12.7)) } } # A remote URL fetched into the cache (remote_avatar_url=) example(Log::CarrierWave::Download) { in_request { rails.info(Log::CarrierWave::Download.new(**upload)) } } diff --git a/test/log_struct/bounded_input_test.rb b/test/log_struct/bounded_input_test.rb new file mode 100644 index 00000000..5b826e86 --- /dev/null +++ b/test/log_struct/bounded_input_test.rb @@ -0,0 +1,50 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +module LogStruct + class BoundedInputTest < ActiveSupport::TestCase + test "symbolize_keys converts every key of a Hash within the limit" do + LogStruct.config.limits.max_hash_keys = 3 + + assert_equal({a: 1, b: {"c" => 2}, "3": 3}, BoundedInput.symbolize_keys({"a" => 1, :b => {"c" => 2}, 3 => 3})) + end + + test "symbolize_keys leaves the last slot to the count of the keys it leaves out" do + LogStruct.config.limits.max_hash_keys = 3 + symbolized = BoundedInput.symbolize_keys((1..10).to_h { |i| ["k#{i}", i] }) + + assert_equal({k1: 1, k2: 2, _more_keys: 8}, symbolized) + # The formatter writes it as it is, though the test app's filter_parameters have :key (Rails' default + # list has :_key), which matches _more_keys + assert_equal symbolized, Formatter.new.process_values(symbolized) + end + + test "symbolize_keys counts oversized keys in the slots they leave" do + LogStruct.config.limits.max_hash_keys = 3 + LogStruct.config.limits.max_string_bytes = 8 + long_key = "k" * 9 + + within = BoundedInput.symbolize_keys({long_key => "hidden", "short" => 1}) + past = BoundedInput.symbolize_keys({long_key => "hidden", "short" => 1, "next" => 2, "last" => 3}) + + assert_equal({short: 1, _oversized_keys: 1}, within) + assert_equal({short: 1, _more_keys: 2, _oversized_keys: 1}, past) + [within, past].each do |symbolized| + assert_equal symbolized, Formatter.new.process_values(symbolized) + end + end + + test "symbolize_keys counts the keys of ActionController::Parameters without converting them" do + LogStruct.config.limits.max_hash_keys = 2 + params = ::ActionController::Parameters.new("visible" => "yes", "nested" => {"token" => "secret"}, "last" => 1) + + symbolized = params.stub(:to_unsafe_h, -> { raise "full conversion" }) do + BoundedInput.symbolize_keys(params) + end + + assert_equal({visible: "yes", _more_keys: 2}, symbolized) + end + end +end diff --git a/test/log_struct/configuration_test.rb b/test/log_struct/configuration_test.rb index 2b7acd1d..8a7fe84c 100644 --- a/test/log_struct/configuration_test.rb +++ b/test/log_struct/configuration_test.rb @@ -284,11 +284,14 @@ def test_failing_rails_filter_blocks_filter_the_value LogStruct.config.filters.path_matchers = [] with_rails_filter_parameters([filter]) { LogStruct.merge_rails_filter_parameters! } reported = [] - line = LogStruct.stub(:handle_exception, ->(error, **) { reported << error.message }) do + line = LogStruct.stub(:handle_exception, ->(error, source:, context:) { reported << [error.message, context] }) do Formatter.new.call(:info, Time.now, nil, {private_data: "synthetic-secret", public_data: "visible"}) end - assert_equal ["filter failed"], reported + assert_equal ["RuntimeError: filter_parameters entry failed"], reported.map(&:first) + # The key can be the secret + assert_equal [%i[filter filter_label]], reported.map { |(_message, context)| context.keys } + assert_not_includes reported.to_s, "private_data" assert_not_includes line, "synthetic-secret" assert_equal "visible", JSON.parse(line)["public_data"] end @@ -309,6 +312,37 @@ def test_rails_filter_blocks_with_three_arguments_get_the_parameters assert_includes phone_line, "phone-1" end + def test_rails_filter_blocks_with_three_arguments_get_the_parameters_the_line_leaves_out + number_filter = ->(key, _value, params) { key == "number" && params[:kinds].include?("card") } + + with_rails_filter_parameters([number_filter]) do + LogStruct.merge_rails_filter_parameters! + end + # The size limits only cut what's written: "card" is past max_array_items + kinds = Array.new(LogStruct.config.limits.max_array_items, "phone") << "card" + + line = Formatter.new.call(::Logger::INFO, Time.now, nil, {msg: "Saved", params: {kinds: kinds, number: "card-1"}}) + + assert_not_includes line, "card-1" + end + + def test_rails_filter_blocks_with_three_arguments_leave_lines_with_long_backtraces_readable + with_rails_filter_parameters([->(key, _value, _params) { key == "pin" }]) do + LogStruct.merge_rails_filter_parameters! + end + error = Log::Error.new( + source: Source::App, + error_class: RuntimeError, + message: "Failed", + backtrace: Array.new(30) { |line| "app/models/user.rb:#{line + 1}:in 'save'" } + ) + + line = JSON.parse(Formatter.new.call(::Logger::ERROR, Time.now, nil, error)) + + assert_equal %w[app error Failed RuntimeError], line.values_at("src", "evt", "msg", "error_class") + assert_equal LogStruct.config.limits.max_backtrace_lines + 1, line["backtrace"].size + end + def test_rails_dotted_filters_match_nested_keys_by_their_path with_rails_filter_parameters(["profile.nickname", /\Ateam\.motto\z/]) do LogStruct.merge_rails_filter_parameters! diff --git a/test/log_struct/formatter_test.rb b/test/log_struct/formatter_test.rb index e1987ab8..752b40fc 100644 --- a/test/log_struct/formatter_test.rb +++ b/test/log_struct/formatter_test.rb @@ -5,6 +5,26 @@ module LogStruct class FormatterTest < ActiveSupport::TestCase + class BrokenJsonValue + def as_json(*) + raise "secret-in-error-message" + end + + def inspect + "secret-in-inspect" + end + + def to_s + "secret-in-string" + end + end + + class EmailKey + def to_s + "person@example.com" + end + end + def setup @formatter = Formatter.new @severity = ::Logger::INFO @@ -96,13 +116,13 @@ def to_s # Create an instance of it error_obj = my_gid_class.new - # Need to stub LogStruct.handle_exception to avoid actually reporting an error - LogStruct.stub(:handle_exception, nil) do - # Test the error handling path in process_values + reported = [] + LogStruct.stub(:handle_exception, ->(error, **) { reported << error.message }) do result = @formatter.process_values(error_obj) assert_equal "[GLOBALID_ERROR]", result end + assert_equal ["StandardError: GlobalID formatting failed"], reported end def test_tagged_logging_support @@ -161,11 +181,19 @@ def test_scrubs_keys_and_symbol_values assert_match(/\A\[EMAIL:[a-f0-9]+\]\z/, result.dig(key, :status)) end - def test_scrubs_the_keys_of_filtered_hashes + def test_scrubs_object_hash_keys + line = @formatter.call(@severity, @time, @progname, {data: {EmailKey.new => "visible"}}) + result = JSON.parse(line)["data"] + + assert_equal ["[EMAIL:#{StringScrubber.value_hash("person@example.com")}]"], result.keys + refute_includes line, "person@example.com" + end + + def test_omits_the_keys_of_filtered_hashes log_line = @formatter.call(@severity, @time, @progname, {msg: "tokens", tokens: {"ada@example.com" => "t1"}}) assert_not_includes log_line, "ada@example.com" - assert_match(/\[EMAIL:[a-f0-9]+\]/, JSON.parse(log_line).dig("tokens", "_filtered", "_keys", 0)) + assert_equal({"_class" => "Hash", "_keys_count" => 1}, JSON.parse(log_line).dig("tokens", "_filtered")) end def test_process_values_formats_hashes_recursively @@ -332,7 +360,7 @@ def test_filtered_summaries_leave_out_how_big_the_values_are result = JSON.parse(log_line) assert_equal({"_class" => "Array", "_count" => 1}, result.dig("password", "_filtered")) - assert_equal({"_class" => "Hash", "_keys_count" => 1, "_keys" => ["value"]}, result.dig("creds", "_filtered")) + assert_equal({"_class" => "Hash", "_keys_count" => 1}, result.dig("creds", "_filtered")) end def test_an_email_has_one_hash_under_a_key_and_in_text_however_its_written @@ -382,17 +410,14 @@ def initialize end def test_does_not_log_raw_values_when_processing_fails - exploding_hash = Class.new(Hash) do - def each(*) - raise "boom" - end - end.new - exploding_hash[:password] = "hunter2" + exploding_hash = {password: "hunter2"} LogStruct.stub(:handle_exception, nil) do - result = @formatter.process_values(exploding_hash) + result = exploding_hash.stub(:each_pair, -> { raise "boom" }) do + @formatter.process_values(exploding_hash) + end - assert_equal({_filtered: {_class: exploding_hash.class.name, _keys_count: 1}}, result) + assert_equal({_filtered: {_class: "Hash", _keys_count: 1}}, result) end end @@ -606,6 +631,43 @@ def test_writes_the_first_keys_of_a_large_hash assert_equal({"k1" => 1, "k2" => 2, "k3" => 3, "_more_keys" => 7}, result["data"]) end + def test_oversized_keys_and_their_values_are_omitted + LogStruct.config.limits.max_string_bytes = 32 + key = "secret-in-key-#{"x" * 10_000}" + + line = @formatter.call(@severity, @time, @progname, {data: {key => "secret-in-value", "safe" => "visible"}}) + parsed = JSON.parse(line) + + assert_equal({"safe" => "visible", "_oversized_keys" => 1}, parsed["data"]) + assert_operator line.bytesize, :<, 500 + refute_includes line, "secret-in-key" + refute_includes line, "secret-in-value" + end + + def test_action_controller_parameters_obey_hash_limits_without_full_conversion + LogStruct.config.limits.max_hash_keys = 1 + params = ::ActionController::Parameters.new("visible" => "yes", "ignored" => {"token" => "secret-in-omitted-value"}) + + params.stub(:to_unsafe_h, -> { raise "full conversion" }) do + result = @formatter.process_values(params) + + assert_equal({visible: "yes", _more_keys: 1}, result) + end + end + + def test_serialization_failure_does_not_inspect_or_stringify_the_object + reported = [] + line = LogStruct.stub(:handle_exception, ->(error, source:, context:) { reported << [error.message, source, context] }) do + @formatter.call(@severity, @time, @progname, BrokenJsonValue.new) + end + + assert_equal "[SERIALIZATION_ERROR]", JSON.parse(line)["msg"] + refute_includes line, "secret-in-inspect" + refute_includes line, "secret-in-string" + assert_equal ["RuntimeError: value serialization failed"], reported.map(&:first) + assert_equal [{object_class: "LogStruct::FormatterTest::BrokenJsonValue"}], reported.map(&:last) + end + class KeyProbe attr_reader :visits @@ -694,11 +756,14 @@ def as_json(*) end end.new - log_line = LogStruct.stub(:is_production?, true) do + reported = [] + log_line = LogStruct.stub(:handle_exception, ->(error, **) { reported << error }) do @formatter.call(@severity, @time, @progname, {msg: "Loop", value: looping}) end assert_equal "Loop", JSON.parse(log_line)["msg"] + assert_equal ["SystemStackError: value formatting failed"], reported.map(&:message) + assert_match(/formatter_test\.rb:\d+:in .as_json'/, reported.first.backtrace.first) end private diff --git a/test/log_struct/hash_utils_test.rb b/test/log_struct/hash_utils_test.rb index b7feefdd..06788fd9 100644 --- a/test/log_struct/hash_utils_test.rb +++ b/test/log_struct/hash_utils_test.rb @@ -37,5 +37,27 @@ class HashUtilsTest < ActiveSupport::TestCase test "hashes are 16 characters by default" do assert_equal 16, ConfigStruct::Filters.new.hash_length end + + test "hashing a long value uses a bounded input" do + LogStruct.config.filters.hash_salt = "salt:" + LogStruct.config.limits.max_string_bytes = 8 + value = "x" * 10_000 + expected = OpenSSL::HMAC.hexdigest("SHA256", "salt:", "xxxxxxxx\0#{value.bytesize}")[0...LogStruct.config.filters.hash_length] + + assert_equal expected, HashUtils.hash_value(value) + end + + test "fallback key is random and private to the process" do + LogStruct.configure { |c| c.filters.hash_salt = nil } + HashUtils.reset! + + ::Rails.stub(:application, nil) do + key = HashUtils.hash_key + + assert_equal 32, key.bytesize + refute_equal "l0g5t0p", key + assert_equal key, HashUtils.hash_key + end + end end end diff --git a/test/log_struct/integrations/ahoy_test.rb b/test/log_struct/integrations/ahoy_test.rb index cee0b839..408052f4 100644 --- a/test/log_struct/integrations/ahoy_test.rb +++ b/test/log_struct/integrations/ahoy_test.rb @@ -72,6 +72,22 @@ def track(name, properties = {}, _options = {}) assert_equal({_class: "String"}, json["properties"]) end + test "parameter properties are bounded without converting the full input" do + LogStruct.config.limits.max_hash_keys = 2 + params = ::ActionController::Parameters.new("visible" => "yes", "ignored" => {"password" => "secret"}, "also_ignored" => 1) + + params.stub(:to_unsafe_h, -> { raise "full conversion" }) do + assert_equal({visible: "yes", _more_keys: 2}, Integrations::Ahoy.properties_for_log(params)) + end + end + + test "the line counts every property left out" do + LogStruct.config.limits.max_hash_keys = 3 + properties = Integrations::Ahoy.properties_for_log((1..10).to_h { |i| ["k#{i}", i] }) + + assert_equal({k1: 1, k2: 2, _more_keys: 8}, Formatter.new.process_values(properties)) + end + test "Ahoy gets the arguments as they were given, so its defaults still apply" do LogStruct.stub(:info, ->(_log) {}) do assert_equal [:ok, "signup", {}], ::Ahoy::Tracker.new.track("signup") diff --git a/test/log_struct/integrations/carrierwave_test.rb b/test/log_struct/integrations/carrierwave_test.rb index 5ba8e582..e996c39c 100644 --- a/test/log_struct/integrations/carrierwave_test.rb +++ b/test/log_struct/integrations/carrierwave_test.rb @@ -40,8 +40,10 @@ class AvatarUploader < CarrierWave::Uploader::Base upload = logged_lines.find { |line| line["src"] == "carrierwave" } - assert_equal ["upload", "avatar.png", "avatar.png", 9, "file"], - upload&.values_at("evt", "file_id", "filename", "size", "storage") + assert_equal ["upload", LogStruct::HashUtils.hash_value("avatar.png"), 9, "file"], + upload&.values_at("evt", "file_id", "size", "storage") + refute_includes upload.to_s, "avatar.png" + refute upload.key?("store_path") assert_kind_of Float, upload["duration_ms"] end @@ -66,7 +68,9 @@ class AvatarUploader < CarrierWave::Uploader::Base delete = logged_lines.find { |line| line["src"] == "carrierwave" && line["evt"] == "delete" } - assert_equal ["avatar.png", "file", "CarrierWaveIntegrationTest::AvatarUploader"], delete&.values_at("file_id", "storage", "uploader") + assert_equal [LogStruct::HashUtils.hash_value("avatar.png"), "file", "CarrierWaveIntegrationTest::AvatarUploader"], + delete&.values_at("file_id", "storage", "uploader") + refute_includes delete.to_s, "avatar.png" refute_path_exists File.join(@root, "uploads", "avatar.png") end @@ -80,7 +84,9 @@ class AvatarUploader < CarrierWave::Uploader::Base download = logged_lines.find { |line| line["src"] == "carrierwave" && line["evt"] == "download" } - assert_equal ["remote.png", "remote.png", "image/png", 12], download&.values_at("file_id", "filename", "mime_type", "size") + assert_equal [LogStruct::HashUtils.hash_value("remote.png"), "image/png", 12], + download&.values_at("file_id", "mime_type", "size") + refute_includes download.to_s, "remote.png" refute_includes download.to_s, "secret" end diff --git a/test/log_struct/integrations/lograge_formatter_test.rb b/test/log_struct/integrations/lograge_formatter_test.rb index ab34da3b..3ab90c30 100644 --- a/test/log_struct/integrations/lograge_formatter_test.rb +++ b/test/log_struct/integrations/lograge_formatter_test.rb @@ -39,10 +39,10 @@ def test_normalizes_symbol_and_mixed_types assert_in_delta 1.23, log.view assert_in_delta 0.45, log.database - # params should be symbolized deeply + # Top-level param names are normalized for the struct; nested values stay untouched until formatting. refute_nil log.params assert_equal "bar", log.params[:foo] - assert_equal "x", log.params[:nested][:inner] + assert_equal "x", log.params[:nested]["inner"] # ensure serialize works without type errors json_hash = log.serialize diff --git a/test/log_struct/param_filters_test.rb b/test/log_struct/param_filters_test.rb index 4cd07e4c..b31b6dd8 100644 --- a/test/log_struct/param_filters_test.rb +++ b/test/log_struct/param_filters_test.rb @@ -37,6 +37,23 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_failing_matcher_filters_value end + test "a matcher failure does not report its value or key" do + LogStruct.config.filters.filter_matchers << ConfigStruct::FilterMatcher.new( + callable: ->(key, value) { raise "#{key}: #{value}" }, + label: "failed matcher" + ) + reported = [] + + line = LogStruct.stub(:handle_exception, ->(error, source:, context:) { reported << [error.message, source, context] }) do + Formatter.new.call(:info, Time.now, nil, {"sensitive_field" => "secret-in-value"}) + end + + refute_includes line, "secret-in-value" + refute_includes reported.to_s, "sensitive_field" + refute_includes reported.to_s, "secret-in-value" + assert_equal ["RuntimeError: filter matcher failed"], reported.map(&:first) + end + test "a failing path matcher filters the value when its error is handled" do LogStruct.config.filters.path_matchers << ConfigStruct::PathFilterMatcher.new( callable: ->(path, _value, _root) { @@ -49,13 +66,42 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_failing_matcher_filters_value end + test "path matcher roots keep what the size limits leave out of the line" do + LogStruct.config.limits.max_hash_keys = 1 + LogStruct.config.limits.max_array_items = 1 + LogStruct.config.limits.max_string_bytes = 8 + long_key = "k" * 100 + root = ConfigStruct::LazyRoot.new({"first" => [1, 2, 3], "second" => {long_key => "v"}}) + + assert_equal({"first" => [1, 2, 3], "second" => {long_key => "v"}}, root.value) + assert_equal "v", root.value[:second][long_key] + end + + test "path matcher roots fail closed when the parameters are too large to copy" do + matcher = ConfigStruct::PathFilterMatcher.new(callable: ->(_path, _value, _root) { false }, label: "root matcher") + cyclic = {"name" => "Ada"} + cyclic["self"] = cyclic + too_deep = (BoundedInput::MAX_DEPTH + 1).times.reduce({}) { |inner, _level| {"nested" => inner} } + roots = [ + ConfigStruct::LazyRoot.new({"items" => Array.new(BoundedInput::MAX_NODES, 1)}), + ConfigStruct::LazyRoot.new(too_deep), + ConfigStruct::LazyRoot.new(cyclic) + ] + + roots.each do |root| + assert_same ConfigStruct::LazyRoot::UNAVAILABLE, root.value + assert matcher.matches?(["name"], "Ada", root) + end + refute matcher.matches?(["name"], "Ada", ConfigStruct::LazyRoot.new({"items" => Array.new(BoundedInput::MAX_NODES - 2, 1)})) + end + private def assert_failing_matcher_filters_value reported = [] line = LogStruct.stub(:handle_exception, ->(error, **) { reported << error.message }) do Formatter.new.call(:info, Time.now, nil, {private_data: "synthetic-secret", public_data: "visible"}) end - assert_equal ["matcher failed"], reported + assert_equal ["RuntimeError: filter matcher failed"], reported assert_not_includes line, "synthetic-secret" assert_equal({"_filtered" => {"_class" => "String"}}, JSON.parse(line)["private_data"]) assert_equal "visible", JSON.parse(line)["public_data"] @@ -105,7 +151,7 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_equal 1, reported.size report = T.must(reported.first) - assert_equal "matcher failed", report.first + assert_equal "RuntimeError: filter matcher failed", report.first assert_equal({_filtered: {_class: String}}, report.last[:matcher]) assert_nil Thread.current[:logstruct_formatting_internal_error] end @@ -124,9 +170,9 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_equal({_class: String}, s) end - test "summarize_hash has the keys, not the size" do - assert_equal({_class: Hash, _keys_count: 2, _keys: [:a, :b]}, ParamFilters.summarize_hash({a: 1, b: 2})) - assert_equal({_class: Hash, _keys_count: 1, _keys: [:value]}, ParamFilters.summarize_hash({value: "secret"})) + test "summarize_hash does not expose child keys" do + assert_equal({_class: Hash, _keys_count: 2}, ParamFilters.summarize_hash({a: 1, b: 2})) + assert_equal({_class: Hash, _keys_count: 1}, ParamFilters.summarize_hash({"secret-as-key" => "hidden"})) end test "summarize_hash handles non symbol keys" do @@ -134,7 +180,7 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_equal Hash, summary[:_class] assert_equal 1, summary[:_keys_count] - assert_equal ["1"], summary[:_keys] + refute summary.key?(:_keys) end test "summarize_array reports the count, not the size, and handles empty" do diff --git a/tools/codegen/templates/sorbet/_event_class_body.rb.erb b/tools/codegen/templates/sorbet/_event_class_body.rb.erb index e7f435cd..62b5d438 100644 --- a/tools/codegen/templates/sorbet/_event_class_body.rb.erb +++ b/tools/codegen/templates/sorbet/_event_class_body.rb.erb @@ -26,6 +26,16 @@ extend T::Sig include LogStruct::Log::Interfaces::AdditionalDataField const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil include LogStruct::Log::Shared::MergeAdditionalDataFields + + FIELD_KEYS = T.let( + [<%= (base_fields + event_only(event)).map { |f| "LogField::#{f.enum_name}" }.join(", ") %>].to_set(&:serialize).freeze, + T::Set[Symbol] + ) + + sig { override.returns(T::Set[Symbol]) } + def field_keys + FIELD_KEYS + end <% end -%> <% if add_request_fields -%>