Close the filtering gaps: keys by their words, HMAC hashes, card numbers, URL secrets, request IDs and SQL binds

.cspell/project-words.txt

@@ -78,3 +78,8 @@ wordlists
 xorshift
 yardoc
 tsort
+HMAC
+otps
+crypted
+jwts
+Luhn

README.md

@@ -14,7 +14,7 @@ We support all your other favorite gems too, like Sidekiq, Sentry, and Shrine. (
 - Error handling and reporting
 - Metadata collection for rich context
 - Lograge integration for structured request logging
-- Sensitive data scrubbing for strings (inspired by the Logstop gem)
+- Sensitive data filtering by key name, and scrubbing for strings: emails, card numbers, URL passwords and tokens (inspired by the Logstop gem)
 - Host authorization logging for security violations
 - Rack middleware for enhanced error logging
 - Type checking with Sorbet and RBS annotations
@@ -52,10 +52,10 @@ Once initialized (and enabled), the gem automatically includes its modules into
 - `ActiveSupport::TaggedLogging` is patched to support both Hashes and Strings (only when LogStruct is enabled)
 - `ActionMailer::Base` includes error handling and event logging modules
 - We configure `Lograge` for request logging
-- Lograge request logs include request metadata (request_id, source_ip, user_agent, referer, host, content_type, accept) and custom fields from `lograge_custom_options`
+- Lograge request logs include request metadata (request_id, source_ip, user_agent, referer without its query string, host, content_type, accept) and custom fields from `lograge_custom_options`
 - A Rack middleware is inserted to catch and log errors, including security violations (IP spoofing, CSRF, blocked hosts, etc.)
 - Structured logging is set up for ActiveJob, Sidekiq, Shrine, etc.
-- LogStruct also filters the keys in Rails' `config.filter_parameters`, matched the way Rails matches them (`:passw` filters `user_password`), so the same keys are filtered in every log line. Rails' own list is left in place, so `request.filtered_parameters`, `ActiveRecord::Base.filter_attributes` and other Rails features keep filtering, and values Rails has already replaced are logged as `[FILTERED]`. Configure extra sensitive keys via `LogStruct.config.filters`.
+- LogStruct also filters the keys in Rails' `config.filter_parameters`, matched the way Rails matches them (`:passw` filters `user_password`, `"card.number"` filters a nested key by its path, and filter blocks get the parameters), so the same keys are filtered in every log line. Rails' own list is left in place, so `request.filtered_parameters`, `ActiveRecord::Base.filter_attributes` and other Rails features keep filtering, and values Rails has already replaced are logged as `[FILTERED]`. Configure extra sensitive keys via `LogStruct.config.filters`.
 - When `RAILS_LOG_TO_STDOUT` is set, we log to STDOUT only. Otherwise, we log to STDOUT by default without adding a file appender to avoid duplicate logs.
 
 ### Default behavior by process type

docs/app/docs/soc2-compliance/page.tsx

@@ -75,10 +75,10 @@ export default function SOC2CompliancePage() {
       </p>
 
       <ul className="list-disc list-inside space-y-2 text-neutral-600 dark:text-neutral-300 mb-6">
-        <li>Passwords, tokens, and API keys</li>
-        <li>Credit card numbers and CVV codes</li>
-        <li>Social security numbers and other PII</li>
-        <li>Email addresses and phone numbers</li>
+        <li>Passwords, tokens, API keys, cookies and sessions, filtered by their key names</li>
+        <li>Credit card numbers, CVV codes and social security numbers</li>
+        <li>Email addresses, phone numbers and passwords in URLs, scrubbed out of every string</li>
+        <li>Sensitive query params in URLs, such as reset tokens and OAuth codes</li>
         <li>Any custom sensitive fields you configure</li>
       </ul>
 
@@ -92,16 +92,12 @@ export default function SOC2CompliancePage() {
 
       <CodeBlock language="json">
         {`{
-  "email": {
-    "_filtered": {
-      "_class": "String",
-      "_hash": "a1b2c3d4e5f6"  // Deterministic hash for correlation
-    }
-  },
-  "payment_data": {
+  "email": "[EMAIL:a1b2c3d4e5f6a7b8]",  // The same hash for the same address
+  "credentials": {
     "_filtered": {
       "_class": "Hash",
-      "_size": 3  // Size helps understand data volume
+      "_keys_count": 2,  // How much was filtered, without the values
+      "_keys": ["username", "password"]
     }
   }
 }`}
@@ -111,13 +107,14 @@ export default function SOC2CompliancePage() {
         Deterministic Email Hashes
       </HeadingWithAnchor>
       <p className="text-neutral-600 dark:text-neutral-300 mb-4">
-        For email addresses, LogStruct generates deterministic hashes that allow you to:
+        For email addresses, LogStruct logs a deterministic hash (an HMAC keyed with your app&apos;s{' '}
+        <code>secret_key_base</code>, or with <code>hash_salt</code>) that allows you to:
       </p>
 
       <ul className="list-disc list-inside space-y-2 text-neutral-600 dark:text-neutral-300 mb-6">
         <li>
           <strong>Trace requests across services</strong> - The same email always produces the same
-          hash
+          hash in one app, and in every app that sets the same <code>hash_salt</code>
         </li>
         <li>
           <strong>Correlate user actions</strong> - Follow a user&apos;s journey without exposing
@@ -141,10 +138,15 @@ export default function SOC2CompliancePage() {
           <strong>_class</strong> - Shows whether it was a String, Hash, Array, etc.
         </li>
         <li>
-          <strong>_size</strong> - For collections, shows how many items were filtered
+          <strong>_keys_count</strong> and <strong>_keys</strong> - For Hashes, how many keys were
+          filtered and the first 10 of them
+        </li>
+        <li>
+          <strong>_count</strong> - For Arrays, how many items were filtered
         </li>
         <li>
-          <strong>_hash</strong> - Deterministic hash for correlation (email addresses only)
+          <strong>_hash</strong> - Deterministic hash for correlation (email addresses, and any keys
+          you add to <code>filter_keys_with_hashes</code>)
         </li>
       </ul>
 
@@ -182,7 +184,8 @@ export default function SOC2CompliancePage() {
           entering logs
         </li>
         <li>
-          <strong>Compliance by default</strong> - No manual scrubbing or post-processing required
+          <strong>On by default</strong> - Filtering and scrubbing apply to every line without any
+          setup
         </li>
         <li>
           <strong>Audit evidence</strong> - Clear filtering metadata demonstrates data protection
@@ -195,11 +198,13 @@ export default function SOC2CompliancePage() {
       </HeadingWithAnchor>
       <ul className="list-disc list-inside space-y-2 text-neutral-600 dark:text-neutral-300 mb-6">
         <li>
-          <strong>PII protection</strong> - Email addresses and other PII are automatically filtered
+          <strong>PII protection</strong> - Email addresses, phone numbers and other PII are
+          filtered or scrubbed automatically
         </li>
         <li>
-          <strong>Retention compliance</strong> - Filtered logs can be retained longer without
-          privacy concerns
+          <strong>Retention</strong> - Filtered logs hold far less personal data. Hashed emails are
+          still pseudonymous personal data, and scrubbing works by pattern, so keep your usual
+          retention limits
         </li>
         <li>
           <strong>Data minimization</strong> - Only necessary debugging metadata is preserved

docs/lib/log-generation/__tests__/random-data-generator.test.ts

@@ -56,17 +56,17 @@ describe('RandomDataGenerator', () => {
   test('should generate filtered email addresses', () => {
     const filteredEmail = generator.randomFilteredEmailString();
 
-    expect(filteredEmail).toMatch(/^\[EMAIL:[a-f0-9]{6}\]$/);
+    expect(filteredEmail).toMatch(/^\[EMAIL:[a-f0-9]{16}\]$/);
   });
 
   test('should generate filtered email address objects', () => {
     const filteredEmailObj = generator.randomFilteredEmailObject();
 
     expect(filteredEmailObj).toHaveProperty('_filtered');
     expect(filteredEmailObj._filtered).toHaveProperty('_class', 'String');
-    expect(filteredEmailObj._filtered).toHaveProperty('_bytes');
+    expect(filteredEmailObj._filtered).not.toHaveProperty('_bytes');
     expect(filteredEmailObj._filtered).toHaveProperty('_hash');
-    expect(filteredEmailObj._filtered._hash).toMatch(/^[a-f0-9]{6}$/);
+    expect(filteredEmailObj._filtered._hash).toMatch(/^[a-f0-9]{16}$/);
   });
 
   test('should generate filtered hash objects', () => {
@@ -76,7 +76,8 @@ describe('RandomDataGenerator', () => {
     expect(filteredHash._filtered).toHaveProperty('_class', 'Hash');
     expect(filteredHash._filtered).toHaveProperty('_keys_count');
     expect(filteredHash._filtered).toHaveProperty('_keys');
-    expect(filteredHash._filtered).toHaveProperty('_bytes');
+    // Sensitive keys hide the size
+    expect(filteredHash._filtered).not.toHaveProperty('_bytes');
 
     // Check that keys include password-related fields
     expect(filteredHash._filtered._keys).toContain('password');
@@ -87,7 +88,7 @@ describe('RandomDataGenerator', () => {
 
     expect(filteredArray).toHaveProperty('_filtered');
     expect(filteredArray._filtered).toHaveProperty('_class', 'Array');
-    expect(filteredArray._filtered).toHaveProperty('_length', 5);
+    expect(filteredArray._filtered).toHaveProperty('_count', 5);
     expect(filteredArray._filtered).toHaveProperty('_bytes');
   });
 

docs/lib/log-generation/log-generator.ts

@@ -250,6 +250,8 @@ export class LogGenerator extends RandomDataGenerator {
   }
 
   private applyFilters(log: JsonRecord): JsonRecord {
+    // Like LogStruct's defaults: keys filtered by name, and emails and phone numbers scrubbed
+    // out of every string
     const SENSITIVE_KEYS = new Set<string>([
       'password',
       'password_confirmation',
@@ -259,34 +261,28 @@ export class LogGenerator extends RandomDataGenerator {
       'api_key',
       'authorization',
       'credit_card',
-      'phone',
       'ssn',
-      'tax_id',
-      'email',
     ]);
-    const emailRe = /^[^\s@]+@[^\s@]+\.[^\s@]+$/i;
+    const emailRe = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi;
+    const phoneRe = /\b\d{3}[\s-]\d{3}[\s-]\d{4}\b/g;
 
     const filterValue = (key: string, value: any): any => {
       const k = key.toLowerCase();
-      if (typeof value === 'string') {
-        if (SENSITIVE_KEYS.has(k)) {
-          if (k.includes('password')) return this.randomPassword(true);
-          if (k.includes('credit')) return this.randomCreditCard(true);
-          if (k.includes('phone')) return this.randomPhone(true);
-          if (k.includes('email')) return this.randomFilteredEmailString();
-          return '[FILTERED]';
-        }
-        if (emailRe.test(value)) return this.randomFilteredEmailString();
+      if (SENSITIVE_KEYS.has(k)) {
+        if (typeof value === 'string') return this.filteredString();
+        if (Array.isArray(value)) return this.filteredArray(value.length);
+        if (value && typeof value === 'object') return this.filteredHash('password');
         return value;
       }
+      if (typeof value === 'string') {
+        return value
+          .replace(emailRe, () => this.randomFilteredEmailString())
+          .replace(phoneRe, '[PHONE]');
+      }
       if (Array.isArray(value)) {
-        if (SENSITIVE_KEYS.has(k)) return this.filteredArray(value.length);
         return value.map((v) => filterValue(key, v));
       }
       if (value && typeof value === 'object') {
-        if (['data', 'payload', 'attributes', 'request_body'].includes(k)) {
-          return this.filteredHash('json');
-        }
         return walk(value as Record<string, any>);
       }
       return value;

docs/lib/log-generation/random-data-generator.ts

@@ -6,9 +6,9 @@ import { SampleData } from './sample-data';
 export interface FilteredValue {
   _filtered: {
     _class: string;
-    _bytes: number;
+    _bytes?: number;
     _hash?: string;
-    _length?: number;
+    _count?: number;
     _keys?: string[];
     _keys_count?: number;
   };
@@ -110,7 +110,7 @@ export class RandomDataGenerator {
    * Generate a filtered email address string (for sensitive data)
    */
   randomFilteredEmailString(): string {
-    const emailHash = this.randomHex(6);
+    const emailHash = this.randomHex(16);
     // Return the simple [EMAIL:hash] format
     return `[EMAIL:${emailHash}]`;
   }
@@ -119,17 +119,23 @@ export class RandomDataGenerator {
    * Generate a filtered email address as an object (for sensitive data)
    */
   randomFilteredEmailObject(): FilteredValue {
-    const emailHash = this.randomHex(6);
+    const emailHash = this.randomHex(16);
     // Return the object format with _filtered property
     return {
       _filtered: {
         _class: 'String',
-        _bytes: 24 + this.randomInt(0, 10), // Randomize byte size a bit
         _hash: emailHash,
       },
     };
   }
 
+  /**
+   * A filtered String: LogStruct logs only its class, not its length
+   */
+  filteredString(): FilteredValue {
+    return { _filtered: { _class: 'String' } };
+  }
+
   /**
    * Generate a filtered password
    */
@@ -187,12 +193,13 @@ export class RandomDataGenerator {
         keys = ['data', 'payload', 'attributes', 'request_body'];
     }
 
+    // The size is left out when any of the keys are sensitive
     return {
       _filtered: {
         _class: 'Hash',
         _keys_count: keys.length,
         _keys: keys,
-        _bytes: this.randomInt(20, 500),
+        ...(sensitivity === 'json' ? { _bytes: this.randomInt(20, 500) } : {}),
       },
     };
   }
@@ -208,7 +215,7 @@ export class RandomDataGenerator {
     return {
       _filtered: {
         _class: 'Array',
-        _length: itemCount,
+        _count: itemCount,
         _bytes: itemCount * this.randomInt(10, 50),
       },
     };