Add split chunks mode, add http charts, remove sorting

.github/workflows/golangci-lint.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-          go-version: stable
+          go-version: oldstable
       - uses: actions/checkout@v4
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v9.2.0

cmd/catp/catp/app.go

@@ -14,7 +14,6 @@
 	"path/filepath"
 	"runtime/pprof"
 	"slices"
-	"sort"
 	"strings"
 	"sync/atomic"
 	"syscall"
@@ -87,6 +86,12 @@
 		"lines from different files will go to output simultaneously (out of order of files, but in order of lines in each file)\n"+
 		"use 0 for multi-threaded zst decoder (slightly faster at cost of more CPU)")
 
+	flag.BoolVar(&r.splitChunks, "split-chunks", false, "experimental: with -parallel N (N >= 2), split each uncompressed file (>= 1 MB per chunk)\n"+
+		"into up to N byte-range chunks processed in parallel as virtual files named file.chunkNN,\n"+
+		"(line order is lost in STDOUT/-output, with -out-dir each chunk goes to its own file, so concatenating them in name order restores the order),\n"+
+		"line numbers are per chunk, .gz and .zst files are not split,\n"+
+		"incompatible with -start-line, -end-line, -samples and STDIN")
+
 	cpuProfile := flag.String("dbg-cpu-prof", "", "write first 10 seconds of CPU profile to file")
 	memProfile := flag.String("dbg-mem-prof", "", "write heap profile to file after 10 seconds")
 	output := flag.String("output", "", "output to file (can have .gz or .zst ext for compression) instead of STDOUT")
@@ -181,7 +186,16 @@
 		}
 	}
 
-	sort.Strings(files)
+	if r.splitChunks {
+		switch {
+		case r.parallel < 2:
+			return errors.New("-split-chunks requires -parallel 2 or more")
+		case r.startLine != 0 || r.endLine != 0 || r.samples != 0:
+			return errors.New("-split-chunks is incompatible with -start-line, -end-line and -samples")
+		case isStdin:
+			return errors.New("-split-chunks is incompatible with STDIN")
+		}
+	}
 
 	if *output != "" && r.outDir == "" {
 		w, closer, err := makeWriter(*output)
@@ -280,7 +294,7 @@
 		sem := make(chan struct{}, r.parallel)
 		errs := make(chan error, r.parallel)
 
-		for _, fn := range files {
+		for _, j := range r.jobs(files) {
 			select {
 			case err := <-errs:
 				return err
@@ -292,8 +306,8 @@
 					<-sem
 				}()
 
-				if err := r.cat(fn); err != nil {
-					errs <- fmt.Errorf("%s: %w", fn, err)
+				if err := r.catChunk(j.name, j.chunk); err != nil {
+					errs <- fmt.Errorf("%s: %w", j.name, err)
 				}
 			}()
 		}

cmd/catp/catp/app_test.go

@@ -427,3 +427,62 @@ func Test_Main_samples_gap(t *testing.T) {
 		prev = n
 	}
 }
+
+func Test_Main_splitChunks(t *testing.T) {
+	dir := t.TempDir()
+	src := dir + "/foo.log"
+
+	var sb strings.Builder
+
+	var want strings.Builder
+
+	for i := 0; sb.Len() < 5<<20; i++ {
+		line := "line " + strconv.Itoa(i) + strings.Repeat("x", i%97)
+		if i%7 == 0 {
+			line += " xyz"
+			want.WriteString(line + "\n")
+		}
+
+		sb.WriteString(line + "\n")
+	}
+
+	if err := os.WriteFile(src, []byte(sb.String()), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	out := dir + "/out"
+	if err := os.Mkdir(out, 0o700); err != nil {
+		t.Fatal(err)
+	}
+
+	resetFlags()
+
+	os.Args = []string{"catp", "-no-progress", "-split-chunks", "-parallel", "4", "-pass", "xyz", "-out-dir", out, src}
+
+	if err := catp.Main(); err != nil {
+		t.Fatal(err)
+	}
+
+	var got strings.Builder
+
+	for i := range 4 {
+		b, err := os.ReadFile(out + "/foo.log.chunk" + strconv.Itoa(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		got.Write(b)
+	}
+
+	if got.String() != want.String() {
+		t.Fatalf("chunked output differs: got %d bytes, want %d", got.Len(), want.Len())
+	}
+
+	resetFlags()
+
+	os.Args = []string{"catp", "-split-chunks", "-start-line", "1", "-parallel", "2", src}
+
+	if err := catp.Main(); err == nil {
+		t.Fatal("expected incompatible flags error")
+	}
+}