From c4124f411167df300e286ade9736cae84a06b5c1 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Tue, 20 Oct 2026 13:23:25 +0000 Subject: [PATCH 1/2] Download only the selected Harness for self-hosted machines A self-hosted machine downloaded one 300 MB archive per platform holding Node and every Harness, again for every Session. The release now splits each platform into a small base archive (oac-daemon, bundle.json) and one archive per component. The bootstrap fetches the base and starts oac-daemon install, which downloads Node and the selected Harnesses after onboarding, resumes after resets, and keeps verified archives in native-download keyed by version for later Sessions. Core serves the archives as a generic catalog and knows nothing of Harnesses. --- .github/workflows/native.yml | 7 +- apps/daemon/internal/cli/native_download.go | 426 ++++++++++++++++++ .../internal/cli/native_download_test.go | 201 +++++++++ apps/daemon/internal/cli/native_install.go | 3 + apps/daemon/internal/daemonize/fork_unix.go | 27 -- .../internal/daemonize/fork_unix_test.go | 44 -- .../environment-executor-credentials.md | 2 +- .../zh/environment-executor-credentials.md | 4 +- docs/getting-started/install-options.md | 2 +- docs/getting-started/self-hosted.md | 18 +- docs/maintainers.md | 6 +- docs/zh/getting-started/install-options.md | 4 +- docs/zh/getting-started/self-hosted.md | 20 +- docs/zh/maintainers.md | 8 +- scripts/build-native-catalog.mjs | 22 +- scripts/build-native-catalog.test.mjs | 27 +- scripts/build-native-installer-ci.mjs | 3 +- scripts/build-native-installer.mjs | 13 +- scripts/build-native-installer.test.mjs | 19 +- scripts/core-distribution-manifest.py | 24 +- scripts/core-distribution-manifest.test.py | 9 +- scripts/native-onboarding-smoke.mjs | 34 +- scripts/publish-core-release.py | 8 +- scripts/publish-core-release.test.py | 4 +- .../nativeinstaller/assets/bootstrap.ps1 | 208 +-------- .../nativeinstaller/assets/bootstrap.sh | 119 ++--- .../bootstrap_interrupt_unix_test.go | 69 +-- .../bootstrap_recovery_test.go | 107 ++--- .../bootstrap_resume_unix_test.go | 188 -------- .../core/internal/nativeinstaller/catalog.go | 25 +- .../internal/nativeinstaller/catalog_test.go | 10 +- 31 files changed, 881 insertions(+), 780 deletions(-) create mode 100644 apps/daemon/internal/cli/native_download.go create mode 100644 apps/daemon/internal/cli/native_download_test.go delete mode 100644 apps/daemon/internal/daemonize/fork_unix_test.go delete mode 100644 services/core/internal/nativeinstaller/bootstrap_resume_unix_test.go diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 7c4837d7e..36c755103 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -144,16 +144,11 @@ jobs: export CLAUDE_CODE_GIT_BASH_PATH='C:\Program Files\Git\bin\bash.exe' fi node scripts/native-harness-smoke.mjs --codex-binary "$RUNNER_TEMP/native-installer/components/codex/bin/codex${{ runner.os == 'Windows' && '.exe' || '' }}" --claude-runtime "$RUNNER_TEMP/native-installer/components/claude" - - name: Archive the native distribution - if: inputs.upload-artifacts - run: | - cd "$RUNNER_TEMP" - tar -C native-installer -czf "oac-native-installer-${{ runner.os }}-${{ runner.arch }}.tar.gz" . - uses: actions/upload-artifact@v6 if: inputs.upload-artifacts with: name: oac-native-installer-${{ runner.os }}-${{ runner.arch }} - path: ${{ runner.temp }}/oac-native-installer-*.tar.gz + path: ${{ runner.temp }}/native-archives/*.tar.gz if-no-files-found: error retention-days: 7 compression-level: 0 diff --git a/apps/daemon/internal/cli/native_download.go b/apps/daemon/internal/cli/native_download.go new file mode 100644 index 000000000..3b1b46c74 --- /dev/null +++ b/apps/daemon/internal/cli/native_download.go @@ -0,0 +1,426 @@ +package cli + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/paths" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimefs" +) + +const ( + nativeStall = 60 * time.Second + nativeAttempts = 5 +) + +var ( + nativeRetryDelay = time.Second + nativeDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) + errNativeRedirect = errors.New("install: a download redirect must use HTTPS without credentials") + errNativeNoArchive = errors.New("install: this Core has no qualified archive for this component and platform") +) + +// nativeClient follows redirects only to HTTPS, as the release host requires. +var nativeClient = &http.Client{CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 5 || req.URL.Scheme != "https" || req.URL.User != nil { + return errNativeRedirect + } + return nil +}} + +// nativeInstallerBase is where Core serves this release's installer archives. +func nativeInstallerBase(onboardURL string) string { + return strings.TrimSuffix(onboardURL, "installation") + "install/" + Version +} + +// obtainNativeComponents makes the distribution directory hold node and each selected Harness. +// Without a Core address nothing is downloaded and a missing component is an error. +func obtainNativeComponents(ctx context.Context, out io.Writer, o *nativeInstallOptions, selected []string) error { + var missing []string + for _, name := range append([]string{"node"}, selected...) { + _, inBundle := os.Lstat(nativeComponentRoot(o.Bundle, name)) + _, installed := os.Lstat(nativeComponentRoot(o.Directory, name)) + if inBundle != nil && installed != nil { + missing = append(missing, name) + } + } + if len(missing) == 0 { + return nil + } + if o.OnboardURL == "" { + return fmt.Errorf("install: component %s is missing from %s; extract oac-native-%s-%s-%s-%s.tar.gz into it and retry", missing[0], o.Bundle, Version, runtime.GOOS, runtime.GOARCH, missing[0]) + } + return fetchNativeComponents(ctx, out, o.OnboardURL, o.Bundle, missing) +} + +// fetchNativeComponents downloads the archive of each named component and extracts it into the +// distribution directory. Verified archives stay in the Runtime home, keyed by version, so a later +// installation of the same release reuses them. This is the only owner of that directory's lock. +func fetchNativeComponents(ctx context.Context, out io.Writer, onboardURL, bundle string, names []string) error { + home, err := paths.Root() + if err != nil { + return err + } + cache := filepath.Join(home, "native-download") + if err = runtimefs.EnsurePrivateDir(cache); err != nil { + return err + } + held, err := os.OpenRoot(cache) + if err != nil { + return err + } + defer held.Close() + unlock, err := lockNativeDownloads(ctx, out, held) + if err != nil { + return err + } + defer unlock() + pruneNativeDownloads(cache) + dir := filepath.Join(cache, Version) + if err = os.MkdirAll(dir, 0700); err != nil { + return err + } + for _, name := range names { + file := runtime.GOOS + "-" + runtime.GOARCH + "-" + name + archive, err := fetchNativeArchive(ctx, out, nativeInstallerBase(onboardURL)+"/"+file, dir, file) + if err == nil { + err = extractNativeComponent(ctx, out, archive, bundle, name) + } + if err != nil { + return err + } + } + return nil +} + +func lockNativeDownloads(ctx context.Context, out io.Writer, held *os.Root) (func(), error) { + for waiting := false; ; waiting = true { + if unlock, err := runtimefs.LockDirectory(held); err == nil { + return unlock, nil + } + if !waiting { + fmt.Fprintln(out, "Waiting for another native download to finish...") + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(500 * time.Millisecond): + } + } +} + +// pruneNativeDownloads keeps the current version and recent distribution directories of running +// installers. Failing to remove an old entry is harmless. +func pruneNativeDownloads(cache string) { + entries, _ := os.ReadDir(cache) + for _, entry := range entries { + info, err := entry.Info() + recent := err == nil && strings.HasPrefix(entry.Name(), "bundle.") && time.Since(info.ModTime()) < 24*time.Hour + if entry.Name() != Version && !recent { + _ = os.RemoveAll(filepath.Join(cache, entry.Name())) + } + } +} + +// fetchNativeArchive returns the verified cached archive, downloading or resuming it when needed. +func fetchNativeArchive(ctx context.Context, out io.Writer, url, dir, file string) (string, error) { + want, err := fetchNativeChecksum(ctx, out, url+".sha256") + if err != nil { + return "", err + } + archive := filepath.Join(dir, file+".tar.gz") + if got, err := nativeFileDigest(ctx, archive); err == nil { + if got == want { + fmt.Fprintf(out, "Using the downloaded %s archive.\n", file) + return archive, nil + } + fmt.Fprintf(out, "The downloaded %s archive failed verification and was discarded.\n", file) + _ = os.Remove(archive) + } + part := archive + ".part" + for pass := 0; ; pass++ { + resumed := false + if info, err := os.Stat(part); err == nil && info.Size() > 0 { + resumed = true + fmt.Fprintf(out, "Resuming the earlier %s download...\n", file) + } + if err = retryNativeDownload(ctx, out, func() (bool, error) { return downloadNativeFile(ctx, out, url+".tar.gz", part, file) }); err != nil { + return "", err + } + got, err := nativeFileDigest(ctx, part) + if err != nil { + return "", err + } + if got == want { + return archive, os.Rename(part, archive) + } + _ = os.Remove(part) + // A resumed file may have been damaged earlier; one clean download decides. + if !resumed || pass > 0 { + return "", errors.New("install: archive checksum mismatch; the download was discarded, rerun the command to download it again") + } + } +} + +func fetchNativeChecksum(ctx context.Context, out io.Writer, url string) (string, error) { + var digest string + err := retryNativeDownload(ctx, out, func() (bool, error) { + response, retry, err := getNative(ctx, url, 0) + if err != nil { + return retry, err + } + defer response.Body.Close() + raw, err := io.ReadAll(io.LimitReader(response.Body, 1025)) + digest = strings.TrimSpace(string(raw)) + if err != nil { + return true, err + } + if !nativeDigest.MatchString(digest) { + return false, errors.New("install: Core returned an invalid archive checksum") + } + return false, nil + }) + return digest, err +} + +// getNative returns a 200 or 206 response, or whether its failure is worth another attempt. +func getNative(ctx context.Context, url string, offset int64) (*http.Response, bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, false, errors.New("install: invalid Core installer address") + } + if offset > 0 { + request.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) + } + response, err := nativeClient.Do(request) + if err != nil { + return nil, !errors.Is(err, errNativeRedirect), err + } + code := response.StatusCode + switch { + case code == http.StatusOK || code == http.StatusPartialContent || code == http.StatusRequestedRangeNotSatisfiable: + return response, false, nil + case code == http.StatusNotFound: + err = errNativeNoArchive + default: + err = fmt.Errorf("install: download failed (HTTP %d); check Core and the download host", code) + } + response.Body.Close() + return nil, code == 408 || code == 429 || code >= 500, err +} + +func retryNativeDownload(ctx context.Context, out io.Writer, attempt func() (bool, error)) error { + for n := 1; ; n++ { + retry, err := attempt() + if err == nil { + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + if !retry || nativeDiskFull(err) { + return err + } + if n == nativeAttempts { + return fmt.Errorf("%w (downloaded data is kept; check the network to Core and GitHub, set HTTPS_PROXY if a proxy is required, and rerun the command)", err) + } + fmt.Fprintf(out, "Download interrupted; retrying (%d/%d)...\n", n, nativeAttempts-1) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(n) * nativeRetryDelay): + } + } +} + +// downloadNativeFile appends to dest from where an earlier attempt stopped and fails when no data +// arrives for nativeStall. There is no total time limit. +func downloadNativeFile(ctx context.Context, out io.Writer, url, dest, label string) (bool, error) { + var offset int64 + if info, err := os.Stat(dest); err == nil { + offset = info.Size() + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stall := time.AfterFunc(nativeStall, cancel) + defer stall.Stop() + response, retry, err := getNative(ctx, url, offset) + if err != nil { + return retry, err + } + defer response.Body.Close() + flags := os.O_WRONLY | os.O_CREATE | os.O_APPEND + switch response.StatusCode { + case http.StatusRequestedRangeNotSatisfiable: + return false, nil // The partial file already holds everything; the checksum decides. + case http.StatusOK: + flags, offset = os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0 + } + if response.ContentLength > 0 { + if err = requireNativeSpace(filepath.Dir(dest), uint64(response.ContentLength)); err != nil { + return false, err + } + } + file, err := os.OpenFile(dest, flags, 0600) + if err != nil { + return false, err + } + progress := &nativeProgress{out: out, label: label, done: offset} + if response.ContentLength > 0 { + progress.total = offset + response.ContentLength + } + _, err = io.Copy(file, &nativeBody{Reader: response.Body, stall: stall, progress: progress}) + if closeErr := file.Close(); err == nil { + err = closeErr + } + return err != nil, err +} + +type nativeBody struct { + io.Reader + stall *time.Timer + progress *nativeProgress +} + +func (b *nativeBody) Read(p []byte) (int, error) { + n, err := b.Reader.Read(p) + if n > 0 { + b.stall.Reset(nativeStall) + b.progress.add(int64(n)) + } + return n, err +} + +// nativeProgress prints one short line for each tenth of a download. +type nativeProgress struct { + out io.Writer + label string + done, total int64 + step int64 +} + +func (p *nativeProgress) add(n int64) { + p.done += n + if p.total == 0 { + return + } + if step := p.done * 10 / p.total; step > p.step { + p.step = step + fmt.Fprintf(p.out, "Downloading %s: %d%% (%d of %d MiB)\n", p.label, step*10, p.done>>20, p.total>>20) + } +} + +func nativeFileDigest(ctx context.Context, path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + hash := sha256.New() + if _, err = nativeCopy(ctx, hash, file); err != nil { + return "", err + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// extractNativeComponent unpacks components// from a verified archive into the distribution +// directory. Only regular files and directories under that path are accepted; per-file digests are +// checked later against bundle.json. +func extractNativeComponent(ctx context.Context, out io.Writer, archive, bundle, name string) error { + var size uint64 + if err := eachNativeEntry(ctx, archive, name, func(h *tar.Header, _ string, _ io.Reader) error { + size += uint64(h.Size) + return nil + }); err != nil { + return err + } + parent := filepath.Join(bundle, "components") + if err := os.MkdirAll(parent, 0700); err != nil { + return err + } + if err := requireNativeSpace(parent, size); err != nil { + return err + } + fmt.Fprintf(out, "Extracting %s...\n", name) + stage := filepath.Join(parent, ".extract-"+name) + if err := os.RemoveAll(stage); err != nil { + return err + } + defer os.RemoveAll(stage) + if err := eachNativeEntry(ctx, archive, name, func(h *tar.Header, rel string, r io.Reader) error { + target := filepath.Join(stage, filepath.FromSlash(rel)) + if h.Typeflag == tar.TypeDir { + return os.MkdirAll(target, 0700) + } + if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600) + if err != nil { + return err + } + _, err = nativeCopy(ctx, file, r) + if closeErr := file.Close(); err == nil { + err = closeErr + } + return err + }); err != nil { + return err + } + return os.Rename(stage, nativeComponentRoot(bundle, name)) +} + +// eachNativeEntry visits the files and directories of a component archive; the root directory itself is skipped. +func eachNativeEntry(ctx context.Context, archive, name string, visit func(*tar.Header, string, io.Reader) error) error { + file, err := os.Open(archive) + if err != nil { + return err + } + defer file.Close() + zipped, err := gzip.NewReader(nativeCopyReader{ctx: ctx, Reader: file}) + if err != nil { + return errors.New("install: component archive is invalid; remove it and retry") + } + reader := tar.NewReader(zipped) + prefix := "components/" + name + "/" + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("install: component archive is invalid or incomplete: %w", err) + } + if header.Typeflag != tar.TypeDir && header.Typeflag != tar.TypeReg { + return errors.New("install: component archive may contain only files and directories") + } + path := header.Name + if header.Typeflag == tar.TypeDir { + path = strings.TrimSuffix(path, "/") + "/" + } + rel, ok := strings.CutPrefix(path, prefix) + rel = strings.TrimSuffix(rel, "/") + if !ok || (rel == "" && header.Typeflag != tar.TypeDir) || (rel != "" && !validBundlePath(rel)) { + return errors.New("install: component archive contains an unexpected path") + } + if rel == "" { + continue + } + if err = visit(header, rel, reader); err != nil { + return err + } + } +} diff --git a/apps/daemon/internal/cli/native_download_test.go b/apps/daemon/internal/cli/native_download_test.go new file mode 100644 index 000000000..d64ebbf16 --- /dev/null +++ b/apps/daemon/internal/cli/native_download_test.go @@ -0,0 +1,201 @@ +package cli + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +// fakeCore serves the installation endpoints and one archive per component. +type fakeCore struct { + *httptest.Server + mu sync.Mutex + archives map[string][]byte + bodies map[string]int // archive downloads by component + breakOn string // component whose first download is cut off halfway + status int // answer for /installation + redirect string // when set, archives redirect here +} + +func newFakeCore(t *testing.T, workspace string) (*fakeCore, string) { + t.Helper() + environment := uuid.NewString() + core := &fakeCore{archives: map[string][]byte{}, bodies: map[string]int{}, status: 200} + core.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + core.mu.Lock() + defer core.mu.Unlock() + switch { + case strings.HasSuffix(r.URL.Path, "/installation/claim"): + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/installation"): + if core.status != 200 { + w.WriteHeader(core.status) + return + } + _ = json.NewEncoder(w).Encode(v1.NativeInstallationContext{Version: Version, ProtocolVersion: proto.Version, EnvironmentID: environment, + RemoteURL: strings.Replace(core.URL, "http://", "ws://", 1) + "/api/v1/agent-daemon/ws", Workspace: workspace, Harness: nativeHarnesses["codex"].AgentKind}) + default: + name, ext, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/api/v1/agent-daemon/install/"+Version+"/"+runtime.GOOS+"-"+runtime.GOARCH+"-"), ".") + data, ok := core.archives[name] + switch { + case !ok: + w.WriteHeader(http.StatusNotFound) + case ext == "sha256": + fmt.Fprintf(w, "%x\n", sha256.Sum256(data)) + case core.redirect != "": + http.Redirect(w, r, core.redirect, http.StatusTemporaryRedirect) + case name == core.breakOn && r.Header.Get("Range") == "": + core.bodies[name]++ + w.Header().Set("Content-Length", fmt.Sprint(len(data))) + _, _ = w.Write(data[:len(data)/2]) + w.(http.Flusher).Flush() + panic(http.ErrAbortHandler) + default: + if r.Header.Get("Range") == "" { + core.bodies[name]++ + } + http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(data)) + } + } + })) + t.Cleanup(core.Close) + return core, environment +} + +// add publishes a component archive and returns its bundle.json entry. +func (c *fakeCore) add(t *testing.T, name string) nativeComponent { + t.Helper() + data := []byte("fixture " + name + strings.Repeat(" padding", 4096)) + sum := sha256.Sum256(data) + var archive bytes.Buffer + zipped := gzip.NewWriter(&archive) + writer := tar.NewWriter(zipped) + for _, header := range []*tar.Header{{Name: "components/" + name + "/", Typeflag: tar.TypeDir, Mode: 0755}, {Name: "components/" + name + "/program", Typeflag: tar.TypeReg, Mode: 0755, Size: int64(len(data))}} { + if err := writer.WriteHeader(header); err != nil { + t.Fatal(err) + } + } + _, _ = writer.Write(data) + if writer.Close() != nil || zipped.Close() != nil { + t.Fatal("cannot build archive") + } + c.archives[name] = archive.Bytes() + return nativeComponent{Version: nativePins[name], Files: map[string]nativeFile{"program": {SHA256: hex.EncodeToString(sum[:]), Executable: true}}} +} + +func nativeDownloadInstall(t *testing.T, core *fakeCore, bundle string, extra ...string) error { + t.Helper() + t.Setenv(v1.NativeInstallGrantEnv, "grant") + out := new(bytes.Buffer) + rc := &runContext{stdin: strings.NewReader(""), stdout: out, stderr: out} + o, err := parseNativeInstall(rc, append([]string{"--non-interactive", "--onboard-url", core.URL + "/api/v1/agent-daemon/installation", "--bundle-dir", bundle, "--harness", "codex"}, extra...)) + if err != nil { + return err + } + return installNativeOptions(context.Background(), rc, &o) +} + +func writeBundle(t *testing.T, bundle string, components map[string]nativeComponent) { + t.Helper() + raw, _ := json.Marshal(nativeBundle{Schema: 1, DaemonVersion: Version, OS: runtime.GOOS, Arch: runtime.GOARCH, Components: components}) + if err := os.WriteFile(filepath.Join(bundle, "bundle.json"), raw, 0600); err != nil { + t.Fatal(err) + } +} + +func TestNativeInstallDownloadsOnlySelectedComponentsOnceAndResumes(t *testing.T) { + home := t.TempDir() + t.Setenv("OAC_RUNTIME_HOME", home) + nativeRetryDelay = time.Millisecond + probeNativeInstallation = func(context.Context, string, []string) error { return nil } + core, _ := newFakeCore(t, t.TempDir()) + components := map[string]nativeComponent{} + for _, name := range []string{"node", "codex", "claude"} { + components[name] = core.add(t, name) + } + core.breakOn = "codex" + cache := filepath.Join(home, "native-download") + if err := os.MkdirAll(filepath.Join(cache, "older-version"), 0700); err != nil { + t.Fatal(err) + } + // Two Sessions of one release share the machine's cache. + for i := 0; i < 2; i++ { + bundle := t.TempDir() + writeBundle(t, bundle, components) + if i == 1 { + // A damaged cached archive is discarded and fetched again. + if err := os.WriteFile(filepath.Join(cache, Version, runtime.GOOS+"-"+runtime.GOARCH+"-node.tar.gz"), []byte("damaged"), 0600); err != nil { + t.Fatal(err) + } + } + if err := nativeDownloadInstall(t, core, bundle, "--install-dir", t.TempDir()); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(nativeComponentRoot(bundle, "claude")); err == nil { + t.Fatal("downloaded a Harness the Session does not use") + } + } + if core.bodies["node"] != 2 || core.bodies["claude"] != 0 { + t.Fatalf("unexpected downloads: %v", core.bodies) + } + if core.bodies["codex"] != 1 { + t.Fatalf("codex was fetched %d times, want its interrupted download resumed and then reused", core.bodies["codex"]) + } + if _, err := os.Stat(filepath.Join(cache, "older-version")); err == nil { + t.Fatal("archives of another version were kept") + } +} + +func TestNativeInstallRejectsGrantBeforeDownloading(t *testing.T) { + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + core, _ := newFakeCore(t, t.TempDir()) + core.status = http.StatusUnauthorized + if err := nativeDownloadInstall(t, core, t.TempDir()); err == nil || !strings.Contains(err.Error(), "authorization rejected") { + t.Fatalf("expected a rejected grant, got %v", err) + } + if len(core.bodies) != 0 { + t.Fatal("downloaded after a rejected grant") + } +} + +func TestNativeInstallRefusesRedirectToHTTP(t *testing.T) { + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + core, _ := newFakeCore(t, t.TempDir()) + components := map[string]nativeComponent{"node": core.add(t, "node"), "codex": core.add(t, "codex")} + core.redirect = "http://127.0.0.1:1/archive.tar.gz" + bundle := t.TempDir() + writeBundle(t, bundle, components) + if err := nativeDownloadInstall(t, core, bundle, "--install-dir", t.TempDir()); err == nil || !strings.Contains(err.Error(), "HTTPS") { + t.Fatalf("expected an HTTPS-only redirect error, got %v", err) + } +} + +func TestNativeInstallFromDistributionNamesMissingArchive(t *testing.T) { + rc, args, _, bundle := nativeInstallFixture(t) + if err := os.RemoveAll(nativeComponentRoot(bundle, "codex")); err != nil { + t.Fatal(err) + } + err := runInstall(rc, args) + want := fmt.Sprintf("oac-native-%s-%s-%s-codex.tar.gz", Version, runtime.GOOS, runtime.GOARCH) + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected an error naming %s, got %v", want, err) + } +} diff --git a/apps/daemon/internal/cli/native_install.go b/apps/daemon/internal/cli/native_install.go index a59fc9d5c..71fbc9d3f 100644 --- a/apps/daemon/internal/cli/native_install.go +++ b/apps/daemon/internal/cli/native_install.go @@ -184,6 +184,9 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO } } slices.Sort(all) + if err = obtainNativeComponents(ctx, rc.stdout, o, selected); err != nil { + return err + } // Verify existing components before adding any new one; never repair or // upgrade an installed dependency as a side effect of adding a Harness. if len(previous.Harnesses) > 0 { diff --git a/apps/daemon/internal/daemonize/fork_unix.go b/apps/daemon/internal/daemonize/fork_unix.go index 142ba56c7..d729a25be 100644 --- a/apps/daemon/internal/daemonize/fork_unix.go +++ b/apps/daemon/internal/daemonize/fork_unix.go @@ -3,38 +3,11 @@ package daemonize import ( - "fmt" - "os" "os/exec" - "strconv" "syscall" ) func configureBackground(cmd *exec.Cmd) (func(), func() error, error) { - if err := closeInheritedOnExec(); err != nil { - return nil, nil, err - } cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} return func() {}, func() error { return nil }, nil } - -// closeInheritedOnExec keeps descriptors this process inherited, such as the -// native bootstrap's download lock, out of the detached daemon, which outlives -// its caller. Go opens its own descriptors close-on-exec already. -func closeInheritedOnExec() error { - dir, err := os.Open("/dev/fd") - if err != nil { - return fmt.Errorf("daemonize: list open descriptors: %w", err) - } - names, err := dir.Readdirnames(-1) - dir.Close() - if err != nil { - return fmt.Errorf("daemonize: list open descriptors: %w", err) - } - for _, name := range names { - if fd, err := strconv.Atoi(name); err == nil && fd > 2 { - syscall.CloseOnExec(fd) - } - } - return nil -} diff --git a/apps/daemon/internal/daemonize/fork_unix_test.go b/apps/daemon/internal/daemonize/fork_unix_test.go deleted file mode 100644 index a3caf9f55..000000000 --- a/apps/daemon/internal/daemonize/fork_unix_test.go +++ /dev/null @@ -1,44 +0,0 @@ -//go:build unix - -package daemonize - -import ( - "path/filepath" - "syscall" - "testing" - "time" - - "golang.org/x/sys/unix" -) - -// The native bootstrap holds its download lock on an inheritable descriptor. -// The detached daemon must not keep that lock after its caller exits. -func TestSpawnLeavesInheritedLockBehind(t *testing.T) { - dir := privateTempDir(t) - lockPath := filepath.Join(dir, "download.lock") - inherited, err := syscall.Open(lockPath, syscall.O_RDWR|syscall.O_CREAT, 0o600) - if err != nil { - t.Fatal(err) - } - if err := unix.Flock(inherited, unix.LOCK_EX|unix.LOCK_NB); err != nil { - t.Fatal(err) - } - t.Setenv(spawnTestChildEnv, "1") - pidPath := filepath.Join(dir, "child.pid") - if _, err := Spawn([]string{"oac-daemon", "child-mode"}, ReExecOptions{LogPath: filepath.Join(dir, "child.log"), PIDPath: pidPath}); err != nil { - t.Fatalf("Spawn: %v", err) - } - t.Cleanup(func() { _ = StopPIDFile(pidPath, 2*time.Second) }) - if err := syscall.Close(inherited); err != nil { - t.Fatal(err) - } - - next, err := syscall.Open(lockPath, syscall.O_RDWR|syscall.O_CLOEXEC, 0) - if err != nil { - t.Fatal(err) - } - defer syscall.Close(next) - if err := unix.Flock(next, unix.LOCK_EX|unix.LOCK_NB); err != nil { - t.Fatalf("the background child kept the caller's lock: %v", err) - } -} diff --git a/contracts/agents-api/environment-executor-credentials.md b/contracts/agents-api/environment-executor-credentials.md index 87bb9c0cb..06863c4e4 100644 --- a/contracts/agents-api/environment-executor-credentials.md +++ b/contracts/agents-api/environment-executor-credentials.md @@ -31,7 +31,7 @@ The installer calls these machine routes on Core: | Route | Authorization | Purpose | | --- | --- | --- | | `GET /api/v1/agent-daemon/install/{version}/bootstrap.sh`, `bootstrap.ps1` | None | Platform bootstrap scripts | -| `GET /api/v1/agent-daemon/install/{version}/{os}-{arch}.sha256`, `{os}-{arch}.tar.gz` | None | Installer checksum and archive. Core serves a local copy, or redirects (307) to the versioned release URL in its catalog | +| `GET /api/v1/agent-daemon/install/{version}/{os}-{arch}[-{component}].sha256`, `{os}-{arch}[-{component}].tar.gz` | None | Checksum and archive of a platform's base installer or of one of its components, named in its `bundle.json`. The installer downloads only the components it needs. Core serves a local copy, or redirects (307) to the versioned release URL in its catalog | | `POST /api/v1/agent-daemon/installation` | Grant | The frozen binding: `version`, `protocol_version`, `environment_id`, `remote_url`, `workspace_directory`, `harness` | | `POST /api/v1/agent-daemon/installation/claim` | Grant | `{"executor_token":"SECRET"}`; 204 | diff --git a/contracts/agents-api/zh/environment-executor-credentials.md b/contracts/agents-api/zh/environment-executor-credentials.md index c874080cd..b711a247f 100644 --- a/contracts/agents-api/zh/environment-executor-credentials.md +++ b/contracts/agents-api/zh/environment-executor-credentials.md @@ -1,7 +1,7 @@ --- title: "执行器凭据" source: contracts/agents-api/environment-executor-credentials.md -source_hash: b97d92cb23ad4c7abd5ab71aa768b3b733ab8912c35f7cfbfb73e0c1b7dc028c +source_hash: bd7758cbe88d35995dcdffafc381f4a1d6b36ef70f9d0932f88d4d3b67ec1ab6 --- 执行器凭据允许 `oac-daemon` 为一个 `self_hosted` Environment 注册并连接。它只授权该 Environment 的私有 daemon 传输（`/api/v1/agent-daemon/*`），不授权 `/v1`、`/core/v1`、sandbox node 注册或项目资源。项目的 principal 是其执行 principal。Core 只保存密钥摘要。 @@ -33,7 +33,7 @@ grant 绑定 Environment、Session 创建者的 principal 和 Core 构建版本 | 路由 | 授权 | 用途 | | --- | --- | --- | | `GET /api/v1/agent-daemon/install/{version}/bootstrap.sh`, `bootstrap.ps1` | 无 | 平台 bootstrap 脚本 | -| `GET /api/v1/agent-daemon/install/{version}/{os}-{arch}.sha256`, `{os}-{arch}.tar.gz` | 无 | 安装程序校验和与归档；Core 提供本地副本，或以 307 重定向到目录中的版本化发布 URL | +| `GET /api/v1/agent-daemon/install/{version}/{os}-{arch}[-{component}].sha256`, `{os}-{arch}[-{component}].tar.gz` | 无 | 平台基础安装程序或其某个组件（在其 `bundle.json` 中列出）的校验和与归档；安装程序只下载所需的组件。Core 提供本地副本，或以 307 重定向到目录中的版本化发布 URL | | `POST /api/v1/agent-daemon/installation` | Grant | 固定绑定：`version`、`protocol_version`、`environment_id`、`remote_url`、`workspace_directory`、`harness` | | `POST /api/v1/agent-daemon/installation/claim` | Grant | `{"executor_token":"SECRET"}`；204 | diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index 258175e69..00326e29d 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -82,7 +82,7 @@ Stop with `docker compose stop` using the same files and environment. Back up th The installer downloads release files from GitHub Releases with this shell's network settings, including `HTTPS_PROXY`. Docker pulls the images through its daemon, which does not read the shell's proxy variables. Every image, PostgreSQL included, comes from GHCR (`ghcr.io`); Docker Hub is not needed. When GHCR is reachable only through a proxy or mirror, configure [the daemon's proxy](https://docs.docker.com/engine/daemon/proxy/) or a [registry mirror](https://docs.docker.com/docker-hub/image-library/mirror/), then rerun the installer. -A Linux amd64 host without that access installs from the release asset `oac--linux-amd64-offline.tar.gz`, which contains every image and node artifact and the Linux amd64 self-hosted installer. Self-hosted machines on macOS and Windows download their installer from GitHub Releases. Copy it to the host, extract it and run its `install.py` with Python 3.9 or newer and Docker. No checkout, Go, Node or pnpm is needed on that host. Keep the extracted bundle until installation completes; the installed services and node downloads do not depend on its original location afterward. The same archive is what [Develop OpenAgentCore](../development.md#install-a-checkout-without-publishing-a-release) builds from a checkout. +A Linux amd64 host without that access installs from the release asset `oac--linux-amd64-offline.tar.gz`, which contains every image and node artifact and the Linux amd64 self-hosted installer with every Harness. Self-hosted machines on macOS and Windows download their installer from GitHub Releases. Copy it to the host, extract it and run its `install.py` with Python 3.9 or newer and Docker. No checkout, Go, Node or pnpm is needed on that host. Keep the extracted bundle until installation completes; the installed services and node downloads do not depend on its original location afterward. The same archive is what [Develop OpenAgentCore](../development.md#install-a-checkout-without-publishing-a-release) builds from a checkout. ### `install.py` options diff --git a/docs/getting-started/self-hosted.md b/docs/getting-started/self-hosted.md index 40ecb6590..270cbf638 100644 --- a/docs/getting-started/self-hosted.md +++ b/docs/getting-started/self-hosted.md @@ -21,14 +21,14 @@ The Session brings its own model provider; the installation default never applie The installer brings its own pinned Node.js and Harness versions (listed in [`scripts/build-native-installer.mjs`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/scripts/build-native-installer.mjs)) and leaves other installations of those tools untouched. On a platform without a matching installer, the command fails. -The installer download is about 320 MiB. Free space is needed on the disk that holds the Runtime home and the installation directory: +The download depends on the Harnesses you install. The installer (9 MiB) and Node.js (59 MiB) are always needed, plus 121 MiB for Codex, 99 MiB for Claude Code or 23 MiB for MiniMax Code, so a Session with one Harness downloads 90 to 189 MiB. Free space is needed on the disk that holds the Runtime home and the installation directory: | | One Harness | All three | | --- | --- | --- | -| Free space at the peak | about 2 GiB | about 2.2 GiB | +| Free space at the peak | about 1.3 GiB | about 2.1 GiB | | Installed size | about 560 MiB | about 940 MiB | -With one Harness, the peak is the archive (316 MiB), the unpacked installer (907 MiB) and the installed files (about 560 MiB). The unpacked copy is removed when the command ends and the archive when the installation completes. +With Codex, the peak is the archives (180 MiB), the unpacked installer (about 560 MiB) and the installed files (about 560 MiB). The unpacked copy is removed when the command ends. The archives stay in `native-download` under the Runtime home so that another Session of the same release on this machine downloads only the 9 MiB installer. The machine needs: @@ -37,7 +37,7 @@ The machine needs: - Python and pip when the Session's packages need them; - any system packages your setup needs. The daemon never runs apt, sudo or another elevation command, so install them through the host's normal administration. -No administrator privileges or Docker are needed. The Unix download command also uses `curl`, `tar`, `gzip`, a SHA-256 tool and the system file-lock command (`flock` on Linux, `lockf` on macOS). Windows needs its system `tar.exe`; the command checks for it before downloading. The Runtime home must allow executable files to run. On a Unix `noexec` mount, choose another absolute directory with `OAC_RUNTIME_HOME` before running the command. +No administrator privileges or Docker are needed. The Unix download command also uses `curl`, `tar` and a SHA-256 tool. Windows needs its system `tar.exe`; the command checks for it before downloading. The Runtime home must allow executable files to run. On a Unix `noexec` mount, choose another absolute directory with `OAC_RUNTIME_HOME` before running the command. ## Connect a machine @@ -66,7 +66,7 @@ No administrator privileges or Docker are needed. The Unix download command also print(installation["commands"]["posix"]) # use "powershell" on Windows ``` -2. Run the command on the target machine with the account that should run the tools. It downloads the installer matched to this Core, verifies its checksum, asks which Harnesses to install and where, installs them, creates the workspace if needed, starts the daemon and checks its connection. +2. Run the command on the target machine with the account that should run the tools. It downloads the installer matched to this Core, verifies its checksum, asks which Harnesses to install and where, downloads Node.js and the selected Harnesses, installs them, creates the workspace if needed, starts the daemon and checks its connection. 3. Send a Turn. A connected machine proves only authentication; the first Turn checks the Harness and the model. In Web, open the Session and copy the command under **Connect a host**. @@ -86,7 +86,7 @@ The installer reports three results:
If the download stops -The installer checks the grant before it downloads anything, so an expired command costs no download. The archive download resumes after a reset, fails only when it stalls (no data for 60 seconds) and gives up after five attempts; rerun the command to continue where it stopped. A verified archive is kept in `native-download` under the Runtime home until the installation completes, so a rerun after a failure needs no new download; it is checked against Core's checksum on every run and discarded if it fails. Disk space is checked before downloading, extracting and copying components. If a download or installation is interrupted, rerun the command: it clears unfinished temporary copies while preserving completed components, credentials and the workspace. The small `download.lock` file in `native-download` remains for concurrency control. An active download or installation is never cleared by another run. If the command expires, copy a fresh one from the Session. +The installer checks the grant before it downloads a Harness, so an expired command costs only the 9 MiB installer. Each archive download resumes after a reset, fails only when it stalls (no data for 60 seconds) and gives up after five attempts; rerun the command to continue where it stopped. Verified archives stay in `native-download/` under the Runtime home. A later run or Session of the same version checks them against Core's checksum and reuses them, discarding any that fails; archives of other versions are removed. Two installations on one machine take turns downloading. Disk space is checked before downloading, extracting and copying components. If a download or installation is interrupted, rerun the command: it clears unfinished temporary copies while preserving completed components, credentials and the workspace. If the command expires, copy a fresh one from the Session.
@@ -144,7 +144,7 @@ The install command claims the Session's machine credential once, and a second c To reconnect the Session without the original directory, rotate its credential and install from an extracted distribution into a new directory: 1. In Web, open the Session and choose **Rotate** in **Executor credentials**. Save the credential JSON it returns. -2. Download `/.tar.gz` and extract it, where `` is the first URL of the install command (it ends with the Core version) and `` is `linux-amd64`, `darwin-arm64` or `windows-amd64`, matching the rows of [Platforms](#platforms). +2. Download `/.tar.gz` and `/-.tar.gz` for `node` and each Harness you install (`codex`, `claude` or `minimax`), and extract them into one directory. `` is the first URL of the install command (it ends with the Core version) and `` is `linux-amd64`, `darwin-arm64` or `windows-amd64`, matching the rows of [Platforms](#platforms). 3. Run the installer as in [Install from an extracted distribution](#install-from-an-extracted-distribution) with the Session's `remote_url`, Environment ID, workspace and the saved credential file. Or create a new Session and run its command. @@ -163,7 +163,7 @@ On Windows PowerShell: & "$HOME\.oac\environments\\bin\oac-daemon.exe" uninstall ``` -It stops the daemon, lists what it deletes and asks for confirmation. It deletes only what an installation owns inside its directory (`bin`, `daemon`, `components`, `capabilities`, `runtime`, `initialization` and `packages`, with their credential file, native history and Harnesses) and then the directory itself if nothing else is left in it. Anything else in the directory stays, and so do the workspace and any credential file or capability directory outside the installation; the command names each one it keeps. The `native-download` directory and its `download.lock` are never touched. Where nothing is installed it says so and exits 0. If a file cannot be deleted, the command names it and can be run again once it is released. On Windows, the final deletion finishes a few seconds after the command exits. +It stops the daemon, lists what it deletes and asks for confirmation. It deletes only what an installation owns inside its directory (`bin`, `daemon`, `components`, `capabilities`, `runtime`, `initialization` and `packages`, with their credential file, native history and Harnesses) and then the directory itself if nothing else is left in it. Anything else in the directory stays, and so do the workspace and any credential file or capability directory outside the installation; the command names each one it keeps. The `native-download` cache is never touched; delete it to reclaim the space. Where nothing is installed it says so and exits 0. If a file cannot be deleted, the command names it and can be run again once it is released. On Windows, the final deletion finishes a few seconds after the command exits. Then, in Web, open the Session and choose **Revoke** in **Executor credentials**, or **Delete** the Session; Core keeps accepting the machine's credential until you do. @@ -218,4 +218,4 @@ The same installer accepts an already extracted distribution and a credential fi "$HOME/.oac/my-runtime/bin/oac-daemon" start ``` -Use the Session's `remote_url` and Environment ID. The distribution is the archive named in [Reconnect an existing Session](#reconnect-an-existing-session). In PowerShell, run `.\oac-daemon.exe` with native absolute paths. The installation directory must be missing, empty or hold an installation. This mode needs an existing workspace and does not start the daemon until you run `start`. +Use the Session's `remote_url` and Environment ID. The distribution is the directory described in [Reconnect an existing Session](#reconnect-an-existing-session): `oac-daemon` and `bundle.json` from the platform archive, and `components/` from the archives of `node` and each selected Harness. A missing component fails with the name of the archive to extract; this mode never downloads. In PowerShell, run `.\oac-daemon.exe` with native absolute paths. The installation directory must be missing, empty or hold an installation. This mode needs an existing workspace and does not start the daemon until you run `start`. diff --git a/docs/maintainers.md b/docs/maintainers.md index a07d422f1..74a1a22ef 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -47,14 +47,16 @@ A distribution carries the docs listed in `BUNDLED_DOCS` in `scripts/core-distri ### Native installers -Self-hosted machines install `oac-daemon` from per-platform native installers: Linux amd64, macOS arm64 and Windows amd64. Each is built on its own OS by the `native-check` workflow (`scripts/build-native-installer.mjs`, whose `pins` object fixes the Node.js and Harness versions) and verified on every selected native check. Manual packaging runs and release checks upload `oac-native-installer--.tar.gz` for seven days; ordinary PR and main checks do not upload successful packages. For a local distribution, download the three artifacts from a `native-check` run on that exact commit (a manual run or the release run; pull-request runs build the merge commit and do not match), then assemble the catalog from that checkout: +Self-hosted machines install `oac-daemon` from per-platform native installers: Linux amd64, macOS arm64 and Windows amd64. Each is built on its own OS by the `native-check` workflow (`scripts/build-native-installer.mjs`, whose `pins` object fixes the Node.js and Harness versions) and verified on every selected native check. Manual packaging runs and release checks upload the platform's archives as `oac-native-installer--` for seven days; ordinary PR and main checks do not upload successful packages. For a local distribution, download the three artifacts from a `native-check` run on that exact commit (a manual run or the release run; pull-request runs build the merge commit and do not match), then assemble the catalog from that checkout, with all archives of the three artifacts in one input directory: ```sh node scripts/build-native-catalog.mjs INPUT_DIR OUTPUT_DIR export OAC_NATIVE_INSTALLER_BUILD_DIR=OUTPUT_DIR ``` -The catalog records the commit, the Runtime protocol version, each archive's SHA-256 and, with a release base, its versioned URL. The control archive and Core image carry only `native-installers/catalog.json`; the archives become separate `oac-native--.tar.gz` release assets, and the offline archive holds a copy of the Linux amd64 archive outside the Core image, which its `install.py` copies into the installation for Core to serve. Core redirects the other platforms to their release assets; a catalog without a release base has no such URL, so its offline archive holds every archive. Without a catalog, Sessions report the install command as unavailable, and the release workflow refuses to publish. +Each platform has a small base archive `-.tar.gz` (`oac-daemon` and `bundle.json`, which lists every component with the SHA-256 of each file) and one archive per component, `--node` and `--` (MiniMax Code is not built for Windows). `build-native-installer.mjs` writes them. Core serves these artifacts by name and knows nothing of their content; `oac-daemon install` downloads the components it needs. + +The catalog records the commit, the Runtime protocol version, each artifact's SHA-256 and, with a release base, its versioned URL. The control archive and Core image carry only `native-installers/catalog.json`; the archives become separate `oac-native--.tar.gz` release assets, and the offline archive holds a copy of every Linux amd64 archive outside the Core image, which its `install.py` copies into the installation for Core to serve. Core redirects the other platforms to their release assets; a catalog without a release base has no such URL, so its offline archive holds every archive. Without a catalog, Sessions report the install command as unavailable, and the release workflow refuses to publish. ### Runtime images and helpers diff --git a/docs/zh/getting-started/install-options.md b/docs/zh/getting-started/install-options.md index 86e13c33d..5caa5b326 100644 --- a/docs/zh/getting-started/install-options.md +++ b/docs/zh/getting-started/install-options.md @@ -1,7 +1,7 @@ --- title: "安装选项" source: docs/getting-started/install-options.md -source_hash: d8df8f9bacd4a7ee2246ae0878d2139108a06079d0a6d0245f98a67972839b3e +source_hash: e3503e97930dc4d36f34411e5588d4b9c7102b15eeb14d3763a797c18081af2e --- 本页是安装程序参数的参考，并说明如何用默认方式以外的方式安装：在已有反向代理之后、在托管平台上使用 Docker Compose，或在无法访问互联网的主机上。[默认安装](./install.md)无需任何选项。 @@ -84,7 +84,7 @@ docker compose exec web oac-web core-key 安装程序使用当前 shell 的网络设置（包括 `HTTPS_PROXY`）从 GitHub Releases 下载发行文件。Docker 通过其守护进程拉取镜像，守护进程不读取 shell 的代理变量。所有镜像（包括 PostgreSQL）都来自 GHCR（`ghcr.io`），不需要 Docker Hub。如果只能经代理或镜像加速器访问 GHCR，请配置[守护进程代理](https://docs.docker.com/engine/daemon/proxy/)或[镜像加速器](https://docs.docker.com/docker-hub/image-library/mirror/)，然后重新运行安装程序。 -无法访问这些地址的 Linux amd64 主机可使用发行资产 `oac--linux-amd64-offline.tar.gz` 安装，其中包含所有镜像、节点制品和 Linux amd64 自托管安装程序。macOS 和 Windows 上的自托管机器从 GitHub Releases 下载各自的安装程序。将其复制到该主机并解压，然后用 Python 3.9 或更高版本和 Docker 运行其中的 `install.py`。该主机上不需要检出代码、Go、Node 或 pnpm。安装完成前请保留解压后的安装包；之后已安装的服务和节点下载不再依赖它原来的位置。[开发 OpenAgentCore](../development.md#install-a-checkout-without-publishing-a-release)从检出代码构建的就是同一个归档。 +无法访问这些地址的 Linux amd64 主机可使用发行资产 `oac--linux-amd64-offline.tar.gz` 安装，其中包含所有镜像、节点制品和包含全部 Harness 的 Linux amd64 自托管安装程序。macOS 和 Windows 上的自托管机器从 GitHub Releases 下载各自的安装程序。将其复制到该主机并解压，然后用 Python 3.9 或更高版本和 Docker 运行其中的 `install.py`。该主机上不需要检出代码、Go、Node 或 pnpm。安装完成前请保留解压后的安装包；之后已安装的服务和节点下载不再依赖它原来的位置。[开发 OpenAgentCore](../development.md#install-a-checkout-without-publishing-a-release)从检出代码构建的就是同一个归档。 ### `install.py` 选项 {#install-py-options} diff --git a/docs/zh/getting-started/self-hosted.md b/docs/zh/getting-started/self-hosted.md index 34591f2dd..d242e63e3 100644 --- a/docs/zh/getting-started/self-hosted.md +++ b/docs/zh/getting-started/self-hosted.md @@ -1,7 +1,7 @@ --- title: "运行自托管 Session" source: docs/getting-started/self-hosted.md -source_hash: ed3a04fb0421631c5633fc1b46d455d2c50c77d696bab09855b19d1c6d8ee92f +source_hash: dd09af89f5c98bb1b0bf3dfef5f85c9c9068ef3d490b2e1f03fd7691b43d0fb0 --- `self_hosted` Session 在应用拥有的机器上运行，例如工作站、虚拟机或容器，而不是 Core 管理的沙箱。本指南面向用 `oac-daemon` 连接此类机器并保持其运行的应用开发者。 @@ -23,14 +23,14 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../ 安装程序自带固定版本的 Node.js 和 Harness（列于 [`scripts/build-native-installer.mjs`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/scripts/build-native-installer.mjs)），不修改这些工具的其他安装。在没有匹配安装程序的平台上，命令会失败。 -安装程序下载约 320 MiB。Runtime 主目录和安装目录所在磁盘需要有空闲空间： +下载量取决于你安装的 Harness。安装程序（9 MiB）和 Node.js（59 MiB）始终需要，再加上 Codex 的 121 MiB、Claude Code 的 99 MiB 或 MiniMax Code 的 23 MiB，因此安装一个 Harness 的 Session 下载 90 到 189 MiB。Runtime 主目录和安装目录所在磁盘需要有空闲空间： | | 一个 Harness | 全部三个 | | --- | --- | --- | -| 高峰时所需空闲空间 | 约 2 GiB | 约 2.2 GiB | +| 高峰时所需空闲空间 | 约 1.3 GiB | 约 2.1 GiB | | 安装后占用 | 约 560 MiB | 约 940 MiB | -只选一个 Harness 时，高峰由归档（316 MiB）、解包后的安装程序（907 MiB）和安装后的文件（约 560 MiB）构成。解包副本在命令结束时删除，归档在安装完成时删除。 +选 Codex 时，高峰由归档（180 MiB）、解包后的安装程序（约 560 MiB）和安装后的文件（约 560 MiB）构成。解包副本在命令结束时删除。归档保留在 Runtime 主目录的 `native-download` 中，因此这台机器上同一发行版的另一个 Session 只需下载 9 MiB 的安装程序。 机器需要： @@ -39,7 +39,7 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../ - Session 的软件包需要时，安装 Python 和 pip； - 环境设置所需的系统软件包。守护进程不运行 apt、sudo 或其他提权命令，请通过主机的常规管理方式安装。 -不需要管理员权限或 Docker。Unix 下载命令还使用 `curl`、`tar`、`gzip`、SHA-256 工具和系统文件锁命令（Linux 为 `flock`，macOS 为 `lockf`）。Windows 需要系统自带的 `tar.exe`，命令会在下载前检查。Runtime 主目录必须允许执行文件。如果 Unix 挂载点设为 `noexec`，请先用 `OAC_RUNTIME_HOME` 指定另一个允许执行的绝对目录，再运行命令。 +不需要管理员权限或 Docker。Unix 下载命令还使用 `curl`、`tar` 和 SHA-256 工具。Windows 需要系统自带的 `tar.exe`，命令会在下载前检查。Runtime 主目录必须允许执行文件。如果 Unix 挂载点设为 `noexec`，请先用 `OAC_RUNTIME_HOME` 指定另一个允许执行的绝对目录，再运行命令。 ## 连接机器 {#connect-a-machine} @@ -68,7 +68,7 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../ print(installation["commands"]["posix"]) # use "powershell" on Windows ``` -2. 在目标机器上，以应运行工具的账号执行命令。命令下载与 Core 匹配的安装程序、验证校验和、询问要安装哪些 Harness 以及安装位置、完成安装、按需创建工作区、启动守护进程并检查连接。 +2. 在目标机器上，以应运行工具的账号执行命令。命令下载与 Core 匹配的安装程序、验证校验和、询问要安装哪些 Harness 以及安装位置、下载 Node.js 和所选 Harness、完成安装、按需创建工作区、启动守护进程并检查连接。 3. 发送一个 Turn。机器已连接只证明认证成功；第一个 Turn 才会检查 Harness 和模型。 在 Web 中打开 Session，复制 **连接主机** 下的命令。 @@ -88,7 +88,7 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../
下载中断时 -安装程序在下载任何内容之前先检查授权，因此过期的命令不会白下载。归档下载在连接重置后会续传，只在传输停滞（60 秒没有数据）时失败，最多尝试五次；重新执行命令即可从中断处继续。校验通过的归档保存在 Runtime 主目录的 `native-download` 中，直到安装完成，因此失败后重跑无需重新下载；每次运行都会对照 Core 提供的校验和检查它，未通过则丢弃。下载、解压和复制组件前会检查磁盘空间。如果下载或安装被中断，重新执行命令：它清理未完成的临时副本，同时保留已完成的组件、凭据和工作区。`native-download` 中的小型 `download.lock` 文件保留用于并发控制。其他运行不会清理仍在进行的下载或安装。命令过期时，从 Session 复制新命令。 +安装程序在下载 Harness 之前先检查授权，因此过期的命令只会下载 9 MiB 的安装程序。每个归档的下载在连接重置后会续传，只在传输停滞（60 秒没有数据）时失败，最多尝试五次；重新执行命令即可从中断处继续。校验通过的归档保存在 Runtime 主目录的 `native-download/` 中。同一版本的后续运行或 Session 会对照 Core 提供的校验和检查并复用它们，未通过的会被丢弃；其他版本的归档会被删除。同一台机器上的两个安装会轮流下载。下载、解压和复制组件前会检查磁盘空间。如果下载或安装被中断，重新执行命令：它清理未完成的临时副本，同时保留已完成的组件、凭据和工作区。命令过期时，从 Session 复制新命令。
@@ -146,7 +146,7 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../ 没有原目录时，轮换该 Session 的凭据，并用已解压的发行包安装到新目录： 1. 在 Web 中打开该 Session，在 **执行器凭据** 中选择 **轮换**，保存返回的凭据 JSON。 -2. 下载并解压 `/.tar.gz`。`` 是安装命令中的第一个 URL（以 Core 版本结尾），`` 是 `linux-amd64`、`darwin-arm64` 或 `windows-amd64`，对应[平台](#platforms)表中的各行。 +2. 下载 `/.tar.gz`，以及 `node` 和你要安装的每个 Harness（`codex`、`claude` 或 `minimax`）的 `/-.tar.gz`，并解压到同一个目录。`` 是安装命令中的第一个 URL（以 Core 版本结尾），`` 是 `linux-amd64`、`darwin-arm64` 或 `windows-amd64`，对应[平台](#platforms)表中的各行。 3. 按[从已解压的发行包安装](#install-from-an-extracted-distribution)运行安装程序，使用该 Session 的 `remote_url`、Environment ID、工作区和保存的凭据文件。 或者新建 Session 并运行它的命令。 @@ -165,7 +165,7 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../ & "$HOME\.oac\environments\\bin\oac-daemon.exe" uninstall ``` -它会停止守护进程，列出将删除的内容并请求确认。它只删除安装目录中属于该安装的条目（`bin`、`daemon`、`components`、`capabilities`、`runtime`、`initialization` 和 `packages`，含凭据文件、原生历史和 Harness），之后若目录中没有其他内容，再删除目录本身。目录中的其他内容会保留，工作区以及安装目录之外的凭据文件或能力目录也会保留；命令会逐一列出保留的路径。`native-download` 目录及其 `download.lock` 不会被触碰。在没有安装的位置运行时，它会说明这一点并以 0 退出。如果某个文件无法删除，命令会指出它，待其被释放后可再次运行。在 Windows 上，最后的删除会在命令退出几秒后完成。 +它会停止守护进程，列出将删除的内容并请求确认。它只删除安装目录中属于该安装的条目（`bin`、`daemon`、`components`、`capabilities`、`runtime`、`initialization` 和 `packages`，含凭据文件、原生历史和 Harness），之后若目录中没有其他内容，再删除目录本身。目录中的其他内容会保留，工作区以及安装目录之外的凭据文件或能力目录也会保留；命令会逐一列出保留的路径。`native-download` 缓存不会被触碰；如需回收空间，请自行删除。在没有安装的位置运行时，它会说明这一点并以 0 退出。如果某个文件无法删除，命令会指出它，待其被释放后可再次运行。在 Windows 上，最后的删除会在命令退出几秒后完成。 随后在 Web 中打开该 Session，在 **执行器凭据** 中选择 **撤销**，或 **删除** 该 Session；在此之前，Core 仍接受这台机器的凭据。 @@ -220,4 +220,4 @@ session = client.beta.agents.sessions.create( "$HOME/.oac/my-runtime/bin/oac-daemon" start ``` -使用 Session 的 `remote_url` 和 Environment ID。发行包即[重新连接已有 Session](#reconnect-an-existing-session) 中的压缩包。在 PowerShell 中，以原生绝对路径运行 `.\oac-daemon.exe`。安装目录必须不存在、为空，或已包含一个安装。此模式要求工作区已存在，且在运行 `start` 前不会启动守护进程。 +使用 Session 的 `remote_url` 和 Environment ID。发行包即[重新连接已有 Session](#reconnect-an-existing-session) 中描述的目录：平台归档中的 `oac-daemon` 和 `bundle.json`，以及 `node` 和每个所选 Harness 的归档中的 `components/`。缺少组件会失败，并指出需要解压的归档；此模式从不下载。在 PowerShell 中，以原生绝对路径运行 `.\oac-daemon.exe`。安装目录必须不存在、为空，或已包含一个安装。此模式要求工作区已存在，且在运行 `start` 前不会启动守护进程。 diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index 101cd5613..013a1cc94 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: 0180ec977dbb3efee2d1af237420d133ca972426cf8d3b873b80ca45a4387e4c +source_hash: f66f608093cc824add3215a9df520082b1679af1852ee213b7a6adfca40221ed --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请按 [安装 Core 和 Web](getting-started/install.md) 操作。安装程序代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装程序](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -49,14 +49,16 @@ ingress 镜像仅包含 `oac` 和四个节点元数据文件：`manifest.json` ### 原生安装程序 {#native-installers} -自托管机器通过各平台的原生安装程序安装 `oac-daemon`：Linux amd64、macOS arm64 和 Windows amd64。每个安装程序均由 `native-check` 工作流在各自的操作系统上构建（`scripts/build-native-installer.mjs`，其中的 `pins` 对象会固定 Node.js 和 Harness 版本），并在每次选中的原生检查中得到验证。手动打包运行和发布检查会上传 `oac-native-installer--.tar.gz` 并保留七天；普通 PR 和 main 检查即使成功也不会上传软件包。要构建本地分发包，请从该确切提交的一次 `native-check` 运行中下载这三个构件（可以是手动运行或发布运行；PR 运行构建的是合并提交，因此不匹配），然后使用该检出内容组装目录： +自托管机器通过各平台的原生安装程序安装 `oac-daemon`：Linux amd64、macOS arm64 和 Windows amd64。每个安装程序均由 `native-check` 工作流在各自的操作系统上构建（`scripts/build-native-installer.mjs`，其中的 `pins` 对象会固定 Node.js 和 Harness 版本），并在每次选中的原生检查中得到验证。手动打包运行和发布检查会将该平台的各个归档上传为 `oac-native-installer--` 并保留七天；普通 PR 和 main 检查即使成功也不会上传软件包。要构建本地分发包，请从该确切提交的一次 `native-check` 运行中下载这三个构件（可以是手动运行或发布运行；PR 运行构建的是合并提交，因此不匹配），然后使用该检出内容组装目录，输入目录中放入这三个构件的所有归档： ```sh node scripts/build-native-catalog.mjs INPUT_DIR OUTPUT_DIR export OAC_NATIVE_INSTALLER_BUILD_DIR=OUTPUT_DIR ``` -该目录会记录提交、Runtime 协议版本、每个归档的 SHA-256，并在提供发布基址时记录其带版本号的 URL。控制归档和 Core 镜像仅携带 `native-installers/catalog.json`；这些归档会成为独立的 `oac-native--.tar.gz` 发布资源，离线归档则会在 Core 镜像之外保留 Linux amd64 归档的一份副本，由其 `install.py` 复制到安装目录中供 Core 提供下载。其他平台由 Core 重定向到对应的发布资源；没有发布基址的目录没有这类 URL，因此其离线归档会保留所有归档。如果没有目录，Sessions 会将安装命令报告为不可用，并且发布工作流会拒绝发布。 +每个平台有一个小型基础归档 `-.tar.gz`（`oac-daemon` 和 `bundle.json`，后者列出每个组件及其每个文件的 SHA-256），以及每个组件一个归档，即 `--node` 和 `--`（MiniMax Code 不为 Windows 构建）。`build-native-installer.mjs` 生成它们。Core 按名称提供这些制品，不了解其内容；`oac-daemon install` 下载所需的组件。 + +该目录会记录提交、Runtime 协议版本、每个制品的 SHA-256，并在提供发布基址时记录其带版本号的 URL。控制归档和 Core 镜像仅携带 `native-installers/catalog.json`；这些归档会成为独立的 `oac-native--.tar.gz` 发布资源，离线归档则会在 Core 镜像之外保留所有 Linux amd64 归档的一份副本，由其 `install.py` 复制到安装目录中供 Core 提供下载。其他平台由 Core 重定向到对应的发布资源；没有发布基址的目录没有这类 URL，因此其离线归档会保留所有归档。如果没有目录，Sessions 会将安装命令报告为不可用，并且发布工作流会拒绝发布。 ### Runtime 镜像和辅助程序 {#runtime-images-and-helpers} diff --git a/scripts/build-native-catalog.mjs b/scripts/build-native-catalog.mjs index ab97b7e96..688772434 100644 --- a/scripts/build-native-catalog.mjs +++ b/scripts/build-native-catalog.mjs @@ -3,7 +3,7 @@ import { createHash } from 'node:crypto'; import { execFileSync } from 'node:child_process'; import { createReadStream } from 'node:fs'; -import { copyFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { copyFile, mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; const [input, output] = process.argv.slice(2); @@ -12,14 +12,20 @@ const version = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }) const protocol_version = (await readFile('internal/agentdaemon/proto/version.go', 'utf8')).match(/const Version = "([^"]+)"/)?.[1]; if (!protocol_version) throw new Error('Runtime protocol version is missing'); await mkdir(output, { recursive: true }); +// INPUT holds -.tar.gz (the daemon and bundle.json) and --.tar.gz per platform. +const names = (await readdir(input)).filter(file => file.endsWith('.tar.gz')).map(file => file.slice(0, -'.tar.gz'.length)).sort(); const artifacts = {}; -for (const [ci, platform] of Object.entries({ 'Linux-X64': 'linux-amd64', 'macOS-ARM64': 'darwin-arm64', 'Windows-X64': 'windows-amd64' })) { - const archive = resolve(input, `oac-native-installer-${ci}.tar.gz`); - const bundle = JSON.parse(execFileSync('tar', ['-xOzf', archive, './bundle.json'], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })); +for (const platform of ['linux-amd64', 'darwin-arm64', 'windows-amd64']) { + const bundle = JSON.parse(execFileSync('tar', ['-xOzf', resolve(input, `${platform}.tar.gz`), 'bundle.json'], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 })); if (bundle.daemon_version !== version || `${bundle.os}-${bundle.arch}` !== platform) throw new Error(`Mismatched native artifact: ${platform}`); - const hash = createHash('sha256'); - for await (const chunk of createReadStream(archive)) hash.update(chunk); - artifacts[platform] = { sha256: hash.digest('hex') }; - await copyFile(archive, join(output, `${platform}.tar.gz`)); + const parts = Object.keys(bundle.components).map(component => `${platform}-${component}`).sort(); + if (JSON.stringify(names.filter(name => name.startsWith(platform + '-'))) !== JSON.stringify(parts)) throw new Error(`Native archives do not match the components of ${platform}`); + for (const name of [platform, ...parts]) { + const archive = resolve(input, `${name}.tar.gz`); + const hash = createHash('sha256'); + for await (const chunk of createReadStream(archive)) hash.update(chunk); + artifacts[name] = { sha256: hash.digest('hex') }; + await copyFile(archive, join(output, `${name}.tar.gz`)); + } } await writeFile(join(output, 'catalog.json'), JSON.stringify({ version, protocol_version, artifacts }, null, 2)+'\n'); diff --git a/scripts/build-native-catalog.test.mjs b/scripts/build-native-catalog.test.mjs index 9328b3d35..4bcc7ecff 100644 --- a/scripts/build-native-catalog.test.mjs +++ b/scripts/build-native-catalog.test.mjs @@ -6,7 +6,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import test from 'node:test'; -test('catalog accepts large native manifests and still rejects a foreign revision', async () => { +test('catalog lists each platform base and component archive and rejects a foreign revision', async () => { const directory = await mkdtemp(join(tmpdir(), 'oac-native-catalog-')); try { const input = join(directory, 'input'); @@ -14,20 +14,27 @@ test('catalog accepts large native manifests and still rejects a foreign revisio const bundle = join(directory, 'bundle'); await Promise.all([mkdir(input), mkdir(bundle)]); const version = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); - for (const [ci, os, arch] of [['Linux-X64', 'linux', 'amd64'], ['macOS-ARM64', 'darwin', 'arm64'], ['Windows-X64', 'windows', 'amd64']]) { - await writeFile(join(bundle, 'bundle.json'), JSON.stringify({ daemon_version: version, os, arch, files: { fixture: 'x'.repeat(2 * 1024 * 1024) } })); - execFileSync('tar', ['-czf', join(input, `oac-native-installer-${ci}.tar.gz`), '-C', bundle, './bundle.json']); + const pack = (name, daemon_version, os, arch, components) => { + return writeFile(join(bundle, 'bundle.json'), JSON.stringify({ daemon_version, os, arch, components, filler: 'x'.repeat(2 * 1024 * 1024) })).then(() => { + execFileSync('tar', ['-czf', join(input, `${name}.tar.gz`), '-C', bundle, 'bundle.json']); + }); + }; + for (const [os, arch] of [['linux', 'amd64'], ['darwin', 'arm64'], ['windows', 'amd64']]) { + const components = { node: {}, codex: {} }; + await pack(`${os}-${arch}`, version, os, arch, components); + for (const name of Object.keys(components)) await pack(`${os}-${arch}-${name}`, version, os, arch, {}); } const script = resolve('scripts/build-native-catalog.mjs'); execFileSync(process.execPath, [script, input, output]); const catalog = JSON.parse(await readFile(join(output, 'catalog.json'), 'utf8')); assert.equal(catalog.version, version); - assert.equal(Object.keys(catalog.artifacts).length, 3); - const linux = await readFile(join(input, 'oac-native-installer-Linux-X64.tar.gz')); - assert.deepEqual(await readFile(join(output, 'linux-amd64.tar.gz')), linux); - assert.equal(catalog.artifacts['linux-amd64'].sha256, createHash('sha256').update(linux).digest('hex')); - await writeFile(join(bundle, 'bundle.json'), JSON.stringify({ daemon_version: 'foreign', os: 'linux', arch: 'amd64' })); - execFileSync('tar', ['-czf', join(input, 'oac-native-installer-Linux-X64.tar.gz'), '-C', bundle, './bundle.json']); + assert.deepEqual(Object.keys(catalog.artifacts).sort(), ['darwin-arm64', 'darwin-arm64-codex', 'darwin-arm64-node', 'linux-amd64', 'linux-amd64-codex', 'linux-amd64-node', 'windows-amd64', 'windows-amd64-codex', 'windows-amd64-node']); + const part = await readFile(join(input, 'linux-amd64-codex.tar.gz')); + assert.deepEqual(await readFile(join(output, 'linux-amd64-codex.tar.gz')), part); + assert.equal(catalog.artifacts['linux-amd64-codex'].sha256, createHash('sha256').update(part).digest('hex')); + await rm(join(input, 'linux-amd64-codex.tar.gz')); + assert.throws(() => execFileSync(process.execPath, [script, input, output], { stdio: 'pipe' }), error => error.stderr.toString().includes('do not match the components')); + await pack('linux-amd64', 'foreign', 'linux', 'amd64', {}); assert.throws(() => execFileSync(process.execPath, [script, input, output], { stdio: 'pipe' }), error => error.stderr.toString().includes('Mismatched native artifact')); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/scripts/build-native-installer-ci.mjs b/scripts/build-native-installer-ci.mjs index ec874a0f6..a77e46c18 100644 --- a/scripts/build-native-installer-ci.mjs +++ b/scripts/build-native-installer-ci.mjs @@ -5,7 +5,7 @@ import { spawnSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { access, mkdir, readdir, realpath, writeFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; -import { buildBundle } from './build-native-installer.mjs'; +import { archiveBundle, buildBundle } from './build-native-installer.mjs'; const root = resolve(process.env.RUNNER_TEMP ?? ''); if (!process.env.RUNNER_TEMP) throw new Error('RUNNER_TEMP is required'); @@ -30,6 +30,7 @@ const candidates = await findCodex(join(root, 'native-tools', 'node_modules', '@ assert.equal(candidates.length, 1, 'Expected exactly one native Codex artifact'); const bundle = join(root, 'native-installer'); const receipt = await buildBundle({ daemon, node: await realpath(node), codex: candidates[0], claude: join(root, 'claude-runtime'), ...(!windows ? { minimax: join(root, 'minimax-runtime') } : {}), output: bundle }); +await archiveBundle(bundle, join(root, 'native-archives')); console.log(JSON.stringify({ stage: 'bundle', os: receipt.os, arch: receipt.arch, components: Object.keys(receipt.components), model_requests: 0 })); const workspace = join(root, 'native-install-workspace'); await mkdir(workspace); const credential = join(root, 'native-install-credential.json'); diff --git a/scripts/build-native-installer.mjs b/scripts/build-native-installer.mjs index 2060f7b66..55c7f4e04 100644 --- a/scripts/build-native-installer.mjs +++ b/scripts/build-native-installer.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // Package trusted, already-built native artifacts. This does not install on a host. import { createHash } from 'node:crypto'; -import { spawnSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { chmod, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, stat, writeFile } from 'node:fs/promises'; import { basename, dirname, isAbsolute, join, relative, sep } from 'node:path'; import { createRequire } from 'node:module'; @@ -178,3 +178,14 @@ export async function buildBundle(options) { return manifest; } finally { await rm(staging, { recursive: true, force: true }); } } + +// Split a built bundle into the small base archive (the daemon and bundle.json) and one archive +// per component. Names are the artifact names Core serves: -[-]. +export async function archiveBundle(bundle, output) { + const { os, arch, components } = await json(join(bundle, 'bundle.json')); + const tar = process.platform === 'win32' ? join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar'; + const pack = (name, entries) => execFileSync(tar, ['-czf', join(output, `${name}.tar.gz`), '-C', bundle, ...entries], { env: { ...process.env, COPYFILE_DISABLE: '1' } }); + await mkdir(output, { recursive: true }); + pack(`${os}-${arch}`, [process.platform === 'win32' ? 'oac-daemon.exe' : 'oac-daemon', 'bundle.json']); + for (const name of Object.keys(components)) pack(`${os}-${arch}-${name}`, [`components/${name}`]); +} diff --git a/scripts/build-native-installer.test.mjs b/scripts/build-native-installer.test.mjs index 8ffa55b1f..700ea462c 100644 --- a/scripts/build-native-installer.test.mjs +++ b/scripts/build-native-installer.test.mjs @@ -4,7 +4,8 @@ import { chmod, copyFile, cp, lstat, mkdir, mkdtemp, readFile, rename, rm, symli import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import test from 'node:test'; -import { buildBundle, copyComponent, prepareNodeEntrypoints, validateNodeCommands } from './build-native-installer.mjs'; +import { execFileSync } from 'node:child_process'; +import { archiveBundle, buildBundle, copyComponent, prepareNodeEntrypoints, validateNodeCommands } from './build-native-installer.mjs'; async function fixture(t) { const root = await mkdtemp(join(tmpdir(), 'oac-native-bundle-test-')); @@ -95,3 +96,19 @@ test('real npm and npx remain runnable from PATH after regular-file copying and assert.equal(files[path].executable, true); } }); + +test('archives hold the daemon alone and each component separately', async t => { + const { root, source } = await fixture(t); + const windows = process.platform === 'win32'; + const daemon = windows ? 'oac-daemon.exe' : 'oac-daemon'; + await writeFile(join(source, daemon), 'daemon'); + await writeFile(join(source, 'bundle.json'), JSON.stringify({ os: 'linux', arch: 'amd64', components: { node: {}, codex: {} } })); + for (const name of ['node', 'codex']) { await mkdir(join(source, 'components', name), { recursive: true }); await writeFile(join(source, 'components', name, 'file'), name); } + const output = join(root, 'archives'); + await archiveBundle(source, output); + const tar = windows ? join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar'; + const list = name => execFileSync(tar, ['-tzf', join(output, `${name}.tar.gz`)], { encoding: 'utf8' }).split(/\r?\n/).filter(Boolean).map(entry => entry.replace(/\/$/, '')).sort(); + assert.deepEqual(list('linux-amd64'), ['bundle.json', daemon].sort()); + assert.deepEqual(list('linux-amd64-codex'), ['components/codex', 'components/codex/file']); + assert.deepEqual(list('linux-amd64-node'), ['components/node', 'components/node/file']); +}); diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index 7aa2b3857..fa9ad6de0 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -238,6 +238,10 @@ def release_base(value): return value.rstrip("/") +# A platform base archive or one named part of it. +NATIVE_ARTIFACT = re.compile(r"(linux|darwin|windows)-(amd64|arm64)(-[a-z]+)?") + + def native_catalog(bundle, stage, revision, source, artifact_base_url=""): """Ship only catalog metadata in Core; publish native archives independently.""" bundle, stage, source = pathlib.Path(bundle), pathlib.Path(stage), pathlib.Path(source) @@ -249,16 +253,16 @@ def native_catalog(bundle, stage, revision, source, artifact_base_url=""): raise ValueError("Native catalog has no qualified artifacts") assets = stage / "native-artifacts" assets.mkdir() - for platform, entry in catalog["artifacts"].items(): - if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): - raise ValueError("Invalid native installer platform") - archive = source / (platform + ".tar.gz") + for name, entry in catalog["artifacts"].items(): + if not NATIVE_ARTIFACT.fullmatch(name): + raise ValueError("Invalid native installer artifact") + archive = source / (name + ".tar.gz") if archive.is_symlink() or sha256(archive) != entry["sha256"]: raise ValueError("Native installer checksum mismatch") - filename = f"oac-native-{revision}-{platform}.tar.gz" + filename = f"oac-native-{revision}-{name}.tar.gz" shutil.copyfile(archive, assets / filename) (assets / (filename + ".sha256")).write_text(entry["sha256"] + " " + filename + "\n") - catalog["artifacts"][platform] = {"sha256": entry["sha256"], **({"url": base + "/" + filename} if base else {})} + catalog["artifacts"][name] = {"sha256": entry["sha256"], **({"url": base + "/" + filename} if base else {})} for directory in (stage / "core/native-installers", bundle / "native-installers"): directory.mkdir(parents=True, exist_ok=True) (directory / "catalog.json").write_text(json.dumps(catalog, indent=2) + "\n") @@ -271,10 +275,10 @@ def native_offline(bundle, stage): if not path.exists(): return catalog = json.loads(path.read_text()) - for platform, entry in catalog["artifacts"].items(): - if platform == "linux-amd64" or "url" not in entry: - source = stage / "native-artifacts" / f"oac-native-{catalog['version']}-{platform}.tar.gz" - os.link(source, path.parent / (platform + ".tar.gz")) + for name, entry in catalog["artifacts"].items(): + if name.startswith("linux-amd64") or "url" not in entry: + source = stage / "native-artifacts" / f"oac-native-{catalog['version']}-{name}.tar.gz" + os.link(source, path.parent / (name + ".tar.gz")) @contextmanager diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index 4f2ec7901..c2f3cf679 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -51,11 +51,11 @@ def descriptor(data, media_type, label): class DistributionTests(unittest.TestCase): - def test_native_payload_is_independent_and_offline_has_only_linux(self): + def test_native_payload_is_independent_and_offline_has_only_linux_archives(self): source = self.stage / "qualified-native" source.mkdir() catalog = {"version": REVISION, "protocol_version": "fixture", "artifacts": {}} - for platform in ("linux-amd64", "darwin-arm64", "windows-amd64"): + for platform in ("linux-amd64", "linux-amd64-codex", "darwin-arm64", "darwin-arm64-codex", "windows-amd64"): raw = ("native:" + platform).encode() (source / (platform + ".tar.gz")).write_bytes(raw) catalog["artifacts"][platform] = {"sha256": hashlib.sha256(raw).hexdigest()} @@ -74,8 +74,9 @@ def test_native_payload_is_independent_and_offline_has_only_linux(self): distribution.archive(self.bundle, "1", "offline") with tarfile.open(self.bundle.with_name(self.bundle.name + "-offline.tar.gz")) as archive: native = [p for p in archive.getmembers() if "native-installers/" in p.name and p.name.endswith(".tar.gz")] - self.assertEqual([pathlib.Path(p.name).name for p in native], ["linux-amd64.tar.gz"]) - self.assertEqual(archive.extractfile(native[0]).read(), (source / "linux-amd64.tar.gz").read_bytes()) + self.assertEqual(sorted(pathlib.Path(p.name).name for p in native), ["linux-amd64-codex.tar.gz", "linux-amd64.tar.gz"]) + for member in native: + self.assertEqual(archive.extractfile(member).read(), (source / pathlib.Path(member.name).name).read_bytes()) metadata = json.loads((self.bundle / "native-installers/catalog.json").read_text()) for platform, entry in metadata["artifacts"].items(): filename = f"oac-native-{REVISION}-{platform}.tar.gz" diff --git a/scripts/native-onboarding-smoke.mjs b/scripts/native-onboarding-smoke.mjs index 9bee4193f..96ab637a5 100644 --- a/scripts/native-onboarding-smoke.mjs +++ b/scripts/native-onboarding-smoke.mjs @@ -4,9 +4,8 @@ import assert from 'node:assert/strict'; import { createHash, randomUUID } from 'node:crypto'; import { createServer } from 'node:http'; import { spawn, execFileSync } from 'node:child_process'; -import { createReadStream } from 'node:fs'; -import { existsSync } from 'node:fs'; -import { mkdir, readFile } from 'node:fs/promises'; +import { createReadStream, existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; const root = process.env.RUNNER_TEMP; @@ -18,19 +17,30 @@ const protocol = (await readFile('internal/agentdaemon/proto/version.go', 'utf8' const environment = randomUUID(), session = randomUUID(), device = randomUUID(); const workspace = join(root, 'onboarding-workspace'); const installation = join(root, 'onboarding-installation'); -const archive = join(root, 'onboarding.tar.gz'); -const tar = windows ? join(process.env.SystemRoot, 'System32', 'tar.exe') : 'tar'; -execFileSync(tar, ['-czf', archive, '-C', bundle, '.']); -const digest = createHash('sha256'); -for await (const chunk of createReadStream(archive)) digest.update(chunk); -const checksum = digest.digest('hex'); +// Core serves the base archive and one archive per component, as the release publishes them. +const archives = join(root, 'native-archives'); +const platform = `${manifest.os}-${manifest.arch}`; +const checksums = new Map(), downloaded = new Set(); +async function checksum(name) { + if (!checksums.has(name)) { + const digest = createHash('sha256'); + for await (const chunk of createReadStream(join(archives, `${name}.tar.gz`))) digest.update(chunk); + checksums.set(name, digest.digest('hex')); + } + return checksums.get(name); +} let secret, connected = false, connections = 0, failClaim = true; const authorization = 'fixture-install-authorization'; const server = createServer(async (request, response) => { const url = new URL(request.url, origin); const json = (status, value) => { response.writeHead(status, { 'Content-Type': 'application/json' }); response.end(JSON.stringify(value)); }; - if (url.pathname.endsWith('.sha256')) { response.setHeader('Content-Type', 'text/plain; charset=utf-8'); return response.end(checksum+'\n'); } - if (url.pathname.endsWith('.tar.gz')) return createReadStream(archive).pipe(response); + const artifact = url.pathname.match(/\/install\/[^/]+\/([a-z0-9-]+)\.(sha256|tar\.gz)$/); + if (artifact) { + if (!existsSync(join(archives, `${artifact[1]}.tar.gz`))) return json(404, {}); + if (artifact[2] === 'sha256') { response.setHeader('Content-Type', 'text/plain; charset=utf-8'); return response.end(await checksum(artifact[1])+'\n'); } + downloaded.add(artifact[1]); + return createReadStream(join(archives, `${artifact[1]}.tar.gz`)).pipe(response); + } if (url.pathname.endsWith('bootstrap.sh') || url.pathname.endsWith('bootstrap.ps1')) return createReadStream(resolve('services/core/internal/nativeinstaller/assets', url.pathname.split('/').at(-1))).pipe(response); const body = []; for await (const chunk of request) body.push(chunk); if (url.pathname.endsWith('/installation') || url.pathname.endsWith('/claim')) { @@ -87,9 +97,9 @@ try { const result = await run(windows ? 'powershell.exe' : 'bash', [...bootstrapArgs, '--install-dir', installation], '\n\n'); assert.equal(result.code, 0, `Interactive bootstrap failed: ${result.output}`); assert.match(result.output, /Daemon connection: connected/); + assert.deepEqual([...downloaded].sort(), [platform, `${platform}-codex`, `${platform}-node`], 'Only the base, Node and the Session Harness may be downloaded'); assert.match(result.output, /Model configuration: not checked/); assert.equal(connections, 1); - // The running daemon must not keep the bootstrap's download lock. const repeat = await run(windows ? 'powershell.exe' : 'bash', [...bootstrapArgs, '--install-dir', installation, '--non-interactive', '--harness', 'codex']); assert.equal(repeat.code, 0, `Repeat failed: ${repeat.output}`); assert.equal(connections, 1, 'Repeated installation created a duplicate daemon'); diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index c480919db..c31e5b34f 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -268,10 +268,10 @@ def publish(assets, repository, revision, tag, mode): catalog = json.load(archive.extractfile(stem + "/native-installers/catalog.json")) if catalog["version"] != revision or not catalog["artifacts"]: raise ValueError("Native installer catalog does not match the release") - for platform, entry in catalog["artifacts"].items(): - if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): - raise ValueError("Invalid native installer platform") - path = assets / f"oac-native-{revision}-{platform}.tar.gz" + for name, entry in catalog["artifacts"].items(): + if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)(-[a-z]+)?", name): + raise ValueError("Invalid native installer artifact") + path = assets / f"oac-native-{revision}-{name}.tar.gz" if (distribution.sha256(path) != entry["sha256"] or path.with_name(path.name + ".sha256").read_text() != entry["sha256"] + " " + path.name + "\n"): raise ValueError("Native installer checksum mismatch") diff --git a/scripts/publish-core-release.test.py b/scripts/publish-core-release.test.py index b7870eef7..584b2e4b8 100644 --- a/scripts/publish-core-release.test.py +++ b/scripts/publish-core-release.test.py @@ -34,7 +34,7 @@ def setUp(self): (self.assets / (name + ".sha256")).write_text( hashlib.sha256(b"archive fixture").hexdigest() + " " + name + "\n") catalog = {"version": self.revision, "artifacts": {}} - for platform in ("linux-amd64", "darwin-arm64", "windows-amd64"): + for platform in ("linux-amd64", "darwin-arm64", "windows-amd64-codex"): name = f"oac-native-{self.revision}-{platform}.tar.gz" (self.assets / name).write_bytes(b"native archive") checksum = hashlib.sha256(b"native archive").hexdigest() @@ -71,7 +71,7 @@ def test_draft_publishes_images_and_stays_unpublished(self): self.assertEqual(len(self.release["assets"]), 28) def test_missing_native_asset_refuses_release_creation(self): - (self.assets / f"oac-native-{self.revision}-windows-amd64.tar.gz").unlink() + (self.assets / f"oac-native-{self.revision}-windows-amd64-codex.tar.gz").unlink() with self.assertRaises(FileNotFoundError): self.publish() self.assertEqual(self.writes(), []) diff --git a/services/core/internal/nativeinstaller/assets/bootstrap.ps1 b/services/core/internal/nativeinstaller/assets/bootstrap.ps1 index c04665665..cbbc03e28 100644 --- a/services/core/internal/nativeinstaller/assets/bootstrap.ps1 +++ b/services/core/internal/nativeinstaller/assets/bootstrap.ps1 @@ -3,214 +3,46 @@ param( [Parameter(ValueFromRemainingArguments=$true)][string[]]$InstallArguments ) $ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' # Only the installer receives the grant; every other program started below runs without it. $grant = $env:OAC_INSTALL_GRANT Remove-Item Env:OAC_INSTALL_GRANT -ErrorAction SilentlyContinue $tar = Join-Path $env:SystemRoot 'System32\tar.exe' if (!(Test-Path -LiteralPath $tar -PathType Leaf)) { throw 'Windows tar.exe is required. Install the Windows archive tools, then rerun this command.' } -Add-Type -AssemblyName System.Net.Http -if (!("OacNativeDownloadSpace" -as [type])) { - Add-Type -TypeDefinition @' -using System; -using System.ComponentModel; -using System.Runtime.InteropServices; -public static class OacNativeDownloadSpace { - [DllImport("kernel32.dll", CharSet=CharSet.Unicode, SetLastError=true)] - static extern bool GetDiskFreeSpaceEx(string path, out ulong available, out ulong total, out ulong free); - public static ulong Available(string path) { - ulong available, total, free; - if (!GetDiskFreeSpaceEx(path, out available, out total, out free)) throw new Win32Exception(Marshal.GetLastWin32Error()); - return available; - } -} -'@ -} $architecture = @{x64='amd64';arm64='arm64'}[[System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant()] if (!$architecture) { throw 'Unsupported processor architecture.' } $root = if ($env:OAC_RUNTIME_HOME) { $env:OAC_RUNTIME_HOME } else { Join-Path $HOME '.oac' } if (![IO.Path]::IsPathRooted($root)) { throw 'OAC_RUNTIME_HOME must be an absolute directory.' } -$cache = Join-Path ([IO.Path]::GetFullPath($root)) 'native-download' -$work = Join-Path $cache 'staging' -$lock = $null -$client = $null -$ownsStaging = $false -$terminal = ![Console]::IsOutputRedirected -$ProgressPreference = if ($terminal) { 'Continue' } else { 'SilentlyContinue' } -function Assert-OrdinaryPath([string]$Path) { - if ((Test-Path -LiteralPath $Path) -and ((Get-Item -Force -LiteralPath $Path).Attributes -band [IO.FileAttributes]::ReparsePoint)) { - throw 'Native download paths must not be symbolic links or junctions.' - } -} -function Assert-Space([long]$Required) { - $available = [OacNativeDownloadSpace]::Available($cache) - if ($available -lt ($Required + 64MB)) { throw 'Not enough disk space for the native installer. Free space in the Runtime home and retry.' } -} -function Show-Bytes([string]$Label, [long]$Bytes, [long]$Total) { - if (!$terminal) { return } - $percent = if ($Total -gt 0) { [Math]::Min(100, [int](100.0 * $Bytes / $Total)) } else { -1 } - Write-Progress -Activity $Label -Status ("{0:N1} MiB" -f ($Bytes / 1MB)) -PercentComplete $percent -} -# Resume after resets and fail only when a transfer stalls, with no total time cap. A partial file is kept in the download directory for the next run. -function Get-NativeFile([string]$Uri, [string]$Destination, [long]$Limit) { - for ($attempt=1; $attempt -le 5; $attempt++) { - $request = $null; $response = $null; $inputStream = $null; $outputStream = $null; $retryable = $true - try { - [long]$offset = 0 - if (Test-Path -LiteralPath $Destination) { $offset = (Get-Item -LiteralPath $Destination).Length } - $address = [Uri]$Uri - for ($redirect=0; $redirect -le 5; $redirect++) { - $request = [Net.Http.HttpRequestMessage]::new([Net.Http.HttpMethod]::Get, $address) - if ($offset -gt 0) { $request.Headers.Range = [Net.Http.Headers.RangeHeaderValue]::new($offset, $null) } - $response = $client.SendAsync($request, [Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult() - $request.Dispose(); $request = $null - $status = [int]$response.StatusCode - if ($status -notin @(301,302,303,307,308)) { break } - if ($redirect -eq 5 -or !$response.Headers.Location) { $retryable=$false; throw 'Installer download has too many or invalid redirects.' } - $next = [Uri]::new($address, $response.Headers.Location) - if ($next.Scheme -ne 'https' -or $next.UserInfo) { $retryable=$false; throw 'Installer download redirect must use HTTPS without credentials.' } - $response.Dispose(); $response=$null; $address=$next - } - if ($status -eq 416) { Remove-Item -LiteralPath $Destination -Force; throw 'The server cannot resume the earlier download; starting it again.' } - if ($status -notin @(200,206)) { - $retryable = $status -in @(408,429,500,502,503,504) - if ($status -eq 404) { throw 'This Core has no qualified installer for this platform.' } - throw "Installer download failed (HTTP $status). Check Core and the download host." - } - # A 200 answer ignores the Range header, so start the file again. - $mode = [IO.FileMode]::Append - if ($status -eq 200) { $mode = [IO.FileMode]::Create; $offset = 0 } - $total = $response.Content.Headers.ContentLength - if ($null -eq $total) { $total = 0 } - $retryable=$false - if ($offset + $total -gt $Limit) { throw 'Installer download exceeds the available space or metadata size limit.' } - Assert-Space $total - $outputStream = [IO.File]::Open($Destination, $mode, [IO.FileAccess]::Write, [IO.FileShare]::None) - $retryable=$true - $inputStream = $response.Content.ReadAsStreamAsync().GetAwaiter().GetResult() - $buffer = New-Object byte[] 65536 - [long]$received = 0 - $clock = [Diagnostics.Stopwatch]::StartNew() - [long]$lastProgress=0 - while ($true) { - $cancel = [Threading.CancellationTokenSource]::new() - try { - $cancel.CancelAfter(60000) - $pending = $inputStream.ReadAsync($buffer, 0, $buffer.Length, $cancel.Token) - if (!$pending.Wait(60000)) { throw 'Installer download stalled; retry when the connection is available.' } - $read = $pending.GetAwaiter().GetResult() - } finally { $cancel.Dispose() } - if ($read -eq 0) { break } - $received += $read - $retryable=$false - if ($offset + $received -gt $Limit) { throw 'Installer download exceeds the available space or metadata size limit.' } - $outputStream.Write($buffer,0,$read) - $retryable=$true - if ($clock.ElapsedMilliseconds - $lastProgress -ge 200) { Show-Bytes 'Downloading installer' ($offset + $received) ($offset + $total); $lastProgress=$clock.ElapsedMilliseconds } - } - if ($total -gt 0 -and $received -ne $total) { throw 'Installer download was interrupted.' } - $retryable=$false - $outputStream.Flush($true) - return - } catch { - if (!$retryable) { throw } - if ($attempt -eq 5) { throw 'Installer download failed after five attempts. Check the network to Core and GitHub and any proxy settings, then rerun the command; downloaded data is kept and resumed.' } - Write-Host "Download interrupted; retrying ($attempt/4)..." +function Get-NativeFile([string]$Uri, [string]$Destination) { + for ($attempt=1; $attempt -le 3; $attempt++) { + try { Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Destination -ErrorAction Stop; return } + catch { + if ($_.Exception.Response -and [int]$_.Exception.Response.StatusCode -eq 404) { throw 'This Core has no qualified installer for this platform.' } + if ($attempt -eq 3) { throw 'Installer download failed. Check the network to Core and GitHub (set HTTPS_PROXY if a proxy is required), then rerun the command.' } + Write-Host "Download interrupted; retrying ($attempt/2)..." Start-Sleep -Seconds $attempt - } finally { - if ($inputStream) { $inputStream.Dispose() } - if ($outputStream) { $outputStream.Dispose() } - if ($request) { $request.Dispose() } - if ($response) { $response.Dispose() } - if ($terminal) { Write-Progress -Activity 'Downloading installer' -Completed } } } } -function Get-ArchiveHash([string]$Path) { (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant() } -# Reject a rejected grant before any large download. Only a 401 stops here; any other answer is reported by the installer. -function Test-Grant { - if (!$grant) { return } - $rejected = $false - try { - $message = [Net.Http.HttpRequestMessage]::new([Net.Http.HttpMethod]::Post, ($Base -replace '/install/[^/]+$', '/installation')) - $message.Headers.Authorization = [Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $grant) - $answer = $client.SendAsync($message).GetAwaiter().GetResult() - $rejected = [int]$answer.StatusCode -eq 401 - } catch { } finally { if ($message) { $message.Dispose() }; if ($answer) { $answer.Dispose() } } - if ($rejected) { throw 'Installation authorization rejected or expired; copy a fresh command from the Session.' } -} +# The installer downloads the Harness archives into the download directory and extracts them beside the executable. +$cache = Join-Path ([IO.Path]::GetFullPath($root)) 'native-download' +$work = Join-Path $cache ('bundle.' + [Guid]::NewGuid().ToString('N').Substring(0, 8)) try { - Assert-OrdinaryPath $cache - [IO.Directory]::CreateDirectory($cache) | Out-Null - $lockPath = Join-Path $cache 'download.lock' - Assert-OrdinaryPath $lockPath - try { $lock = [IO.File]::Open($lockPath, [IO.FileMode]::OpenOrCreate, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None) } - catch { throw 'Cannot lock native downloads. Another download may be running; check permissions or wait and retry.' } - Assert-OrdinaryPath $work - if (Test-Path -LiteralPath $work) { - # A killed PowerShell host can leave its native child alive. Do not remove its source bundle. - $children = Get-CimInstance Win32_Process -Filter "Name = 'tar.exe' OR Name = 'oac-daemon.exe'" - foreach ($child in $children) { - if (($child.ExecutablePath -and $child.ExecutablePath.StartsWith($work, [StringComparison]::OrdinalIgnoreCase)) -or ($child.CommandLine -and $child.CommandLine.IndexOf($work, [StringComparison]::OrdinalIgnoreCase) -ge 0)) { - throw 'A native installer is still using the download directory; wait for it to finish and retry.' - } - } - Remove-Item -LiteralPath $work -Recurse -Force - } [IO.Directory]::CreateDirectory($work) | Out-Null - $ownsStaging = $true - $handler = [Net.Http.HttpClientHandler]::new() - $handler.AllowAutoRedirect = $false - $client = [Net.Http.HttpClient]::new($handler) - $client.Timeout = [TimeSpan]::FromSeconds(15) - Test-Grant - Assert-Space 0 Write-Host 'Downloading the installer matched to Core...' - $checksum = Join-Path $work 'checksum' - Get-NativeFile "$Base/windows-$architecture.sha256" $checksum 1024 - $expected = [IO.File]::ReadAllText($checksum).Trim() + Get-NativeFile "$Base/windows-$architecture.sha256" (Join-Path $work 'checksum') + $expected = ([IO.File]::ReadAllText((Join-Path $work 'checksum'))).Trim() if ($expected -cnotmatch '^[0-9a-f]{64}$') { throw 'Core returned an invalid installer checksum.' } - # The verified archive outlives this run so a later failure, such as a wrong option, needs no new download; it is removed once the installation completes. - $archive = Join-Path $cache "$expected.tar.gz" - Get-ChildItem -LiteralPath $cache -Filter '*.tar.gz*' -File | Where-Object { $_.Name -ne "$expected.tar.gz" -and $_.Name -ne "$expected.tar.gz.part" } | Remove-Item -Force - if (Test-Path -LiteralPath $archive) { - Write-Host 'Verifying the installer archive from an earlier run...' - if ((Get-ArchiveHash $archive) -ne $expected) { Remove-Item -LiteralPath $archive -Force; Write-Host 'The earlier archive failed verification and was discarded.' } - } - if (!(Test-Path -LiteralPath $archive)) { - $part = "$archive.part" - if ((Test-Path -LiteralPath $part) -and (Get-Item -LiteralPath $part).Length -gt 0) { Write-Host 'Resuming the earlier download...' } - $available = [OacNativeDownloadSpace]::Available($cache) - Get-NativeFile "$Base/windows-$architecture.tar.gz" $part ($available-64MB) - Write-Host 'Verifying the installer archive...' - if ((Get-ArchiveHash $part) -ne $expected) { Remove-Item -LiteralPath $part -Force; throw 'Installer checksum mismatch; the download was discarded. Rerun the command to download it again.' } - Move-Item -LiteralPath $part -Destination $archive - } - Write-Host 'Checking extraction space...' - $file = [IO.File]::OpenRead($archive) - $gzip = [IO.Compression.GZipStream]::new($file,[IO.Compression.CompressionMode]::Decompress) - try { - $buffer = New-Object byte[] 65536 - [long]$unpacked=0 - while (($count=$gzip.Read($buffer,0,$buffer.Length)) -gt 0) { $unpacked += $count } - Assert-Space $unpacked - } finally { $gzip.Dispose(); $file.Dispose() } - $bundle = Join-Path $work 'bundle' - [IO.Directory]::CreateDirectory($bundle) | Out-Null - Write-Host 'Extracting the installer...' - & $tar -xzf $archive -C $bundle + $archive = Join-Path $work 'installer.tar.gz' + Get-NativeFile "$Base/windows-$architecture.tar.gz" $archive + if ((Get-FileHash -Algorithm SHA256 -LiteralPath $archive).Hash.ToLowerInvariant() -ne $expected) { throw 'Installer checksum mismatch; the download was discarded. Rerun the command to download it again.' } + & $tar -xzf $archive -C $work if ($LASTEXITCODE -ne 0) { throw 'Installer extraction failed. Check disk space, quota and filesystem permissions.' } - $endpoint = $Base -replace '/install/[^/]+$', '/installation' - Write-Host 'Starting installation...' + Remove-Item -LiteralPath $archive -Force $env:OAC_INSTALL_GRANT = $grant - try { & (Join-Path $bundle 'oac-daemon.exe') install --onboard-url $endpoint @InstallArguments } + try { & (Join-Path $work 'oac-daemon.exe') install --onboard-url ($Base -replace '/install/[^/]+$', '/installation') @InstallArguments } finally { Remove-Item Env:OAC_INSTALL_GRANT -ErrorAction SilentlyContinue } if ($LASTEXITCODE -ne 0) { throw 'Installation or connection failed; follow the installer guidance and retry.' } - Remove-Item -LiteralPath $archive -Force } finally { - if ($client) { $client.Dispose() } - if ($lock) { - # Only clean staging created by this invocation, after its native child returned. - try { if ($ownsStaging -and (Test-Path -LiteralPath $work)) { Remove-Item -LiteralPath $work -Recurse -Force } } - finally { $lock.Dispose() } - } + Remove-Item -LiteralPath $work -Recurse -Force -ErrorAction SilentlyContinue } diff --git a/services/core/internal/nativeinstaller/assets/bootstrap.sh b/services/core/internal/nativeinstaller/assets/bootstrap.sh index 8436104c1..b43b57927 100644 --- a/services/core/internal/nativeinstaller/assets/bootstrap.sh +++ b/services/core/internal/nativeinstaller/assets/bootstrap.sh @@ -8,9 +8,8 @@ unset OAC_INSTALL_GRANT fail() { printf '%s\n' "$*" >&2; exit 1; } case "$(uname -s)" in Linux) os=linux;; Darwin) os=darwin;; *) fail 'Unsupported operating system.';; esac case "$(uname -m)" in x86_64) arch=amd64;; arm64|aarch64) arch=arm64;; *) fail 'Unsupported processor architecture.';; esac -for tool in curl tar gzip df awk wc; do command -v "$tool" >/dev/null || fail "Required command missing: $tool"; done +for tool in curl tar; do command -v "$tool" >/dev/null || fail "Required command missing: $tool"; done if command -v sha256sum >/dev/null; then hash=(sha256sum); else hash=(shasum -a 256); command -v shasum >/dev/null || fail 'Required command missing: shasum'; fi -if [ "$os" = darwin ]; then command -v lockf >/dev/null || fail 'Required command missing: lockf'; else command -v flock >/dev/null || fail 'Required command missing: flock (util-linux)'; fi umask 077 root=${OAC_RUNTIME_HOME:-${HOME:?HOME must be set}/.oac} case "$root" in /*) ;; *) fail 'OAC_RUNTIME_HOME must be an absolute directory.';; esac @@ -19,17 +18,8 @@ root=$(cd "$root" && pwd -P) cache=$root/native-download [ ! -L "$cache" ] || fail 'Native download directory must not be a symbolic link.' mkdir -p "$cache" -[ -d "$cache" ] && [ -O "$cache" ] || fail 'Native download directory must belong to the current account.' -chmod 700 "$cache" -lock=$cache/download.lock -[ ! -L "$lock" ] && { [ ! -e "$lock" ] || { [ -f "$lock" ] && [ -O "$lock" ]; }; } || fail 'Invalid native download lock.' -# Children inherit this descriptor, so killing only the shell cannot expose an active download. -exec 9>>"$lock" -if [ "$os" = darwin ]; then lockf -s -t 0 9; else flock -n 9; fi || fail 'Another native download is running; wait for it to finish and retry.' -work=$cache/staging -[ ! -L "$work" ] && { [ ! -e "$work" ] || { [ -d "$work" ] && [ -O "$work" ]; }; } || fail 'Invalid native download staging directory.' -rm -rf "$work" -mkdir "$work" +# The installer downloads the Harness archives into this directory and extracts them beside the executable. +work=$(mktemp -d "$cache/bundle.XXXXXX") || fail 'Cannot write the native download directory.' active= cleanup() { rm -rf "$work"; } interrupt() { @@ -47,91 +37,38 @@ run() { active= return "$result" } -space() { - local available - available=$(df -Pk "$work" | awk 'NR==2 {print $4}') - case "$available" in ''|*[!0-9]*) fail 'Cannot determine free disk space.';; esac - [ "$available" -ge "$(( ($1 + 67108864 + 1023) / 1024 ))" ] || fail 'Not enough disk space for the native installer. Free space in the Runtime home and retry.' -} progress=(-sS) if [ -t 2 ]; then progress=(--progress-bar --show-error); fi -# Resume after resets and fail only when a transfer stalls, with no total time cap. A partial file is kept in the download directory for the next run. fetch() { - local url=$1 destination=$2 limit=$3 status code attempt - for attempt in 1 2 3 4 5; do + local attempt code status + for attempt in 1 2 3; do code=0 - run curl "${progress[@]}" --fail --location --proto-redir =https --connect-timeout 15 --speed-time 60 --speed-limit 1024 --continue-at - --max-filesize "$limit" --write-out '%{http_code}' "$url" -o "$destination" >"$work/http-status" || code=$? - status=$(cat "$work/http-status") + run curl "${progress[@]}" --fail --location --proto-redir =https --connect-timeout 15 --speed-time 60 --speed-limit 1024 --write-out '%{http_code}' "$1" -o "$2" >"$work/http-status" || code=$? [ "$code" -ne 0 ] || return 0 - case "$status" in 404) fail 'This Core has no qualified installer for this platform.';; esac - case "$code" in - 23) fail 'Cannot write the installer download. Check disk space, quota and directory permissions.';; - 63) fail 'Installer download exceeds the available space or metadata size limit.';; - 33) rm -f "$destination";; # The server cannot resume; start the file again. - esac - case "$code:$status" in - 22:416) return 0;; # The partial file already holds the whole download; the checksum decides. - 6:*|7:*|16:*|18:*|28:*|33:*|35:*|52:*|55:*|56:*|92:*|22:408|22:429|22:500|22:502|22:503|22:504) - if [ "$attempt" -lt 5 ]; then printf 'Download interrupted; retrying (%s/4)...\n' "$attempt" >&2; run sleep "$attempt"; continue; fi;; - esac - fail "Installer download failed (HTTP ${status:-unknown}, curl $code). Check the network to Core and GitHub (set HTTPS_PROXY if a proxy is required), then rerun the command; downloaded data is kept and resumed." + status=$(cat "$work/http-status") + [ "$status" != 404 ] || fail 'This Core has no qualified installer for this platform.' + [ "$code" -ne 23 ] || fail 'Cannot write the installer download. Check disk space, quota and directory permissions.' + [ "$attempt" -lt 3 ] || break + printf 'Download interrupted; retrying (%s/2)...\n' "$attempt" >&2 + run sleep "$attempt" done + fail "Installer download failed (HTTP ${status:-unknown}, curl $code). Check the network to Core and GitHub (set HTTPS_PROXY if a proxy is required), then rerun the command." } -# Reject a rejected grant before any large download. Only a 401 stops here; any other answer is reported by the installer. -check_grant() { - [ -n "$grant" ] || return 0 - local status - printf 'header = "Authorization: Bearer %s"\n' "$grant" | run curl -sS --config - --request POST --connect-timeout 15 --max-time 30 --write-out '%{http_code}' --output /dev/null "${base%/install/*}/installation" >"$work/http-status" 2>/dev/null || : - status=$(cat "$work/http-status") - [ "$status" != 401 ] || fail 'Installation authorization rejected or expired; copy a fresh command from the Session.' -} -verified() { - run "${hash[@]}" "$1" >"$work/digest" || return 1 - local actual rest - read -r actual rest <"$work/digest" - [ "$actual" = "$expected" ] -} -space 0 -check_grant printf 'Downloading the installer matched to Core...\n' -fetch "$base/$os-$arch.sha256" "$work/checksum" 1024 +fetch "$base/$os-$arch.sha256" "$work/checksum" expected=$(cat "$work/checksum") [[ "$expected" =~ ^[0-9a-f]{64}$ ]] || fail 'Core returned an invalid installer checksum.' -# The verified archive outlives this run so a later failure, such as a wrong option, needs no new download; it is removed once the installation completes. -archive=$cache/$expected.tar.gz -for stale in "$cache"/*.tar.gz "$cache"/*.tar.gz.part; do - case "$stale" in "$archive"|"$archive.part") ;; *) rm -f "$stale";; esac -done -if [ -f "$archive" ]; then - printf 'Verifying the installer archive from an earlier run...\n' - verified "$archive" || { rm -f "$archive"; printf 'The earlier archive failed verification and was discarded.\n' >&2; } -fi -# An interrupt between the last byte and the rename leaves a complete part file; accept it without asking the server for more. -if [ ! -f "$archive" ] && [ -s "$archive.part" ] && verified "$archive.part"; then mv "$archive.part" "$archive"; fi -if [ ! -f "$archive" ]; then - [ ! -s "$archive.part" ] || printf 'Resuming the earlier download...\n' - available=$(df -Pk "$work" | awk 'NR==2 {print $4}') - fetch "$base/$os-$arch.tar.gz" "$archive.part" "$((available * 1024 - 67108864))" - printf 'Verifying the installer archive...\n' - verified "$archive.part" || { rm -f "$archive.part"; fail 'Installer checksum mismatch; the download was discarded. Rerun the command to download it again.'; } - mv "$archive.part" "$archive" -fi -printf 'Checking extraction space...\n' -# Count the verified tar stream without storing another copy, including its headers and padding. -run bash -o pipefail -c 'gzip -dc "$1" | wc -c' -- "$archive" >"$work/unpacked-size" || { rm -f "$archive"; fail 'Installer archive is incomplete or invalid; it was discarded. Rerun the command to download it again.'; } -unpacked=$(cat "$work/unpacked-size") -space "$unpacked" -printf 'Extracting the installer...\n' -mkdir "$work/bundle" -run tar -xzf "$archive" -C "$work/bundle" || fail 'Installer extraction failed. Check disk space, quota and filesystem permissions.' -[ -x "$work/bundle/oac-daemon" ] || fail 'Cannot start the native installer. Check executable permissions and use a Runtime home that permits execution.' -if OAC_INSTALL_GRANT=$grant run "$work/bundle/oac-daemon" install --onboard-url "${base%/install/*}/installation" "$@"; then - rm -f "$archive" - exit 0 -else - code=$? - case "$code" in - 126|127) fail 'Cannot start the native installer. Check that the Runtime home permits execution (not a noexec mount) and that this host has the required native libraries.';; - *) exit "$code";; - esac -fi +fetch "$base/$os-$arch.tar.gz" "$work/installer.tar.gz" +run "${hash[@]}" "$work/installer.tar.gz" >"$work/digest" +read -r actual rest <"$work/digest" +[ "$actual" = "$expected" ] || fail 'Installer checksum mismatch; the download was discarded. Rerun the command to download it again.' +run tar -xzf "$work/installer.tar.gz" -C "$work" || fail 'Installer extraction failed. Check disk space, quota and filesystem permissions.' +rm -f "$work/installer.tar.gz" +[ -x "$work/oac-daemon" ] || fail 'Cannot start the native installer. Check executable permissions and use a Runtime home that permits execution.' +code=0 +OAC_INSTALL_GRANT=$grant run "$work/oac-daemon" install --onboard-url "${base%/install/*}/installation" "$@" || code=$? +case "$code" in + 0) ;; + 126|127) fail 'Cannot start the native installer. Check that the Runtime home permits execution (not a noexec mount) and that this host has the required native libraries.';; + *) exit "$code";; +esac diff --git a/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go b/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go index 7acfc6f63..9313a864d 100644 --- a/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go +++ b/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go @@ -18,69 +18,6 @@ import ( "time" ) -func TestBootstrapParentDeathKeepsActiveDownloadLocked(t *testing.T) { - archive := bootstrapFixtureArchive(t) - started := make(chan struct{}) - release := make(chan struct{}) - var downloads atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if strings.HasSuffix(r.URL.Path, "/installation") { - w.WriteHeader(404) - return - } - if strings.HasSuffix(r.URL.Path, ".sha256") { - fmt.Fprintf(w, "%x\n", sha256.Sum256(archive)) - return - } - w.Header().Set("Content-Length", fmt.Sprint(len(archive))) - if downloads.Add(1) == 1 { - w.Write(archive[:1]) - w.(http.Flusher).Flush() - close(started) - <-release - return - } - http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(archive)) - })) - defer server.Close() - defer close(release) - home := t.TempDir() - command := bootstrapCommand(t, server.URL, home) - command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - var output bytes.Buffer - command.Stdout = &output - command.Stderr = &output - if err := command.Start(); err != nil { - t.Fatal(err) - } - // Kill only this fixture's process group, even if an assertion fails. - defer syscall.Kill(-command.Process.Pid, syscall.SIGKILL) - select { - case <-started: - case <-time.After(10 * time.Second): - t.Fatal("download did not start") - } - if err := command.Process.Kill(); err != nil { - t.Fatal(err) - } - retry, err := bootstrapCommand(t, server.URL, home).CombinedOutput() - if err == nil || !bytes.Contains(retry, []byte("Another native download")) { - t.Fatalf("active download was not protected: %v %s", err, retry) - } - if _, err := os.Stat(filepath.Join(home, "native-download", fmt.Sprintf("%x.tar.gz.part", sha256.Sum256(archive)))); err != nil { - t.Fatal("active download removed") - } - syscall.Kill(-command.Process.Pid, syscall.SIGKILL) - _ = command.Wait() - retry, err = bootstrapCommand(t, server.URL, home).CombinedOutput() - if err != nil { - t.Fatalf("recovery failed: %v %s", err, retry) - } - if _, err := os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { - t.Fatal("staging left after recovery") - } -} - func TestBootstrapCommandDiscardsTimedOutResponse(t *testing.T) { curl, err := exec.LookPath("curl") if err != nil { @@ -133,7 +70,8 @@ func TestBootstrapInterruptSaysHowToContinue(t *testing.T) { })) defer server.Close() defer close(release) - command := bootstrapCommand(t, server.URL, t.TempDir()) + home := t.TempDir() + command := bootstrapCommand(t, server.URL, home) command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} var output bytes.Buffer command.Stdout = &output @@ -153,4 +91,7 @@ func TestBootstrapInterruptSaysHowToContinue(t *testing.T) { if err := command.Wait(); err == nil || !strings.Contains(output.String(), "Interrupted; rerun the same command to continue.") { t.Fatalf("interrupt message missing: %v %s", err, output.String()) } + if left, _ := filepath.Glob(filepath.Join(home, "native-download", "bundle.*")); len(left) != 0 { + t.Fatalf("interrupt left the extracted installer: %v", left) + } } diff --git a/services/core/internal/nativeinstaller/bootstrap_recovery_test.go b/services/core/internal/nativeinstaller/bootstrap_recovery_test.go index 04a82f2a8..7a979041a 100644 --- a/services/core/internal/nativeinstaller/bootstrap_recovery_test.go +++ b/services/core/internal/nativeinstaller/bootstrap_recovery_test.go @@ -110,11 +110,12 @@ func TestBootstrapRecovery(t *testing.T) { if scenario == "metadata-503" && metadata.Load() != 2 { t.Fatalf("metadata requests=%d", metadata.Load()) } - if scenario == "archive-truncated" && downloads.Load() != 2 { + // PowerShell accepts a short body; the checksum catches it and a rerun downloads again. + if scenario == "archive-truncated" && runtime.GOOS != "windows" && downloads.Load() != 2 { t.Fatalf("download requests=%d", downloads.Load()) } - if _, err := os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { - t.Fatalf("left staging: %v", err) + if left, _ := filepath.Glob(filepath.Join(home, "native-download", "bundle.*")); len(left) != 0 { + t.Fatalf("left the extracted installer: %v", left) } if bytes.Contains(output, []byte("fixture-grant")) || bytes.Contains(output, []byte("\x1b")) { t.Fatal("secret or terminal escape in captured output") @@ -123,58 +124,6 @@ func TestBootstrapRecovery(t *testing.T) { } } -func TestBootstrapDiscardsStaleStagingOnly(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("POSIX executable fixture; Windows recovery is covered separately") - } - home := t.TempDir() - staging := filepath.Join(home, "native-download", "staging") - if err := os.MkdirAll(staging, 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(staging, "interrupted"), []byte("partial"), 0600); err != nil { - t.Fatal(err) - } - keep := filepath.Join(home, "workspace.txt") - if err := os.WriteFile(keep, []byte("keep"), 0600); err != nil { - t.Fatal(err) - } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) })) - defer server.Close() - if output, err := bootstrapCommand(t, server.URL, home).CombinedOutput(); err == nil { - t.Fatalf("unexpected success %s", output) - } - if _, err := os.Stat(staging); !os.IsNotExist(err) { - t.Fatal("stale staging remains") - } - if data, err := os.ReadFile(keep); err != nil || string(data) != "keep" { - t.Fatal("unrelated file changed") - } -} - -func TestBootstrapRejectsLowSpaceBeforeNetwork(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("POSIX disk-space fixture") - } - home := t.TempDir() - bin := t.TempDir() - if err := os.WriteFile(filepath.Join(bin, "df"), []byte("#!/bin/sh\nprintf 'Filesystem 1024-blocks Used Available Capacity Mounted\\nfixture 100 99 1 99%% /\\n'\n"), 0700); err != nil { - t.Fatal(err) - } - var requests atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) - defer server.Close() - command := bootstrapCommand(t, server.URL, home) - command.Env = append(command.Env, "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH")) - output, err := command.CombinedOutput() - if err == nil || !bytes.Contains(output, []byte("Not enough disk space")) || requests.Load() != 0 { - t.Fatalf("low-space guard: %v %s requests=%d", err, output, requests.Load()) - } - if _, err = os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { - t.Fatal("low-space failure left staging") - } -} - func TestBootstrapExplainsUnexecutableInstaller(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("POSIX executable permissions") @@ -209,29 +158,41 @@ func TestBootstrapExplainsUnexecutableInstaller(t *testing.T) { if err == nil || !bytes.Contains(output, []byte("Cannot start the native installer")) { t.Fatalf("%v: %s", err, output) } - if _, err := os.Stat(filepath.Join(home, "native-download", "staging")); !os.IsNotExist(err) { - t.Fatal("staging remains") + if left, _ := filepath.Glob(filepath.Join(home, "native-download", "bundle.*")); len(left) != 0 { + t.Fatal("extracted installer remains") } } -func TestWindowsBootstrapChecksArchiveToolBeforeDownload(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("Windows archive prerequisite") +// Only the installer receives the grant, not curl, tar or the other tools around it. +func TestBootstrapGivesGrantOnlyToInstaller(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX executable fixture") + } + curl, err := exec.LookPath("curl") + if err != nil { + t.Skip("curl unavailable") + } + leak := filepath.Join(t.TempDir(), "leak") + bin := t.TempDir() + wrapper := "#!/bin/sh\n[ -z \"$OAC_INSTALL_GRANT\" ] || echo curl >> " + shellQuote(leak) + "\nexec " + shellQuote(curl) + " \"$@\"\n" + if err = os.WriteFile(filepath.Join(bin, "curl"), []byte(wrapper), 0700); err != nil { + t.Fatal(err) } - var requests atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) + archive := bootstrapScriptArchive(t, "#!/bin/sh\n[ \"$OAC_INSTALL_GRANT\" = fixture-grant ]\n") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sha256") { + fmt.Fprintf(w, "%x\n", sha256.Sum256(archive)) + return + } + w.Write(archive) + })) defer server.Close() - home := filepath.Join(t.TempDir(), "runtime") - command := bootstrapCommand(t, server.URL, home) - quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } - // Keep the real SystemRoot while PowerShell itself loads. - command.Args = []string{command.Path, "-NoProfile", "-NonInteractive", "-Command", - "$env:SystemRoot = " + quote(t.TempDir()) + "; & './assets/bootstrap.ps1' -Base " + quote(server.URL) + ""} - output, err := command.CombinedOutput() - if err == nil || !bytes.Contains(output, []byte("tar.exe is required")) || requests.Load() != 0 { - t.Fatalf("%v: %s", err, output) + command := bootstrapCommand(t, server.URL, t.TempDir()) + command.Env = append(command.Env, "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH")) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("the installer did not receive the grant: %v: %s", err, output) } - if _, err := os.Stat(home); !os.IsNotExist(err) { - t.Fatal("home created before prerequisite check") + if _, err = os.Stat(leak); !os.IsNotExist(err) { + t.Fatal("a download tool received the grant") } } diff --git a/services/core/internal/nativeinstaller/bootstrap_resume_unix_test.go b/services/core/internal/nativeinstaller/bootstrap_resume_unix_test.go deleted file mode 100644 index 63163b051..000000000 --- a/services/core/internal/nativeinstaller/bootstrap_resume_unix_test.go +++ /dev/null @@ -1,188 +0,0 @@ -//go:build unix - -package nativeinstaller - -import ( - "bytes" - "crypto/sha256" - "fmt" - "net/http" - "net/http/httptest" - "os" - "os/exec" - "path/filepath" - "strings" - "sync/atomic" - "testing" - "time" - - v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" -) - -// resumeServer serves the installation endpoint and a ranged archive. -type resumeServer struct { - *httptest.Server - archive []byte - downloads, checks atomic.Int32 - ranges chan string - grantStatus int - stallFirstDownload bool -} - -func newResumeServer(t *testing.T, archive []byte) *resumeServer { - t.Helper() - s := &resumeServer{archive: archive, ranges: make(chan string, 8), grantStatus: 200} - s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case strings.HasSuffix(r.URL.Path, "/installation"): - s.checks.Add(1) - if r.Header.Get("Authorization") != "Bearer fixture-grant" { - t.Error("grant missing from the installation check") - } - w.WriteHeader(s.grantStatus) - fmt.Fprint(w, `{"harness":"codex"}`) - case strings.HasSuffix(r.URL.Path, ".sha256"): - fmt.Fprintf(w, "%s\n", sha256Hex(archive)) - default: - request := s.downloads.Add(1) - s.ranges <- r.Header.Get("Range") - if s.stallFirstDownload && request == 1 { - w.Header().Set("Content-Length", fmt.Sprint(len(archive))) - w.Write(archive[:len(archive)/2]) - w.(http.Flusher).Flush() - <-r.Context().Done() - return - } - http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(archive)) - } - })) - t.Cleanup(s.Close) - return s -} - -func (s *resumeServer) command(t *testing.T, home string, args ...string) *exec.Cmd { - command := exec.Command("bash", append([]string{"assets/bootstrap.sh", s.URL + "/api/v1/agent-daemon/install/test"}, args...)...) - command.Env = append(os.Environ(), "OAC_RUNTIME_HOME="+home, v1.NativeInstallGrantEnv+"=fixture-grant", "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") - return command -} - -func TestBootstrapResumesStalledDownload(t *testing.T) { - curl, err := exec.LookPath("curl") - if err != nil { - t.Skip("curl unavailable") - } - archive := bootstrapFixtureArchive(t) - server := newResumeServer(t, archive) - server.stallFirstDownload = true - // Shorten only the stall window; the script's own retry and resume logic runs. - bin := t.TempDir() - if err = os.WriteFile(filepath.Join(bin, "curl"), []byte("#!/bin/sh\nexec "+shellQuote(curl)+" \"$@\" --speed-time 2\n"), 0700); err != nil { - t.Fatal(err) - } - command := server.command(t, t.TempDir()) - command.Env = append(command.Env, "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH")) - if output, err := command.CombinedOutput(); err != nil { - t.Fatalf("%v: %s", err, output) - } - if first, second := <-server.ranges, <-server.ranges; first != "" || !strings.HasPrefix(second, "bytes=") { - t.Fatalf("second request did not resume: %q then %q", first, second) - } -} - -func TestBootstrapKeepsVerifiedArchiveUntilInstallCompletes(t *testing.T) { - // The first install attempt fails after the download; the rerun needs no new archive and removes it on success. - archive := bootstrapScriptArchive(t, "#!/bin/sh\n[ -e \"$OAC_RESUME_OK\" ]\n") - server := newResumeServer(t, archive) - home, ok := t.TempDir(), filepath.Join(t.TempDir(), "ok") - cached := filepath.Join(home, "native-download", sha256Hex(archive)+".tar.gz") - command := server.command(t, home) - command.Env = append(command.Env, "OAC_RESUME_OK="+ok) - if output, err := command.CombinedOutput(); err == nil { - t.Fatalf("expected the fixture install to fail: %s", output) - } - if _, err := os.Stat(cached); err != nil { - t.Fatal("verified archive was not kept") - } - if err := os.WriteFile(ok, nil, 0600); err != nil { - t.Fatal(err) - } - command = server.command(t, home) - command.Env = append(command.Env, "OAC_RESUME_OK="+ok) - if output, err := command.CombinedOutput(); err != nil { - t.Fatalf("%v: %s", err, output) - } - if server.downloads.Load() != 1 { - t.Fatalf("archive downloaded %d times", server.downloads.Load()) - } - if _, err := os.Stat(cached); !os.IsNotExist(err) { - t.Fatal("archive kept after a completed installation") - } -} - -// A rerun after an interrupt between the last byte and the rename finds a complete .part file; the server answers 416 and the checksum decides. -func TestBootstrapAcceptsCompletePartialDownload(t *testing.T) { - archive := bootstrapFixtureArchive(t) - server := newResumeServer(t, archive) - home := t.TempDir() - part := filepath.Join(home, "native-download", sha256Hex(archive)+".tar.gz.part") - if err := os.MkdirAll(filepath.Dir(part), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(part, archive, 0600); err != nil { - t.Fatal(err) - } - output, err := server.command(t, home).CombinedOutput() - if err != nil || strings.Contains(string(output), "416") { - t.Fatalf("%v: %s", err, output) - } -} - -// Only the installer receives the grant, not curl, tar or the other tools around it. -func TestBootstrapGivesGrantOnlyToInstaller(t *testing.T) { - curl, err := exec.LookPath("curl") - if err != nil { - t.Skip("curl unavailable") - } - leak := filepath.Join(t.TempDir(), "leak") - bin := t.TempDir() - wrapper := "#!/bin/sh\n[ -z \"$OAC_INSTALL_GRANT\" ] || echo curl >> " + shellQuote(leak) + "\nexec " + shellQuote(curl) + " \"$@\"\n" - if err = os.WriteFile(filepath.Join(bin, "curl"), []byte(wrapper), 0700); err != nil { - t.Fatal(err) - } - server := newResumeServer(t, bootstrapScriptArchive(t, "#!/bin/sh\n[ \"$OAC_INSTALL_GRANT\" = fixture-grant ]\n")) - command := server.command(t, t.TempDir()) - command.Env = append(command.Env, "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH")) - if output, err := command.CombinedOutput(); err != nil { - t.Fatalf("the installer did not receive the grant: %v: %s", err, output) - } - if _, err = os.Stat(leak); !os.IsNotExist(err) { - t.Fatal("a download tool received the grant") - } -} - -func TestBootstrapDiscardsCachedArchiveThatFailsVerification(t *testing.T) { - archive := bootstrapFixtureArchive(t) - server := newResumeServer(t, archive) - home := t.TempDir() - cached := filepath.Join(home, "native-download", sha256Hex(archive)+".tar.gz") - if err := os.MkdirAll(filepath.Dir(cached), 0700); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(cached, []byte("corrupt"), 0600); err != nil { - t.Fatal(err) - } - if output, err := server.command(t, home).CombinedOutput(); err != nil || server.downloads.Load() != 1 { - t.Fatalf("%v downloads=%d: %s", err, server.downloads.Load(), output) - } -} - -func TestBootstrapChecksGrantBeforeDownloading(t *testing.T) { - server := newResumeServer(t, bootstrapFixtureArchive(t)) - server.grantStatus = 401 - output, err := server.command(t, t.TempDir()).CombinedOutput() - if err == nil || !strings.Contains(string(output), "authorization rejected or expired") || server.downloads.Load() != 0 || strings.Contains(string(output), "fixture-grant") { - t.Fatalf("%v downloads=%d: %s", err, server.downloads.Load(), output) - } -} - -func sha256Hex(data []byte) string { return fmt.Sprintf("%x", sha256.Sum256(data)) } diff --git a/services/core/internal/nativeinstaller/catalog.go b/services/core/internal/nativeinstaller/catalog.go index cba289bb0..00b3ed656 100644 --- a/services/core/internal/nativeinstaller/catalog.go +++ b/services/core/internal/nativeinstaller/catalog.go @@ -40,7 +40,8 @@ type Catalog struct { var checksum = regexp.MustCompile(`^[0-9a-f]{64}$`) -var platformName = regexp.MustCompile(`^(linux|darwin|windows)-(amd64|arm64)$`) +// An artifact is a platform's base archive or one named part of it. +var artifactName = regexp.MustCompile(`^(linux|darwin|windows)-(amd64|arm64)(-[a-z]+)?$`) // Load checks the matched catalog without downloading execution payloads. Local // offline archives are verified once; the directory stays immutable while serving. @@ -58,25 +59,25 @@ func Load(directory, version string) (*Catalog, error) { return nil, errors.New("native installer catalog does not match this Core build and protocol") } c.local = make(map[string]bool) - for platform, artifact := range c.Artifacts { - if !platformName.MatchString(platform) || !checksum.MatchString(artifact.SHA256) { - return nil, errors.New("invalid native installer platform") + for name, artifact := range c.Artifacts { + if !artifactName.MatchString(name) || !checksum.MatchString(artifact.SHA256) { + return nil, errors.New("invalid native installer artifact") } if artifact.URL != "" { u, err := url.Parse(artifact.URL) - filename := "oac-native-" + c.Version + "-" + platform + ".tar.gz" + filename := "oac-native-" + c.Version + "-" + name + ".tar.gz" if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || !strings.HasSuffix(u.Path, "/"+filename) || strings.Contains(strings.ToLower(u.Path), "/latest/") { return nil, errors.New("invalid versioned native installer URL") } } - file, err := os.Open(filepath.Join(directory, platform+".tar.gz")) + file, err := os.Open(filepath.Join(directory, name+".tar.gz")) if errors.Is(err, os.ErrNotExist) && artifact.URL != "" { continue } if err != nil { return nil, err } - c.local[platform] = true + c.local[name] = true hash := sha256.New() _, err = io.Copy(hash, file) file.Close() @@ -104,22 +105,22 @@ func (c *Catalog) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(raw) return } - platform := strings.TrimSuffix(strings.TrimSuffix(name, ".sha256"), ".tar.gz") - artifact, ok := c.Artifacts[platform] + archive := strings.TrimSuffix(strings.TrimSuffix(name, ".sha256"), ".tar.gz") + artifact, ok := c.Artifacts[archive] if !ok { http.NotFound(w, r) return } - if name == platform+".sha256" { + if name == archive+".sha256" { w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintln(w, artifact.SHA256) return } - if name != platform+".tar.gz" { + if name != archive+".tar.gz" { http.NotFound(w, r) return } - if c.local[platform] { + if c.local[archive] { // An archive outlives the server-wide write timeout on a slow link. _ = http.NewResponseController(w).SetWriteDeadline(time.Time{}) http.ServeFile(w, r, filepath.Join(c.directory, name)) diff --git a/services/core/internal/nativeinstaller/catalog_test.go b/services/core/internal/nativeinstaller/catalog_test.go index 2737780a7..555180951 100644 --- a/services/core/internal/nativeinstaller/catalog_test.go +++ b/services/core/internal/nativeinstaller/catalog_test.go @@ -20,13 +20,15 @@ func TestCatalogRequiresMatchedImmutableArtifacts(t *testing.T) { dir := t.TempDir() data := []byte("archive fixture") hash := sha256.Sum256(data) - manifest := Catalog{Version: "build", ProtocolVersion: proto.Version, Artifacts: map[string]Artifact{"linux-amd64": {SHA256: hex.EncodeToString(hash[:])}}} + manifest := Catalog{Version: "build", ProtocolVersion: proto.Version, Artifacts: map[string]Artifact{"linux-amd64": {SHA256: hex.EncodeToString(hash[:])}, "linux-amd64-codex": {SHA256: hex.EncodeToString(hash[:])}}} raw, _ := json.Marshal(manifest) if err := os.WriteFile(filepath.Join(dir, "catalog.json"), raw, 0600); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(dir, "linux-amd64.tar.gz"), data, 0600); err != nil { - t.Fatal(err) + for _, name := range []string{"linux-amd64.tar.gz", "linux-amd64-codex.tar.gz"} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0600); err != nil { + t.Fatal(err) + } } if _, err := Load(dir, "wrong-build"); err == nil { t.Fatal("accepted mismatched Core") @@ -39,7 +41,7 @@ func TestCatalogRequiresMatchedImmutableArtifacts(t *testing.T) { path string code int }{ - {"build/linux-amd64.tar.gz", 200}, {"build/linux-amd64.sha256", 200}, {"other/linux-amd64.tar.gz", 404}, {"build/windows-arm64.tar.gz", 404}, {"build/catalog.json", 404}, {"build/../../catalog.json", 404}, + {"build/linux-amd64.tar.gz", 200}, {"build/linux-amd64.sha256", 200}, {"build/linux-amd64-codex.tar.gz", 200}, {"build/linux-amd64-codex.sha256", 200}, {"build/linux-amd64-claude.tar.gz", 404}, {"other/linux-amd64.tar.gz", 404}, {"build/windows-arm64.tar.gz", 404}, {"build/catalog.json", 404}, {"build/../../catalog.json", 404}, } { w := httptest.NewRecorder() catalog.ServeHTTP(w, httptest.NewRequest("GET", "/api/v1/agent-daemon/install/"+request.path, nil)) From b50d171218906891dc9a23cbe9c71fffdf6aec28 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Tue, 20 Oct 2026 16:13:58 +0000 Subject: [PATCH 2/2] Download before claiming and bound stalled native requests --- .github/workflows/native.yml | 6 ++- apps/daemon/internal/cli/native_download.go | 53 ++++++++----------- .../internal/cli/native_download_test.go | 18 ++++++- apps/daemon/internal/cli/native_install.go | 23 +++----- docs/getting-started/self-hosted.md | 2 +- docs/zh/getting-started/self-hosted.md | 4 +- scripts/native-onboarding-smoke.mjs | 6 ++- scripts/publish-core-release.py | 2 +- .../nativeinstaller/assets/bootstrap.ps1 | 2 +- .../nativeinstaller/assets/bootstrap.sh | 4 +- .../bootstrap_recovery_test.go | 22 ++++++++ 11 files changed, 85 insertions(+), 57 deletions(-) diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 36c755103..e5394a926 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -136,7 +136,11 @@ jobs: node scripts/build-native-installer-ci.mjs - name: Bootstrap, authenticate, install and connect natively id: onboarding - run: node scripts/native-onboarding-smoke.mjs + run: | + if [[ "$RUNNER_OS" == Windows ]]; then + export CLAUDE_CODE_GIT_BASH_PATH='C:\Program Files\Git\bin\bash.exe' + fi + node scripts/native-onboarding-smoke.mjs - name: Verify native Harness protocols without model requests id: protocol run: | diff --git a/apps/daemon/internal/cli/native_download.go b/apps/daemon/internal/cli/native_download.go index 3b1b46c74..50a003ec2 100644 --- a/apps/daemon/internal/cli/native_download.go +++ b/apps/daemon/internal/cli/native_download.go @@ -21,15 +21,14 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/runtimefs" ) -const ( - nativeStall = 60 * time.Second - nativeAttempts = 5 -) +const nativeAttempts = 5 var ( + nativeStall = 60 * time.Second nativeRetryDelay = time.Second nativeDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) errNativeRedirect = errors.New("install: a download redirect must use HTTPS without credentials") + errNativeStall = errors.New("install: download stalled (no data for 60 seconds)") errNativeNoArchive = errors.New("install: this Core has no qualified archive for this component and platform") ) @@ -61,7 +60,7 @@ func obtainNativeComponents(ctx context.Context, out io.Writer, o *nativeInstall return nil } if o.OnboardURL == "" { - return fmt.Errorf("install: component %s is missing from %s; extract oac-native-%s-%s-%s-%s.tar.gz into it and retry", missing[0], o.Bundle, Version, runtime.GOOS, runtime.GOARCH, missing[0]) + return fmt.Errorf("install: component %s is missing from %s; extract %s-%s-%s.tar.gz into it and retry", missing[0], o.Bundle, runtime.GOOS, runtime.GOARCH, missing[0]) } return fetchNativeComponents(ctx, out, o.OnboardURL, o.Bundle, missing) } @@ -83,9 +82,9 @@ func fetchNativeComponents(ctx context.Context, out io.Writer, onboardURL, bundl return err } defer held.Close() - unlock, err := lockNativeDownloads(ctx, out, held) + unlock, err := runtimefs.LockDirectory(held) if err != nil { - return err + return errors.New("install: another native download is running; wait for it to finish and retry") } defer unlock() pruneNativeDownloads(cache) @@ -106,22 +105,6 @@ func fetchNativeComponents(ctx context.Context, out io.Writer, onboardURL, bundl return nil } -func lockNativeDownloads(ctx context.Context, out io.Writer, held *os.Root) (func(), error) { - for waiting := false; ; waiting = true { - if unlock, err := runtimefs.LockDirectory(held); err == nil { - return unlock, nil - } - if !waiting { - fmt.Fprintln(out, "Waiting for another native download to finish...") - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(500 * time.Millisecond): - } - } -} - // pruneNativeDownloads keeps the current version and recent distribution directories of running // installers. Failing to remove an old entry is harmless. func pruneNativeDownloads(cache string) { @@ -178,15 +161,17 @@ func fetchNativeArchive(ctx context.Context, out io.Writer, url, dir, file strin func fetchNativeChecksum(ctx context.Context, out io.Writer, url string) (string, error) { var digest string err := retryNativeDownload(ctx, out, func() (bool, error) { + ctx, cancel := context.WithTimeoutCause(ctx, nativeStall, errNativeStall) + defer cancel() response, retry, err := getNative(ctx, url, 0) if err != nil { - return retry, err + return retry, stalled(ctx, err) } defer response.Body.Close() raw, err := io.ReadAll(io.LimitReader(response.Body, 1025)) digest = strings.TrimSpace(string(raw)) if err != nil { - return true, err + return true, stalled(ctx, err) } if !nativeDigest.MatchString(digest) { return false, errors.New("install: Core returned an invalid archive checksum") @@ -253,13 +238,13 @@ func downloadNativeFile(ctx context.Context, out io.Writer, url, dest, label str if info, err := os.Stat(dest); err == nil { offset = info.Size() } - ctx, cancel := context.WithCancel(ctx) - defer cancel() - stall := time.AfterFunc(nativeStall, cancel) + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(nil) + stall := time.AfterFunc(nativeStall, func() { cancel(errNativeStall) }) defer stall.Stop() response, retry, err := getNative(ctx, url, offset) if err != nil { - return retry, err + return retry, stalled(ctx, err) } defer response.Body.Close() flags := os.O_WRONLY | os.O_CREATE | os.O_APPEND @@ -286,7 +271,15 @@ func downloadNativeFile(ctx context.Context, out io.Writer, url, dest, label str if closeErr := file.Close(); err == nil { err = closeErr } - return err != nil, err + return err != nil, stalled(ctx, err) +} + +// stalled reports a stall instead of the cancellation it caused. +func stalled(ctx context.Context, err error) error { + if err != nil && errors.Is(context.Cause(ctx), errNativeStall) { + return errNativeStall + } + return err } type nativeBody struct { diff --git a/apps/daemon/internal/cli/native_download_test.go b/apps/daemon/internal/cli/native_download_test.go index d64ebbf16..53d629d71 100644 --- a/apps/daemon/internal/cli/native_download_test.go +++ b/apps/daemon/internal/cli/native_download_test.go @@ -8,7 +8,9 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -124,8 +126,10 @@ func writeBundle(t *testing.T, bundle string, components map[string]nativeCompon func TestNativeInstallDownloadsOnlySelectedComponentsOnceAndResumes(t *testing.T) { home := t.TempDir() t.Setenv("OAC_RUNTIME_HOME", home) + oldDelay, oldProbe := nativeRetryDelay, probeNativeInstallation nativeRetryDelay = time.Millisecond probeNativeInstallation = func(context.Context, string, []string) error { return nil } + t.Cleanup(func() { nativeRetryDelay, probeNativeInstallation = oldDelay, oldProbe }) core, _ := newFakeCore(t, t.TempDir()) components := map[string]nativeComponent{} for _, name := range []string{"node", "codex", "claude"} { @@ -194,8 +198,20 @@ func TestNativeInstallFromDistributionNamesMissingArchive(t *testing.T) { t.Fatal(err) } err := runInstall(rc, args) - want := fmt.Sprintf("oac-native-%s-%s-%s-codex.tar.gz", Version, runtime.GOOS, runtime.GOARCH) + want := fmt.Sprintf("extract %s-%s-codex.tar.gz", runtime.GOOS, runtime.GOARCH) if err == nil || !strings.Contains(err.Error(), want) { t.Fatalf("expected an error naming %s, got %v", want, err) } } + +func TestNativeDownloadFailsClearlyWhenCoreStopsAnswering(t *testing.T) { + oldStall, oldDelay := nativeStall, nativeRetryDelay + nativeStall, nativeRetryDelay = 50*time.Millisecond, time.Millisecond + t.Cleanup(func() { nativeStall, nativeRetryDelay = oldStall, oldDelay }) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { <-release })) + t.Cleanup(func() { close(release); server.Close() }) + if _, err := fetchNativeChecksum(context.Background(), io.Discard, server.URL+"/linux-amd64-codex.sha256"); !errors.Is(err, errNativeStall) { + t.Fatal(err) + } +} diff --git a/apps/daemon/internal/cli/native_install.go b/apps/daemon/internal/cli/native_install.go index 71fbc9d3f..663e93036 100644 --- a/apps/daemon/internal/cli/native_install.go +++ b/apps/daemon/internal/cli/native_install.go @@ -3,8 +3,6 @@ package cli import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "flag" @@ -145,6 +143,10 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO if err = cleanNativeTemporaryFiles(o.Directory); err != nil { return err } + // Download before claiming the credential, so a failed download leaves the Session's credential unclaimed. + if err = obtainNativeComponents(ctx, rc.stdout, o, selected); err != nil { + return err + } if o.OnboardURL != "" { if runtimefs.ValidateLocalPath(o.Workspace) != nil { return errors.New("install: Session workspace is invalid on this platform") @@ -184,9 +186,6 @@ func installNativeOptions(ctx context.Context, rc *runContext, o *nativeInstallO } } slices.Sort(all) - if err = obtainNativeComponents(ctx, rc.stdout, o, selected); err != nil { - return err - } // Verify existing components before adding any new one; never repair or // upgrade an installed dependency as a side effect of adding a Harness. if len(previous.Harnesses) > 0 { @@ -250,21 +249,11 @@ func installNativeBinary(ctx context.Context, root string, existing bool) error return err } dest := filepath.Join(dir, nativeExe("oac-daemon")) - digest := func(name string) (string, error) { - f, e := os.Open(name) - if e != nil { - return "", e - } - defer f.Close() - h := sha256.New() - _, e = nativeCopy(ctx, h, f) - return hex.EncodeToString(h.Sum(nil)), e - } - want, err := digest(exe) + want, err := nativeFileDigest(ctx, exe) if err != nil { return err } - if got, e := digest(dest); e == nil { + if got, e := nativeFileDigest(ctx, dest); e == nil { if got != want { return errors.New("install: existing daemon binary differs; in-place upgrades are unsupported") } diff --git a/docs/getting-started/self-hosted.md b/docs/getting-started/self-hosted.md index 270cbf638..23ddfc8aa 100644 --- a/docs/getting-started/self-hosted.md +++ b/docs/getting-started/self-hosted.md @@ -37,7 +37,7 @@ The machine needs: - Python and pip when the Session's packages need them; - any system packages your setup needs. The daemon never runs apt, sudo or another elevation command, so install them through the host's normal administration. -No administrator privileges or Docker are needed. The Unix download command also uses `curl`, `tar` and a SHA-256 tool. Windows needs its system `tar.exe`; the command checks for it before downloading. The Runtime home must allow executable files to run. On a Unix `noexec` mount, choose another absolute directory with `OAC_RUNTIME_HOME` before running the command. +No administrator privileges or Docker are needed. The Unix download command also uses `curl`, `tar`, `gzip` and a SHA-256 tool. Windows needs its system `tar.exe`; the command checks for it before downloading. The Runtime home must allow executable files to run. On a Unix `noexec` mount, choose another absolute directory with `OAC_RUNTIME_HOME` before running the command. ## Connect a machine diff --git a/docs/zh/getting-started/self-hosted.md b/docs/zh/getting-started/self-hosted.md index d242e63e3..7220ae2ec 100644 --- a/docs/zh/getting-started/self-hosted.md +++ b/docs/zh/getting-started/self-hosted.md @@ -1,7 +1,7 @@ --- title: "运行自托管 Session" source: docs/getting-started/self-hosted.md -source_hash: dd09af89f5c98bb1b0bf3dfef5f85c9c9068ef3d490b2e1f03fd7691b43d0fb0 +source_hash: 3d3ec2e1374094c63bb7946440c50654bc5265bd0ef3808ecefbc0b014ee461f --- `self_hosted` Session 在应用拥有的机器上运行，例如工作站、虚拟机或容器，而不是 Core 管理的沙箱。本指南面向用 `oac-daemon` 连接此类机器并保持其运行的应用开发者。 @@ -39,7 +39,7 @@ Session 自带模型提供商；安装默认模型不适用（[原因](../../../ - Session 的软件包需要时，安装 Python 和 pip； - 环境设置所需的系统软件包。守护进程不运行 apt、sudo 或其他提权命令，请通过主机的常规管理方式安装。 -不需要管理员权限或 Docker。Unix 下载命令还使用 `curl`、`tar` 和 SHA-256 工具。Windows 需要系统自带的 `tar.exe`，命令会在下载前检查。Runtime 主目录必须允许执行文件。如果 Unix 挂载点设为 `noexec`，请先用 `OAC_RUNTIME_HOME` 指定另一个允许执行的绝对目录，再运行命令。 +不需要管理员权限或 Docker。Unix 下载命令还使用 `curl`、`tar`、`gzip` 和 SHA-256 工具。Windows 需要系统自带的 `tar.exe`，命令会在下载前检查。Runtime 主目录必须允许执行文件。如果 Unix 挂载点设为 `noexec`，请先用 `OAC_RUNTIME_HOME` 指定另一个允许执行的绝对目录，再运行命令。 ## 连接机器 {#connect-a-machine} diff --git a/scripts/native-onboarding-smoke.mjs b/scripts/native-onboarding-smoke.mjs index 96ab637a5..cb17bc355 100644 --- a/scripts/native-onboarding-smoke.mjs +++ b/scripts/native-onboarding-smoke.mjs @@ -94,10 +94,12 @@ try { assert.equal(saved.executor_token, secret, 'Secret must survive a lost response'); const script = resolve(`services/core/internal/nativeinstaller/assets/bootstrap.${windows ? 'ps1' : 'sh'}`); const bootstrapArgs = windows ? ['-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', script, '-Base', base] : [script, base]; - const result = await run(windows ? 'powershell.exe' : 'bash', [...bootstrapArgs, '--install-dir', installation], '\n\n'); + // Select every Harness of this platform so each Harness archive passes the installer's extraction and readiness checks. + const harnesses = windows ? ['codex', 'claude'] : ['codex', 'claude', 'minimax']; + const result = await run(windows ? 'powershell.exe' : 'bash', [...bootstrapArgs, '--install-dir', installation], harnesses.join(',') + '\n\n'); assert.equal(result.code, 0, `Interactive bootstrap failed: ${result.output}`); assert.match(result.output, /Daemon connection: connected/); - assert.deepEqual([...downloaded].sort(), [platform, `${platform}-codex`, `${platform}-node`], 'Only the base, Node and the Session Harness may be downloaded'); + assert.deepEqual([...downloaded].sort(), [platform, ...['node', ...harnesses].map(name => `${platform}-${name}`)].sort(), 'Only the base, Node and the selected Harnesses may be downloaded'); assert.match(result.output, /Model configuration: not checked/); assert.equal(connections, 1); const repeat = await run(windows ? 'powershell.exe' : 'bash', [...bootstrapArgs, '--install-dir', installation, '--non-interactive', '--harness', 'codex']); diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index c31e5b34f..8f6308995 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -269,7 +269,7 @@ def publish(assets, repository, revision, tag, mode): if catalog["version"] != revision or not catalog["artifacts"]: raise ValueError("Native installer catalog does not match the release") for name, entry in catalog["artifacts"].items(): - if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)(-[a-z]+)?", name): + if not distribution.NATIVE_ARTIFACT.fullmatch(name): raise ValueError("Invalid native installer artifact") path = assets / f"oac-native-{revision}-{name}.tar.gz" if (distribution.sha256(path) != entry["sha256"] diff --git a/services/core/internal/nativeinstaller/assets/bootstrap.ps1 b/services/core/internal/nativeinstaller/assets/bootstrap.ps1 index cbbc03e28..d713b44d6 100644 --- a/services/core/internal/nativeinstaller/assets/bootstrap.ps1 +++ b/services/core/internal/nativeinstaller/assets/bootstrap.ps1 @@ -18,7 +18,7 @@ function Get-NativeFile([string]$Uri, [string]$Destination) { try { Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Destination -ErrorAction Stop; return } catch { if ($_.Exception.Response -and [int]$_.Exception.Response.StatusCode -eq 404) { throw 'This Core has no qualified installer for this platform.' } - if ($attempt -eq 3) { throw 'Installer download failed. Check the network to Core and GitHub (set HTTPS_PROXY if a proxy is required), then rerun the command.' } + if ($attempt -eq 3) { throw 'Installer download failed. Check the network to Core and GitHub (and the Windows proxy settings), then rerun the command.' } Write-Host "Download interrupted; retrying ($attempt/2)..." Start-Sleep -Seconds $attempt } diff --git a/services/core/internal/nativeinstaller/assets/bootstrap.sh b/services/core/internal/nativeinstaller/assets/bootstrap.sh index b43b57927..d9fa9b3aa 100644 --- a/services/core/internal/nativeinstaller/assets/bootstrap.sh +++ b/services/core/internal/nativeinstaller/assets/bootstrap.sh @@ -8,7 +8,7 @@ unset OAC_INSTALL_GRANT fail() { printf '%s\n' "$*" >&2; exit 1; } case "$(uname -s)" in Linux) os=linux;; Darwin) os=darwin;; *) fail 'Unsupported operating system.';; esac case "$(uname -m)" in x86_64) arch=amd64;; arm64|aarch64) arch=arm64;; *) fail 'Unsupported processor architecture.';; esac -for tool in curl tar; do command -v "$tool" >/dev/null || fail "Required command missing: $tool"; done +for tool in curl tar gzip; do command -v "$tool" >/dev/null || fail "Required command missing: $tool"; done if command -v sha256sum >/dev/null; then hash=(sha256sum); else hash=(shasum -a 256); command -v shasum >/dev/null || fail 'Required command missing: shasum'; fi umask 077 root=${OAC_RUNTIME_HOME:-${HOME:?HOME must be set}/.oac} @@ -18,6 +18,8 @@ root=$(cd "$root" && pwd -P) cache=$root/native-download [ ! -L "$cache" ] || fail 'Native download directory must not be a symbolic link.' mkdir -p "$cache" +[ -d "$cache" ] && [ -O "$cache" ] || fail 'Native download directory must belong to the current account.' +chmod 700 "$cache" # The installer downloads the Harness archives into this directory and extracts them beside the executable. work=$(mktemp -d "$cache/bundle.XXXXXX") || fail 'Cannot write the native download directory.' active= diff --git a/services/core/internal/nativeinstaller/bootstrap_recovery_test.go b/services/core/internal/nativeinstaller/bootstrap_recovery_test.go index 7a979041a..48394854e 100644 --- a/services/core/internal/nativeinstaller/bootstrap_recovery_test.go +++ b/services/core/internal/nativeinstaller/bootstrap_recovery_test.go @@ -196,3 +196,25 @@ func TestBootstrapGivesGrantOnlyToInstaller(t *testing.T) { t.Fatal("a download tool received the grant") } } + +func TestWindowsBootstrapChecksArchiveToolBeforeDownload(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Windows archive prerequisite") + } + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requests.Add(1); w.WriteHeader(500) })) + defer server.Close() + home := filepath.Join(t.TempDir(), "runtime") + command := bootstrapCommand(t, server.URL, home) + quote := func(value string) string { return "'" + strings.ReplaceAll(value, "'", "''") + "'" } + // Keep the real SystemRoot while PowerShell itself loads. + command.Args = []string{command.Path, "-NoProfile", "-NonInteractive", "-Command", + "$env:SystemRoot = " + quote(t.TempDir()) + "; & './assets/bootstrap.ps1' -Base " + quote(server.URL) + ""} + output, err := command.CombinedOutput() + if err == nil || !bytes.Contains(output, []byte("tar.exe is required")) || requests.Load() != 0 { + t.Fatalf("%v: %s", err, output) + } + if _, err := os.Stat(home); !os.IsNotExist(err) { + t.Fatal("home created before prerequisite check") + } +}