Share installed components across self-hosted installations

apps/daemon/internal/cli/native_bundle.go

@@ -84,8 +84,6 @@ func validBundlePath(name string) bool {
 	return name != "." && path.Clean(name) == name && !strings.ContainsAny(name, "\\\x00\r\n:") && filepath.IsLocal(filepath.FromSlash(name)) && name != ".oac-install.json"
 }
 
-func nativeComponentRoot(root, name string) string { return filepath.Join(root, "components", name) }
-
 func componentReceipt(root string) (nativeComponent, error) {
 	var c nativeComponent
 	err := readNativeJSON(filepath.Join(root, ".oac-install.json"), &c)
@@ -130,35 +128,90 @@ func checkComponentFiles(ctx context.Context, directory string, c nativeComponen
 	return nil
 }
 
-// A complete component is published once. An interruption cannot turn a partial
-// copy into an installation; a subsequent run can reuse an already-published one.
-func installNativeComponent(ctx context.Context, source, root, name string, expected nativeComponent) error {
-	dest := nativeComponentRoot(root, name)
+// verifyNativeComponent checks a stored component against its receipt and, when the distribution
+// is at hand, against the distribution's manifest.
+func verifyNativeComponent(ctx context.Context, dir, name string, expected *nativeComponent) error {
+	info, err := os.Lstat(dir)
+	if errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("shared component %s is missing from %s; run install again", name, filepath.Dir(dir))
+	}
+	remedy := "stop the installations that use it, delete " + dir + " (" + nativeDeleteCommand(dir) + ") and run install again"
+	if err != nil || !info.IsDir() || !ownedByCurrentUser(info) {
+		return fmt.Errorf("shared component %s is not a directory of this account; %s", name, remedy)
+	}
+	c, err := componentReceipt(dir)
+	if err != nil || c.Version != nativePins[name] || len(c.Files) == 0 {
+		return fmt.Errorf("shared component %s is missing files or modified; %s", name, remedy)
+	}
+	if expected != nil && !reflect.DeepEqual(c, *expected) {
+		return fmt.Errorf("shared component %s differs from this distribution's bundle.json; %s", name, remedy)
+	}
+	if err = checkComponentFiles(ctx, dir, c); errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrPermission) {
+		return fmt.Errorf("shared component %s is missing files, modified or unreadable; %s", name, remedy)
+	} else if err != nil {
+		return fmt.Errorf("install: cannot verify shared component %s: %w", name, err)
+	}
+	return nil
+}
+
+// ensureNativeComponent makes the store hold name: a stored component is verified, a missing one is
+// copied from the distribution directory or, with a Core address, downloaded. It is published
+// once, complete and read-only, so an interruption leaves nothing partial in place.
+func ensureNativeComponent(ctx context.Context, out io.Writer, o *nativeInstallOptions, name string, expected nativeComponent) error {
+	dest := filepath.Join(o.ComponentDirectory, name)
 	if _, err := os.Lstat(dest); err == nil {
-		got, e := componentReceipt(dest)
-		if e != nil || !reflect.DeepEqual(got, expected) {
-			return fmt.Errorf("install: existing %s is incompatible; preserve it and use a separate installation directory", name)
-		}
-		if err := checkComponentFiles(ctx, dest, got); err != nil {
-			if errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) {
-				return fmt.Errorf("install: existing %s is incomplete or modified; reinstall separately", name)
-			}
-			return fmt.Errorf("install: cannot verify existing %s: %w", name, err)
+		err = nativeInstallPhase(out, "Using installed "+name, func() error { return verifyNativeComponent(ctx, dest, name, &expected) })
+		if err != nil {
+			return err
 		}
-		return nil
+		// An interruption between the rename and the seal leaves the top directory writable.
+		return sealNativeComponent(dest)
 	} else if !errors.Is(err, os.ErrNotExist) {
 		return err
 	}
-	parent := filepath.Dir(dest)
-	if err := os.MkdirAll(parent, 0700); err != nil {
+	source := filepath.Join(o.Bundle, "components", name)
+	_, missing := os.Lstat(source)
+	if missing != nil && o.OnboardURL == "" {
+		return fmt.Errorf("install: component %s is missing from %s; extract %s-%s-%s.tar.gz into it and retry", name, o.Bundle, runtime.GOOS, runtime.GOARCH, name)
+	}
+	stage, err := os.MkdirTemp(o.ComponentDirectory, ".install-"+name+"-")
+	if err != nil {
 		return err
 	}
-	tmp, err := os.MkdirTemp(parent, ".install-"+name+"-")
+	defer removeNativeTree(stage)
+	var archive string
+	if missing == nil {
+		err = nativeInstallPhase(out, "Installing "+name, func() error { return copyNativeComponent(ctx, source, stage, expected) })
+	} else {
+		// A download prints its own progress, which a redrawn phase line would interleave with.
+		fmt.Fprintln(out, "Installing "+name+"...")
+		archive, err = downloadNativeComponent(ctx, out, o.OnboardURL, name, stage, expected)
+	}
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(tmp)
-	src, err := os.OpenRoot(filepath.Join(source, "components", name))
+	raw, _ := json.Marshal(expected)
+	if err = os.WriteFile(filepath.Join(stage, ".oac-install.json"), raw, 0600); err != nil {
+		return err
+	}
+	if err = ctx.Err(); err != nil {
+		return err
+	}
+	if err = setNativeReadOnly(stage); err != nil {
+		return err
+	}
+	if err = os.Rename(stage, dest); err != nil {
+		return err
+	}
+	if archive != "" {
+		_ = os.Remove(archive)
+	}
+	return sealNativeComponent(dest)
+}
+
+// copyNativeComponent copies the manifest's files from the distribution into stage, verifying each.
+func copyNativeComponent(ctx context.Context, source, stage string, expected nativeComponent) error {
+	src, err := os.OpenRoot(source)
 	if err != nil {
 		return err
 	}
@@ -177,16 +230,13 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe
 		}
 		required += uint64(info.Size())
 	}
-	if err = requireNativeSpace(parent, required); err != nil {
+	if err = requireNativeSpace(filepath.Dir(stage), required); err != nil {
 		return err
 	}
 	for name, expectedFile := range expected.Files {
 		if err := ctx.Err(); err != nil {
 			return err
 		}
-		if !validBundlePath(name) {
-			return errors.New("install: invalid component path")
-		}
 		in, err := src.Open(filepath.FromSlash(name))
 		if err != nil {
 			return errors.New("install: component file unavailable")
@@ -196,7 +246,7 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe
 			in.Close()
 			return errors.New("install: component must contain regular files")
 		}
-		destination := filepath.Join(tmp, filepath.FromSlash(name))
+		destination := filepath.Join(stage, filepath.FromSlash(name))
 		if err = os.MkdirAll(filepath.Dir(destination), 0700); err != nil {
 			in.Close()
 			return err
@@ -227,14 +277,7 @@ func installNativeComponent(ctx context.Context, source, root, name string, expe
 			return errors.New("install: component checksum failed")
 		}
 	}
-	raw, _ := json.Marshal(expected)
-	if err = os.WriteFile(filepath.Join(tmp, ".oac-install.json"), raw, 0600); err != nil {
-		return err
-	}
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	return os.Rename(tmp, dest)
+	return nil
 }
 
 func selectedNativeHarnesses(value string) ([]string, error) {

apps/daemon/internal/cli/native_download.go

@@ -45,64 +45,36 @@ func nativeInstallerBase(onboardURL string) string {
 	return strings.TrimSuffix(onboardURL, "installation") + "install/" + Version
 }
 
-// obtainNativeComponents makes the distribution directory hold node and each selected Harness.
-// Without a Core address nothing is downloaded and a missing component is an error.
-func obtainNativeComponents(ctx context.Context, out io.Writer, o *nativeInstallOptions, selected []string) error {
-	var missing []string
-	for _, name := range append([]string{"node"}, selected...) {
-		_, inBundle := os.Lstat(nativeComponentRoot(o.Bundle, name))
-		_, installed := os.Lstat(nativeComponentRoot(o.Directory, name))
-		if inBundle != nil && installed != nil {
-			missing = append(missing, name)
-		}
-	}
-	if len(missing) == 0 {
-		return nil
-	}
-	if o.OnboardURL == "" {
-		return fmt.Errorf("install: component %s is missing from %s; extract %s-%s-%s.tar.gz into it and retry", missing[0], o.Bundle, runtime.GOOS, runtime.GOARCH, missing[0])
-	}
-	return fetchNativeComponents(ctx, out, o.OnboardURL, o.Bundle, missing)
-}
-
-// fetchNativeComponents downloads the archive of each named component and extracts it into the
-// distribution directory. Verified archives stay in the Runtime home, keyed by version, so a later
-// installation of the same release reuses them. This is the only owner of that directory's lock.
-func fetchNativeComponents(ctx context.Context, out io.Writer, onboardURL, bundle string, names []string) error {
+// nativeDownloadDirectory holds only transient files: the installer's bundle.* directories and the
+// partial archive that a retry or a later run resumes.
+func nativeDownloadDirectory() (string, error) {
 	home, err := paths.Root()
 	if err != nil {
-		return err
+		return "", err
 	}
 	cache := filepath.Join(home, "native-download")
 	if err = runtimefs.EnsurePrivateDir(cache); err != nil {
-		return err
-	}
-	held, err := os.OpenRoot(cache)
-	if err != nil {
-		return err
-	}
-	defer held.Close()
-	unlock, err := runtimefs.LockDirectory(held)
-	if err != nil {
-		return errors.New("install: another native download is running; wait for it to finish and retry")
+		return "", err
 	}
-	defer unlock()
 	pruneNativeDownloads(cache)
 	dir := filepath.Join(cache, Version)
-	if err = os.MkdirAll(dir, 0700); err != nil {
-		return err
+	return dir, os.MkdirAll(dir, 0700)
+}
+
+// downloadNativeComponent fetches the archive of one component, verifies it against Core's
+// checksum and extracts it into stage. It returns the archive, which the caller deletes once the
+// component is published.
+func downloadNativeComponent(ctx context.Context, out io.Writer, onboardURL, name, stage string, expected nativeComponent) (string, error) {
+	dir, err := nativeDownloadDirectory()
+	if err != nil {
+		return "", err
 	}
-	for _, name := range names {
-		file := runtime.GOOS + "-" + runtime.GOARCH + "-" + name
-		archive, err := fetchNativeArchive(ctx, out, nativeInstallerBase(onboardURL)+"/"+file, dir, file)
-		if err == nil {
-			err = extractNativeComponent(ctx, out, archive, bundle, name)
-		}
-		if err != nil {
-			return err
-		}
+	file := runtime.GOOS + "-" + runtime.GOARCH + "-" + name
+	archive, err := fetchNativeArchive(ctx, out, nativeInstallerBase(onboardURL)+"/"+file, dir, file)
+	if err != nil {
+		return "", err
 	}
-	return nil
+	return archive, extractNativeComponent(ctx, out, archive, stage, name, expected)
 }
 
 // pruneNativeDownloads keeps the current version and recent distribution directories of running
@@ -118,22 +90,13 @@ func pruneNativeDownloads(cache string) {
 	}
 }
 
-// fetchNativeArchive returns the verified cached archive, downloading or resuming it when needed.
+// fetchNativeArchive returns the verified archive, downloading or resuming it as needed.
 func fetchNativeArchive(ctx context.Context, out io.Writer, url, dir, file string) (string, error) {
 	want, err := fetchNativeChecksum(ctx, out, url+".sha256")
 	if err != nil {
 		return "", err
 	}
-	archive := filepath.Join(dir, file+".tar.gz")
-	if got, err := nativeFileDigest(ctx, archive); err == nil {
-		if got == want {
-			fmt.Fprintf(out, "Using the downloaded %s archive.\n", file)
-			return archive, nil
-		}
-		fmt.Fprintf(out, "The downloaded %s archive failed verification and was discarded.\n", file)
-		_ = os.Remove(archive)
-	}
-	part := archive + ".part"
+	part := filepath.Join(dir, file+".tar.gz.part")
 	for pass := 0; ; pass++ {
 		resumed := false
 		if info, err := os.Stat(part); err == nil && info.Size() > 0 {
@@ -148,7 +111,7 @@ func fetchNativeArchive(ctx context.Context, out io.Writer, url, dir, file strin
 			return "", err
 		}
 		if got == want {
-			return archive, os.Rename(part, archive)
+			return part, nil
 		}
 		_ = os.Remove(part)
 		// A resumed file may have been damaged earlier; one clean download decides.
@@ -329,39 +292,38 @@ func nativeFileDigest(ctx context.Context, path string) (string, error) {
 	return hex.EncodeToString(hash.Sum(nil)), nil
 }
 
-// extractNativeComponent unpacks components/<name>/ from a verified archive into the distribution
-// directory. Only regular files and directories under that path are accepted; per-file digests are
-// checked later against bundle.json.
-func extractNativeComponent(ctx context.Context, out io.Writer, archive, bundle, name string) error {
+// extractNativeComponent unpacks components/<name>/ from a verified archive into stage. Only regular
+// files and directories under that path are accepted, and only files that bundle.json lists; every
+// file is then checked against its digest.
+func extractNativeComponent(ctx context.Context, out io.Writer, archive, stage, name string, expected nativeComponent) error {
 	var size uint64
 	if err := eachNativeEntry(ctx, archive, name, func(h *tar.Header, _ string, _ io.Reader) error {
 		size += uint64(h.Size)
 		return nil
 	}); err != nil {
 		return err
 	}
-	parent := filepath.Join(bundle, "components")
-	if err := os.MkdirAll(parent, 0700); err != nil {
-		return err
-	}
-	if err := requireNativeSpace(parent, size); err != nil {
+	if err := requireNativeSpace(filepath.Dir(stage), size); err != nil {
 		return err
 	}
 	fmt.Fprintf(out, "Extracting %s...\n", name)
-	stage := filepath.Join(parent, ".extract-"+name)
-	if err := os.RemoveAll(stage); err != nil {
-		return err
-	}
-	defer os.RemoveAll(stage)
 	if err := eachNativeEntry(ctx, archive, name, func(h *tar.Header, rel string, r io.Reader) error {
 		target := filepath.Join(stage, filepath.FromSlash(rel))
 		if h.Typeflag == tar.TypeDir {
 			return os.MkdirAll(target, 0700)
 		}
+		listed, ok := expected.Files[rel]
+		if !ok {
+			return errors.New("install: component archive holds a file that bundle.json does not list")
+		}
 		if err := os.MkdirAll(filepath.Dir(target), 0700); err != nil {
 			return err
 		}
-		file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
+		mode := os.FileMode(0600)
+		if listed.Executable {
+			mode = 0700
+		}
+		file, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode)
 		if err != nil {
 			return err
 		}
@@ -373,7 +335,13 @@ func extractNativeComponent(ctx context.Context, out io.Writer, archive, bundle,
 	}); err != nil {
 		return err
 	}
-	return os.Rename(stage, nativeComponentRoot(bundle, name))
+	if err := checkComponentFiles(ctx, stage, expected); err != nil {
+		if errors.Is(err, errNativeComponentMismatch) || errors.Is(err, os.ErrNotExist) {
+			return errors.New("install: component archive does not match bundle.json; Core serves inconsistent installer files")
+		}
+		return err
+	}
+	return nil
 }
 
 // eachNativeEntry visits the files and directories of a component archive; the root directory itself is skipped.