ENG-1643: Add end-to-end integration test for encrypted publish and install

internal/artifactcrypto/artifactcrypto.go

@@ -54,6 +54,10 @@ const (
 	formatVersion = 1
 	// DefaultChunkSize is the plaintext size of every chunk except the last.
 	DefaultChunkSize = 1 << 20 // 1 MiB
+	// PayloadFilename is the conventional name of the encrypted payload file
+	// inside a private-ingredient artifact. The publish (produce) and runtime
+	// (consume) sides share it so they agree on what to write and look for.
+	PayloadFilename = "payload.enc"
 	// maxChunkSize is the largest chunk size accepted from a parsed header.
 	maxChunkSize = 64 << 20 // 64 MiB
 	// maxHeaderLen is the largest serialized header accepted from a stream.

internal/constants/constants.go

@@ -557,3 +557,10 @@ const PrivateIngredientBearerTokenFileConfig = "privateingredient.bearer_token_f
 // PrivateIngredientCacheKeyConfig is the config key that opts into caching the
 // fetched org key on disk (0600) for headless/offline/CI reuse.
 const PrivateIngredientCacheKeyConfig = "privateingredient.cache_key_on_disk"
+
+// PrivateIngredientKeyContractEnvVarName is the name of an environment variable
+// that may carry an org-key contract (the same JSON document the HTTPS key
+// service serves). When set, the org key is read from it and validated exactly
+// like a fetched contract, bypassing the network. It exists for development and
+// integration testing where standing up a TLS key service is impractical.
+const PrivateIngredientKeyContractEnvVarName = "ACTIVESTATE_CLI_PRIVATE_INGREDIENT_KEY_CONTRACT"

internal/runbits/orgkey/env.go

@@ -13,4 +13,5 @@ func SanitizeChildEnv(cfg stringConfigReader, env map[string]string) {
 	if tokenEnv := cfg.GetString(constants.PrivateIngredientBearerTokenEnvConfig); tokenEnv != "" {
 		delete(env, tokenEnv)
 	}
+	delete(env, constants.PrivateIngredientKeyContractEnvVarName)
 }

internal/runbits/orgkey/https.go

@@ -43,7 +43,18 @@ func New(cfg configurable, owner string) Provider {
 }
 
 func (p *provider) Configured() bool {
-	return p.cfg.GetString(constants.PrivateIngredientKeyServiceURLConfig) != ""
+	return p.cfg.GetString(constants.PrivateIngredientKeyServiceURLConfig) != "" || len(envContract()) > 0
+}
+
+// envContract returns an org-key contract supplied directly via the environment,
+// or nil if none is set. This bypasses the HTTPS key service for development and
+// integration testing; the contract is still validated like one fetched from the
+// service (see load).
+func envContract() []byte {
+	if v := os.Getenv(constants.PrivateIngredientKeyContractEnvVarName); v != "" {
+		return []byte(v)
+	}
+	return nil
 }
 
 // Key fetches and validates the org key on first call and returns the cached
@@ -73,6 +84,10 @@ func (p *provider) load(ctx context.Context) (key []byte, keyID string, err erro
 		return nil, "", ErrNotConfigured
 	}
 
+	if raw := envContract(); len(raw) > 0 {
+		return validateContract(raw, p.owner)
+	}
+
 	if p.diskCacheEnabled() {
 		if raw, ok := p.readDiskCache(); ok {
 			if key, keyID, err := validateContract(raw, p.owner); err == nil {

internal/runners/publish/build.go

@@ -2,9 +2,7 @@ package publish
 
 import (
 	"context"
-	"encoding/json"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -79,19 +77,19 @@ func requireOrgNamespace(ns, owner string) error {
 		"The '[ACTIONABLE]--build[/RESET]' flag requires a namespace under '[ACTIONABLE]{{.V0}}[/RESET]'.", org)
 }
 
-// payloadInstallDir is the directory inside the wrapped artifact that holds the
-// deployable payload; the cleartext runtime.json points the consume side at it.
-const payloadInstallDir = "install"
+const (
+	privateBuilderNamespace = "builder"
+	privateBuilderName      = "private-builder"
+)
 
 // buildWrappedArtifact packs srcDir into a wheel under the given metadata,
-// encrypts it under the org key, and wraps the ciphertext together with a
-// cleartext runtime.json into a tar.gz ready for upload. It returns the wrapped
-// archive path and a cleanup function the caller must invoke once the upload is
-// done.
+// encrypts it under the org key, and wraps the ciphertext in a tar.gz ready for
+// upload. It returns the wrapped archive path and a cleanup function the caller
+// must invoke once the upload is done.
 //
-// Only ciphertext plus the cleartext envdef ever reaches the wrapped archive:
-// the plaintext wheel and payload are removed before the function returns, so no
-// plaintext outlives the build.
+// Only the ciphertext ever reaches the wrapped archive: the plaintext wheel and
+// payload are removed before the function returns, so no plaintext outlives the
+// build.
 func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID string) (archivePath string, cleanup func(), rerr error) {
 	tmpDir, err := os.MkdirTemp("", "state-publish-build-")
 	if err != nil {
@@ -109,16 +107,16 @@ func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID
 		return "", nil, errs.Wrap(err, "Could not build a wheel from %s", srcDir)
 	}
 
-	// Assemble the tar.gz that becomes the encrypted payload, placing the wheel
-	// under the install dir the consume side deploys.
+	// Assemble the tar.gz that becomes the encrypted payload, with the wheel at
+	// its root.
 	plaintextPayload := filepath.Join(tmpDir, "payload.tar.gz")
 	if err := archiver.CreateTgz(plaintextPayload, tmpDir, []archiver.FileMap{
-		{Source: wheelPath, Target: path.Join(payloadInstallDir, filepath.Base(wheelPath))},
+		{Source: wheelPath, Target: filepath.Base(wheelPath)},
 	}); err != nil {
 		return "", nil, errs.Wrap(err, "Could not assemble payload")
 	}
 
-	ciphertextPath := filepath.Join(tmpDir, "payload.enc")
+	ciphertextPath := filepath.Join(tmpDir, artifactcrypto.PayloadFilename)
 	if err := encryptFile(plaintextPayload, ciphertextPath, key, keyID); err != nil {
 		return "", nil, errs.Wrap(err, "Could not encrypt payload")
 	}
@@ -131,15 +129,9 @@ func buildWrappedArtifact(srcDir string, meta wheel.Metadata, key []byte, keyID
 		}
 	}
 
-	runtimeJSONPath := filepath.Join(tmpDir, "runtime.json")
-	if err := writeRuntimeJSON(runtimeJSONPath); err != nil {
-		return "", nil, errs.Wrap(err, "Could not write runtime.json")
-	}
-
 	archivePath = filepath.Join(tmpDir, "ingredient.tar.gz")
 	if err := archiver.CreateTgz(archivePath, tmpDir, []archiver.FileMap{
-		{Source: ciphertextPath, Target: "payload.enc"},
-		{Source: runtimeJSONPath, Target: "runtime.json"},
+		{Source: ciphertextPath, Target: artifactcrypto.PayloadFilename},
 	}); err != nil {
 		return "", nil, errs.Wrap(err, "Could not wrap artifact")
 	}
@@ -179,25 +171,3 @@ func encryptFile(srcPath, dstPath string, key []byte, keyID string) (rerr error)
 	}
 	return nil
 }
-
-// writeRuntimeJSON writes the minimal cleartext envdef the consume side reads to
-// deploy the decrypted payload.
-func writeRuntimeJSON(destPath string) error {
-	def := struct {
-		Env        []json.RawMessage `json:"env"`
-		Transforms []json.RawMessage `json:"file_transforms"`
-		InstallDir string            `json:"installdir"`
-	}{
-		Env:        []json.RawMessage{},
-		Transforms: []json.RawMessage{},
-		InstallDir: payloadInstallDir,
-	}
-	b, err := json.Marshal(def)
-	if err != nil {
-		return errs.Wrap(err, "Could not marshal runtime.json")
-	}
-	if err := os.WriteFile(destPath, b, 0644); err != nil {
-		return errs.Wrap(err, "Could not write runtime.json")
-	}
-	return nil
-}

internal/runners/publish/build_test.go

@@ -4,7 +4,6 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
 	"io"
 	"os"
 	"path/filepath"
@@ -110,23 +109,12 @@ func TestBuildWrappedArtifact(t *testing.T) {
 	}
 	defer cleanup()
 
-	// The wrapped archive contains exactly the ciphertext and the cleartext envdef.
+	// The wrapped archive contains exactly the ciphertext.
 	entries := readTarGz(t, archivePath)
-	if got, want := keysOf(entries), []string{"payload.enc", "runtime.json"}; !equalStrings(got, want) {
+	if got, want := keysOf(entries), []string{"payload.enc"}; !equalStrings(got, want) {
 		t.Fatalf("wrapped entries = %v, want %v", got, want)
 	}
 
-	// runtime.json is the minimal envdef pointing at the install dir.
-	var def struct {
-		InstallDir string `json:"installdir"`
-	}
-	if err := json.Unmarshal(entries["runtime.json"], &def); err != nil {
-		t.Fatal(err)
-	}
-	if def.InstallDir != payloadInstallDir {
-		t.Errorf("installdir = %q, want %q", def.InstallDir, payloadInstallDir)
-	}
-
 	// The payload is encrypted before it is wrapped.
 	enc, err := artifactcrypto.IsEncrypted(bytes.NewReader(entries["payload.enc"]))
 	if err != nil {
@@ -143,19 +131,19 @@ func TestBuildWrappedArtifact(t *testing.T) {
 		}
 	}
 
-	// The ciphertext decrypts to a tar.gz holding the wheel under the install dir.
+	// The ciphertext decrypts to a tar.gz holding the wheel.
 	innerPath := filepath.Join(t.TempDir(), "inner.tar.gz")
 	if err := artifactcrypto.Decrypt(bytes.NewReader(entries["payload.enc"]), innerPath, key); err != nil {
 		t.Fatalf("Decrypt: %v", err)
 	}
 	foundWheel := false
 	for _, name := range keysOf(readTarGz(t, innerPath)) {
-		if strings.HasPrefix(name, payloadInstallDir+"/") && strings.HasSuffix(name, ".whl") {
+		if strings.HasSuffix(name, ".whl") {
 			foundWheel = true
 		}
 	}
 	if !foundWheel {
-		t.Error("decrypted payload does not contain the wheel under the install dir")
+		t.Error("decrypted payload does not contain the wheel")
 	}
 
 	// cleanup removes the build temp dir.

internal/runners/publish/publish.go

@@ -222,6 +222,19 @@ func (r *Runner) Run(params *Params) error {
 		return errs.Wrap(err, "Could not prepare request from params")
 	}
 
+	if params.Build != "" {
+		// A --build publish must be ingested by the private-builder. Without a
+		// builder dependency the platform falls back to the noop-builder, which
+		// produces an empty artifact.
+		reqVars.Dependencies = append(reqVars.Dependencies, request.PublishVariableDep{
+			Dependency: request.Dependency{
+				Name:                privateBuilderName,
+				Namespace:           privateBuilderNamespace,
+				VersionRequirements: ">=0",
+			},
+		})
+	}
+
 	if params.Editor {
 		if !r.out.Config().Interactive {
 			return locale.NewInputError("err_uploadingredient_editor_not_supported", "Opening in editor is not supported in non-interactive mode")

pkg/runtime/decrypt_test.go

@@ -72,6 +72,9 @@ func makeTarGz(t *testing.T, files, symlinks map[string]string) []byte {
 
 func writeFile(t *testing.T, path string, data []byte) {
 	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		t.Fatal(err)
+	}
 	if err := os.WriteFile(path, data, 0644); err != nil {
 		t.Fatal(err)
 	}
@@ -96,7 +99,7 @@ func TestFindEncryptedPayload(t *testing.T) {
 	t.Run("finds the encrypted file", func(t *testing.T) {
 		dir := t.TempDir()
 		writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`))
-		payload := filepath.Join(dir, "anything.bin")
+		payload := filepath.Join(dir, artifactcrypto.PayloadFilename)
 		writeFile(t, payload, encryptToBytes(t, []byte("secret"), key))
 		got, err := findEncryptedPayload(dir)
 		if err != nil {
@@ -106,6 +109,35 @@ func TestFindEncryptedPayload(t *testing.T) {
 			t.Errorf("got %q, want %q", got, payload)
 		}
 	})
+
+	// Installed payload is under top-level "installdir/" directory.
+	t.Run("finds encrypted payload", func(t *testing.T) {
+		dir := t.TempDir()
+		writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"installdir"}`))
+		payload := filepath.Join(dir, "installdir", artifactcrypto.PayloadFilename)
+		writeFile(t, payload, encryptToBytes(t, []byte("secret"), key))
+		got, err := findEncryptedPayload(dir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got != payload {
+			t.Errorf("got %q, want %q", got, payload)
+		}
+	})
+
+	// A plaintext file that happens to be named payload.enc is not treated as a
+	// payload.
+	t.Run("ignores a plaintext payload.enc", func(t *testing.T) {
+		dir := t.TempDir()
+		writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), []byte("not encrypted"))
+		got, err := findEncryptedPayload(dir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got != "" {
+			t.Errorf("found a payload for a plaintext file: %q", got)
+		}
+	})
 }
 
 func TestDecryptPayload(t *testing.T) {
@@ -123,7 +155,7 @@ func TestDecryptPayload(t *testing.T) {
 	t.Run("happy path", func(t *testing.T) {
 		dir := t.TempDir()
 		writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"."}`))
-		writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key))
+		writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key))
 
 		s := &setup{opts: &Opts{OrgKey: key}}
 		outcome, err := s.decryptPayload("pkg", dir)
@@ -134,7 +166,7 @@ func TestDecryptPayload(t *testing.T) {
 			t.Fatalf("outcome = %v, want decryptDone", outcome)
 		}
 		// Ciphertext is removed.
-		if _, err := os.Stat(filepath.Join(dir, "payload")); !os.IsNotExist(err) {
+		if _, err := os.Stat(filepath.Join(dir, artifactcrypto.PayloadFilename)); !os.IsNotExist(err) {
 			t.Error("ciphertext was not removed")
 		}
 		// Archive contents extracted in place; the cleartext runtime.json survives.
@@ -163,7 +195,7 @@ func TestDecryptPayload(t *testing.T) {
 
 	t.Run("missing key skips", func(t *testing.T) {
 		dir := t.TempDir()
-		writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key))
+		writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key))
 
 		s := &setup{opts: &Opts{}} // no OrgKey
 		outcome, err := s.decryptPayload("pkg", dir)
@@ -177,7 +209,7 @@ func TestDecryptPayload(t *testing.T) {
 
 	t.Run("wrong key fails closed", func(t *testing.T) {
 		dir := t.TempDir()
-		writeFile(t, filepath.Join(dir, "payload"), encryptToBytes(t, payload, key))
+		writeFile(t, filepath.Join(dir, artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key))
 
 		wrong := make([]byte, artifactcrypto.KeySize) // all zeros
 		s := &setup{opts: &Opts{OrgKey: wrong}}
@@ -199,6 +231,30 @@ func TestDecryptPayload(t *testing.T) {
 			t.Fatalf("outcome = %v, want decryptNotEncrypted", outcome)
 		}
 	})
+
+	// Ensure the decrypted contents land in that same dir so the runtime.json installdir resolves.
+	t.Run("nested payload extracts in place", func(t *testing.T) {
+		dir := t.TempDir()
+		writeFile(t, filepath.Join(dir, "runtime.json"), []byte(`{"installDir":"installdir"}`))
+		writeFile(t, filepath.Join(dir, "installdir", artifactcrypto.PayloadFilename), encryptToBytes(t, payload, key))
+
+		s := &setup{opts: &Opts{OrgKey: key}}
+		outcome, err := s.decryptPayload("pkg", dir)
+		if err != nil {
+			t.Fatalf("decryptPayload: %v", err)
+		}
+		if outcome != decryptDone {
+			t.Fatalf("outcome = %v, want decryptDone", outcome)
+		}
+		// Ciphertext is removed and the archive is extracted alongside it, under
+		// the install dir.
+		if _, err := os.Stat(filepath.Join(dir, "installdir", artifactcrypto.PayloadFilename)); !os.IsNotExist(err) {
+			t.Error("ciphertext was not removed")
+		}
+		if got, _ := os.ReadFile(filepath.Join(dir, "installdir", "pkg", "__init__.py")); string(got) != "print('private')\n" {
+			t.Errorf("payload not extracted under the install dir: got %q", got)
+		}
+	})
 }
 
 func TestPrivateArtifactSurvivesEviction(t *testing.T) {