age: compress large test files

Author: FiloSottile

age/README.md

@@ -51,8 +51,8 @@ processed independently. Each vector is meant to test only one failure (or
 success) scenario.
 
 The file is in two parts: first a textual header, then an empty line, and then
-an age encrypted file. The textual header is a series of key-value pairs,
-separated by a colon and a space, each on their own line.
+an age encrypted file, possibly compressed. The textual header is a series of
+key-value pairs, separated by a colon and a space, each on their own line.
 
 The following header keys are defined. Files with unknown keys should be
 ignored.
@@ -97,6 +97,13 @@ ignored.
 
     The ASCII armor should fail to parse successfully.
 
+- `compressed`
+
+  This key will be `gzip` if the age encrypted file is compressed with gzip.
+  Note that encrypted files usually don't compress well, but large test files in
+  this collection are generated from plaintexts selected to make the ciphertext
+  compressible. **Some of these files can be several megabytes once decompressed.**
+
 - `payload`
 
   This is a hex-encoded SHA-256 hash of the payload. **All** the plaintext that

age/internal/testkit/testkit.go

@@ -6,10 +6,10 @@ package testkit
 
 import (
 	"bytes"
+	"compress/zlib"
 	"crypto/hmac"
 	"crypto/sha256"
 	"encoding/base64"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -31,15 +31,7 @@ var _, TestX25519Identity, _ = bech32.Decode(
 
 var TestX25519Recipient, _ = curve25519.X25519(TestX25519Identity, curve25519.Basepoint)
 
-// These are the file key and nonce used to encrypt any full/multiple-chunk
-// tests. They were generated by a previous iteration of this test suite.
-// Reusing them across files and history makes the repository easier to pack and
-// the test suite easier to compress.
-var LargeTestFileKey, _ = hex.DecodeString("7aa5bdac0e6afeed3dd0a7eccb42af44")
-var LargeTestNonce, _ = hex.DecodeString("c82f71eb82029b77136399e485e879f4")
-var LargeTestFirstChunk = bytes.Repeat([]byte{0}, 64*1024)
-var LargeTestSecondChunk = bytes.Repeat([]byte{1}, 64*1024)
-var LargeTestThirdChunk = bytes.Repeat([]byte{2}, 64*1024)
+const ChunkSize = 64 * 1024
 
 func NotCanonicalBase64(s string) string {
 	// Assuming there are spare zero bits at the end of the encoded bitstring,
@@ -213,29 +205,43 @@ func (f *TestFile) HMAC() {
 	f.HMACLine(h.Sum(nil))
 }
 
-func (f *TestFile) Nonce(nonce []byte) {
+func (f *TestFile) Nonce() {
+	nonce := f.Rand(16)
 	f.streamKey = make([]byte, 32)
 	hkdf.New(sha256.New, f.fileKey, nonce, []byte("payload")).Read(f.streamKey)
 	f.Buf.Write(nonce)
 }
 
-func (f *TestFile) PayloadChunk(plaintext []byte) {
+func (f *TestFile) PayloadChunk(size int) {
+	plaintext := bytes.Repeat([]byte{0}, size)
+	s, _ := chacha20.NewUnauthenticatedCipher(f.streamKey, f.nonce[:])
+	s.SetCounter(1)
+	s.XORKeyStream(plaintext, plaintext)
+	f.payloadChunk(plaintext)
+}
+
+func (f *TestFile) payloadChunk(plaintext []byte) {
 	f.payload.Write(plaintext)
 	aead, _ := chacha20poly1305.New(f.streamKey)
 	f.Buf.Write(aead.Seal(nil, f.nonce[:], plaintext, nil))
-	f.nonce[10]++
+
+	for i := 10; i >= 0; i-- {
+		f.nonce[i]++
+		if f.nonce[i] != 0 {
+			break
+		}
+	}
 }
 
-func (f *TestFile) PayloadChunkFinal(plaintext []byte) {
-	f.payload.Write(plaintext)
+func (f *TestFile) PayloadChunkFinal(size int) {
 	f.nonce[11] = 1
-	aead, _ := chacha20poly1305.New(f.streamKey)
-	f.Buf.Write(aead.Seal(nil, f.nonce[:], plaintext, nil))
+	f.PayloadChunk(size)
 }
 
 func (f *TestFile) Payload(plaintext string) {
-	f.Nonce(f.Rand(16))
-	f.PayloadChunkFinal([]byte(plaintext))
+	f.Nonce()
+	f.nonce[11] = 1
+	f.payloadChunk([]byte(plaintext))
 }
 
 func (f *TestFile) ExpectHeaderFailure() {
@@ -305,6 +311,12 @@ func (f *TestFile) Generate() {
 	if f.comment != "" {
 		fmt.Printf("comment: %s\n", f.comment)
 	}
+	out := io.Writer(os.Stdout)
+	if f.Buf.Len() > 1024 {
+		fmt.Printf("compressed: zlib\n")
+		out, _ = zlib.NewWriterLevel(os.Stdout, zlib.BestCompression)
+		defer out.(*zlib.Writer).Close()
+	}
 	fmt.Println()
-	io.Copy(os.Stdout, &f.Buf)
+	io.Copy(out, &f.Buf)
 }

age/internal/tests/armor_garbage_encoded.go

@@ -11,14 +11,13 @@ import "c2sp.org/CCTV/age/internal/testkit"
 
 func main() {
 	f := testkit.NewTestFile()
-	f.FileKey(testkit.LargeTestFileKey)
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Identity)
 	f.HMAC()
-	f.Nonce(testkit.LargeTestNonce)
-	f.PayloadChunkFinal(testkit.LargeTestFirstChunk)
+	f.Nonce()
+	f.PayloadChunkFinal(testkit.ChunkSize)
 	f.Buf.Write(f.Rand(20))
-	f.ExpectPartialPayload(64 * 1024)
+	f.ExpectPartialPayload(testkit.ChunkSize)
 	file := f.Bytes()
 	f.Buf.Reset()
 	f.BeginArmor("AGE ENCRYPTED FILE")

age/internal/tests/stream_bad_tag_second_chunk.go

@@ -11,17 +11,16 @@ import "c2sp.org/CCTV/age/internal/testkit"
 
 func main() {
 	f := testkit.NewTestFile()
-	f.FileKey(testkit.LargeTestFileKey)
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Identity)
 	f.HMAC()
-	f.Nonce(testkit.LargeTestNonce)
-	f.PayloadChunk(testkit.LargeTestFirstChunk)
-	f.PayloadChunkFinal([]byte("age"))
+	f.Nonce()
+	f.PayloadChunk(testkit.ChunkSize)
+	f.PayloadChunkFinal(1)
 	file := f.Buf.Bytes()
 	f.Buf.Reset()
 	file[len(file)-1] ^= 0b0010_0000
 	f.Buf.Write(file)
-	f.ExpectPartialPayload(64 * 1024)
+	f.ExpectPartialPayload(testkit.ChunkSize)
 	f.Generate()
 }

age/internal/tests/stream_bad_tag_second_chunk_full.go

@@ -11,17 +11,16 @@ import "c2sp.org/CCTV/age/internal/testkit"
 
 func main() {
 	f := testkit.NewTestFile()
-	f.FileKey(testkit.LargeTestFileKey)
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Identity)
 	f.HMAC()
-	f.Nonce(testkit.LargeTestNonce)
-	f.PayloadChunk(testkit.LargeTestFirstChunk)
-	f.PayloadChunkFinal(testkit.LargeTestSecondChunk)
+	f.Nonce()
+	f.PayloadChunk(testkit.ChunkSize)
+	f.PayloadChunkFinal(testkit.ChunkSize)
 	file := f.Buf.Bytes()
 	f.Buf.Reset()
 	file[len(file)-1] ^= 0b0010_0000
 	f.Buf.Write(file)
-	f.ExpectPartialPayload(64 * 1024)
+	f.ExpectPartialPayload(testkit.ChunkSize)
 	f.Generate()
 }

age/internal/tests/stream_last_chunk_empty.go

@@ -11,14 +11,13 @@ import "c2sp.org/CCTV/age/internal/testkit"
 
 func main() {
 	f := testkit.NewTestFile()
-	f.FileKey(testkit.LargeTestFileKey)
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Identity)
 	f.HMAC()
-	f.Nonce(testkit.LargeTestNonce)
-	f.PayloadChunk(testkit.LargeTestFirstChunk)
-	f.PayloadChunkFinal([]byte{})
+	f.Nonce()
+	f.PayloadChunk(testkit.ChunkSize)
+	f.PayloadChunkFinal(0)
 	f.Comment("final STREAM chunk can't be empty unless whole payload is empty")
-	f.ExpectPartialPayload(64 * 1024)
+	f.ExpectPartialPayload(testkit.ChunkSize)
 	f.Generate()
 }

age/internal/tests/stream_last_chunk_full.go

@@ -11,11 +11,10 @@ import "c2sp.org/CCTV/age/internal/testkit"
 
 func main() {
 	f := testkit.NewTestFile()
-	f.FileKey(testkit.LargeTestFileKey)
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Identity)
 	f.HMAC()
-	f.Nonce(testkit.LargeTestNonce)
-	f.PayloadChunkFinal(testkit.LargeTestFirstChunk)
+	f.Nonce()
+	f.PayloadChunkFinal(testkit.ChunkSize)
 	f.Generate()
 }

age/internal/tests/stream_last_chunk_full_second.go

@@ -11,12 +11,11 @@ import "c2sp.org/CCTV/age/internal/testkit"
 
 func main() {
 	f := testkit.NewTestFile()
-	f.FileKey(testkit.LargeTestFileKey)
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Identity)
 	f.HMAC()
-	f.Nonce(testkit.LargeTestNonce)
-	f.PayloadChunk(testkit.LargeTestFirstChunk)
-	f.PayloadChunkFinal(testkit.LargeTestSecondChunk)
+	f.Nonce()
+	f.PayloadChunk(testkit.ChunkSize)
+	f.PayloadChunkFinal(testkit.ChunkSize)
 	f.Generate()
 }

age/internal/tests/stream_no_chunks.go

@@ -14,7 +14,7 @@ func main() {
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Recipient)
 	f.HMAC()
-	f.Nonce(f.Rand(16))
+	f.Nonce()
 	f.ExpectPayloadFailure()
 	f.Generate()
 }

age/internal/tests/stream_no_final.go

@@ -14,8 +14,8 @@ func main() {
 	f.VersionLine("v1")
 	f.X25519(testkit.TestX25519Recipient)
 	f.HMAC()
-	f.Nonce(f.Rand(16))
-	f.PayloadChunk([]byte("age"))
+	f.Nonce()
+	f.PayloadChunk(1)
 	f.ExpectPayloadFailure()
 	f.Generate()
 }