diff --git a/src/aeg/_typing.py b/src/aeg/_typing.py index d4149ca..f493f09 100644 --- a/src/aeg/_typing.py +++ b/src/aeg/_typing.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Protocol if TYPE_CHECKING: + from .random import Random from .util import Buffer __all__ = ["Cipher"] @@ -43,6 +44,8 @@ class Cipher(Protocol): Encryptor: type[_Encryptor] Decryptor: type[_Decryptor] + @staticmethod + def random() -> "Random": ... @staticmethod def random_key() -> bytearray: ... @staticmethod @@ -124,3 +127,35 @@ class Cipher(Protocol): def nonce_increment(nonce: "Buffer") -> None: ... @staticmethod def wipe(buffer: "Buffer") -> None: ... + + # random-access encrypted files (thin wrappers; see aeg.raf) + RAF_ALG_ID: int + + @staticmethod + def new_raf_ctx(): ... + @staticmethod + def raf_scratch_size(chunk_size: int) -> int: ... + @staticmethod + def raf_create(key: "Buffer", ctx, io, rng, config) -> None: ... + @staticmethod + def raf_open(key: "Buffer", ctx, io, rng, config) -> None: ... + @staticmethod + def raf_read( + ctx, length: int, offset: int, *, into: "Buffer | None" = None + ) -> bytearray | memoryview: ... + @staticmethod + def raf_write(ctx, data: "Buffer", offset: int) -> int: ... + @staticmethod + def raf_truncate(ctx, size: int) -> None: ... + @staticmethod + def raf_size(ctx) -> int: ... + @staticmethod + def raf_sync(ctx) -> None: ... + @staticmethod + def raf_close(ctx) -> None: ... + @staticmethod + def raf_merkle_rebuild(ctx) -> None: ... + @staticmethod + def raf_merkle_verify(ctx) -> "int | None": ... + @staticmethod + def raf_merkle_commitment(ctx, hash_len: int) -> bytes: ... diff --git a/src/aeg/aegis128l.py b/src/aeg/aegis128l.py index 4e3638c..6673b05 100644 --- a/src/aeg/aegis128l.py +++ b/src/aeg/aegis128l.py @@ -4,10 +4,12 @@ import errno import secrets +import sys from typing import Literal from ._loader import ffi from ._loader import lib as _lib +from .random import Random from .util import Buffer, new_aligned_struct, nonce_increment, wipe NAME = "AEGIS-128L" #: Algorithm display name @@ -19,6 +21,15 @@ ALIGNMENT = 64 #: Required alignment for internal structures RATE = 64 #: Byte chunk size in internal processing +def random() -> Random: + """Return a new AEGIS stream Random instance using this cipher module. + + Seeded once from the OS, then deterministic: successive calls never + repeat an output block. See aeg.random.Random for the API. + """ + return Random(sys.modules[__name__]) + + def random_key() -> bytearray: """ Generate a secret key using cryptographically secure random bytes. @@ -868,6 +879,220 @@ def new_mac_state(): return new_aligned_struct("aegis128l_mac_state", ALIGNMENT) +RAF_ALG_ID = 1 #: Algorithm identifier in RAF file headers + + +def _raise_errno(operation: str) -> None: + err_num = ffi.errno + err_name = errno.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"{operation} failed: {err_name}") + + +def new_raf_ctx(): + """Allocate and return a new aegis128l_raf_ctx* with proper alignment.""" + return new_aligned_struct("aegis128l_raf_ctx", ALIGNMENT) + + +def raf_scratch_size(chunk_size: int) -> int: + """Return the scratch buffer size required for the given chunk_size.""" + return _lib.aegis128l_raf_scratch_size(chunk_size) + + +def raf_create(key: Buffer, ctx, io, rng, config) -> None: + """Create a new encrypted file, writing the header through io. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileExistsError: If the file exists and AEGIS_RAF_TRUNCATE was not set. + RuntimeError: If creation fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis128l_raf_create(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + if ffi.errno == errno.EEXIST: + raise FileExistsError("file exists (use truncate to overwrite)") + _raise_errno("raf create") + + +def raf_open(key: Buffer, ctx, io, rng, config) -> None: + """Open an existing encrypted file, verifying the header MAC. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileNotFoundError: If the file does not exist. + ValueError: If the header is invalid or authentication fails (wrong key). + RuntimeError: If opening fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis128l_raf_open(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + err_num = ffi.errno + if err_num == errno.ENOENT: + raise FileNotFoundError("file not found") + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf open") + + +def raf_read( + ctx, length: int, offset: int, *, into: Buffer | None = None +) -> bytearray | memoryview: + """Read and decrypt up to length bytes at offset. + + Returns: + The decrypted bytes, trimmed to the number actually read + (fewer at EOF), as bytearray if into not provided, + memoryview of into otherwise. + + Raises: + TypeError: If into is smaller than length. + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + if into is None: + out = bytearray(length) + else: + into = memoryview(into) + if into.nbytes < length: + raise TypeError("into length must be at least length") + out = into + bytes_read = ffi.new("size_t*") + rc = _lib.aegis128l_raf_read( + ctx.ptr, ffi.from_buffer(out), bytes_read, length, offset + ) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf read") + n = bytes_read[0] + return out[:n] if into is None else memoryview(out)[:n] # type: ignore + + +def raf_write(ctx, data: Buffer, offset: int) -> int: + """Encrypt and write data at offset, extending the file as needed. + + Returns: + Number of bytes written. + + Raises: + RuntimeError: If the write fails (errno name included). + """ + data = memoryview(data) + bytes_written = ffi.new("size_t*") + rc = _lib.aegis128l_raf_write( + ctx.ptr, bytes_written, _ptr(data), data.nbytes, offset + ) + if rc != 0: + _raise_errno("raf write") + return bytes_written[0] + + +def raf_truncate(ctx, size: int) -> None: + """Resize the file to size bytes (zero-filled when growing). + + Raises: + RuntimeError: If the truncate fails. + """ + rc = _lib.aegis128l_raf_truncate(ctx.ptr, size) + if rc != 0: + _raise_errno("raf truncate") + + +def raf_size(ctx) -> int: + """Return the logical plaintext file size.""" + size = ffi.new("uint64_t*") + rc = _lib.aegis128l_raf_get_size(ctx.ptr, size) + if rc != 0: + _raise_errno("raf get_size") + return size[0] + + +def raf_sync(ctx) -> None: + """Flush writes to the backing store. + + Raises: + RuntimeError: If the sync fails. + """ + rc = _lib.aegis128l_raf_sync(ctx.ptr) + if rc != 0: + _raise_errno("raf sync") + + +def raf_close(ctx) -> None: + """Close the context, syncing and zeroizing key material.""" + _lib.aegis128l_raf_close(ctx.ptr) + + +def raf_merkle_rebuild(ctx) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Raises: + RuntimeError: If Merkle is not enabled or rebuilding fails. + ValueError: If a chunk fails authentication. + """ + rc = _lib.aegis128l_raf_merkle_rebuild(ctx.ptr) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf merkle_rebuild") + + +def raf_merkle_verify(ctx) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + + Raises: + RuntimeError: If Merkle is not enabled or verification fails. + """ + corrupted = ffi.new("uint64_t*") + rc = _lib.aegis128l_raf_merkle_verify(ctx.ptr, corrupted) + if rc == 0: + return None + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + return corrupted[0] + _raise_errno("raf merkle_verify") + + +def raf_merkle_commitment(ctx, hash_len: int) -> bytes: + """Compute the Merkle root commitment hash. + + Returns: + The commitment hash as bytes of length hash_len. + + Raises: + RuntimeError: If Merkle is not enabled or the call fails. + """ + out = ffi.new(f"uint8_t[{hash_len}]") + rc = _lib.aegis128l_raf_merkle_commitment(ctx.ptr, out, hash_len) + if rc != 0: + _raise_errno("raf merkle_commitment") + return bytes(ffi.buffer(out, hash_len)) + + __all__ = [ # constants "NAME", @@ -877,7 +1102,9 @@ __all__ = [ "MACBYTES_LONG", "ALIGNMENT", "RATE", + "RAF_ALG_ID", # utility functions + "random", "random_key", "random_nonce", "nonce_increment", @@ -895,4 +1122,18 @@ __all__ = [ "Encryptor", "Decryptor", "Mac", + # random-access encrypted files (thin wrappers; see aeg.raf for the high-level API) + "new_raf_ctx", + "raf_scratch_size", + "raf_create", + "raf_open", + "raf_read", + "raf_write", + "raf_truncate", + "raf_size", + "raf_sync", + "raf_close", + "raf_merkle_rebuild", + "raf_merkle_verify", + "raf_merkle_commitment", ] diff --git a/src/aeg/aegis128x2.py b/src/aeg/aegis128x2.py index 83a29d8..c1f8180 100644 --- a/src/aeg/aegis128x2.py +++ b/src/aeg/aegis128x2.py @@ -4,10 +4,12 @@ import errno import secrets +import sys from typing import Literal from ._loader import ffi from ._loader import lib as _lib +from .random import Random from .util import Buffer, new_aligned_struct, nonce_increment, wipe NAME = "AEGIS-128X2" #: Algorithm display name @@ -19,6 +21,15 @@ ALIGNMENT = 64 #: Required alignment for internal structures RATE = 64 #: Byte chunk size in internal processing +def random() -> Random: + """Return a new AEGIS stream Random instance using this cipher module. + + Seeded once from the OS, then deterministic: successive calls never + repeat an output block. See aeg.random.Random for the API. + """ + return Random(sys.modules[__name__]) + + def random_key() -> bytearray: """ Generate a secret key using cryptographically secure random bytes. @@ -868,6 +879,220 @@ def new_mac_state(): return new_aligned_struct("aegis128x2_mac_state", ALIGNMENT) +RAF_ALG_ID = 2 #: Algorithm identifier in RAF file headers + + +def _raise_errno(operation: str) -> None: + err_num = ffi.errno + err_name = errno.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"{operation} failed: {err_name}") + + +def new_raf_ctx(): + """Allocate and return a new aegis128x2_raf_ctx* with proper alignment.""" + return new_aligned_struct("aegis128x2_raf_ctx", ALIGNMENT) + + +def raf_scratch_size(chunk_size: int) -> int: + """Return the scratch buffer size required for the given chunk_size.""" + return _lib.aegis128x2_raf_scratch_size(chunk_size) + + +def raf_create(key: Buffer, ctx, io, rng, config) -> None: + """Create a new encrypted file, writing the header through io. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileExistsError: If the file exists and AEGIS_RAF_TRUNCATE was not set. + RuntimeError: If creation fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis128x2_raf_create(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + if ffi.errno == errno.EEXIST: + raise FileExistsError("file exists (use truncate to overwrite)") + _raise_errno("raf create") + + +def raf_open(key: Buffer, ctx, io, rng, config) -> None: + """Open an existing encrypted file, verifying the header MAC. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileNotFoundError: If the file does not exist. + ValueError: If the header is invalid or authentication fails (wrong key). + RuntimeError: If opening fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis128x2_raf_open(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + err_num = ffi.errno + if err_num == errno.ENOENT: + raise FileNotFoundError("file not found") + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf open") + + +def raf_read( + ctx, length: int, offset: int, *, into: Buffer | None = None +) -> bytearray | memoryview: + """Read and decrypt up to length bytes at offset. + + Returns: + The decrypted bytes, trimmed to the number actually read + (fewer at EOF), as bytearray if into not provided, + memoryview of into otherwise. + + Raises: + TypeError: If into is smaller than length. + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + if into is None: + out = bytearray(length) + else: + into = memoryview(into) + if into.nbytes < length: + raise TypeError("into length must be at least length") + out = into + bytes_read = ffi.new("size_t*") + rc = _lib.aegis128x2_raf_read( + ctx.ptr, ffi.from_buffer(out), bytes_read, length, offset + ) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf read") + n = bytes_read[0] + return out[:n] if into is None else memoryview(out)[:n] # type: ignore + + +def raf_write(ctx, data: Buffer, offset: int) -> int: + """Encrypt and write data at offset, extending the file as needed. + + Returns: + Number of bytes written. + + Raises: + RuntimeError: If the write fails (errno name included). + """ + data = memoryview(data) + bytes_written = ffi.new("size_t*") + rc = _lib.aegis128x2_raf_write( + ctx.ptr, bytes_written, _ptr(data), data.nbytes, offset + ) + if rc != 0: + _raise_errno("raf write") + return bytes_written[0] + + +def raf_truncate(ctx, size: int) -> None: + """Resize the file to size bytes (zero-filled when growing). + + Raises: + RuntimeError: If the truncate fails. + """ + rc = _lib.aegis128x2_raf_truncate(ctx.ptr, size) + if rc != 0: + _raise_errno("raf truncate") + + +def raf_size(ctx) -> int: + """Return the logical plaintext file size.""" + size = ffi.new("uint64_t*") + rc = _lib.aegis128x2_raf_get_size(ctx.ptr, size) + if rc != 0: + _raise_errno("raf get_size") + return size[0] + + +def raf_sync(ctx) -> None: + """Flush writes to the backing store. + + Raises: + RuntimeError: If the sync fails. + """ + rc = _lib.aegis128x2_raf_sync(ctx.ptr) + if rc != 0: + _raise_errno("raf sync") + + +def raf_close(ctx) -> None: + """Close the context, syncing and zeroizing key material.""" + _lib.aegis128x2_raf_close(ctx.ptr) + + +def raf_merkle_rebuild(ctx) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Raises: + RuntimeError: If Merkle is not enabled or rebuilding fails. + ValueError: If a chunk fails authentication. + """ + rc = _lib.aegis128x2_raf_merkle_rebuild(ctx.ptr) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf merkle_rebuild") + + +def raf_merkle_verify(ctx) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + + Raises: + RuntimeError: If Merkle is not enabled or verification fails. + """ + corrupted = ffi.new("uint64_t*") + rc = _lib.aegis128x2_raf_merkle_verify(ctx.ptr, corrupted) + if rc == 0: + return None + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + return corrupted[0] + _raise_errno("raf merkle_verify") + + +def raf_merkle_commitment(ctx, hash_len: int) -> bytes: + """Compute the Merkle root commitment hash. + + Returns: + The commitment hash as bytes of length hash_len. + + Raises: + RuntimeError: If Merkle is not enabled or the call fails. + """ + out = ffi.new(f"uint8_t[{hash_len}]") + rc = _lib.aegis128x2_raf_merkle_commitment(ctx.ptr, out, hash_len) + if rc != 0: + _raise_errno("raf merkle_commitment") + return bytes(ffi.buffer(out, hash_len)) + + __all__ = [ # constants "NAME", @@ -877,7 +1102,9 @@ __all__ = [ "MACBYTES_LONG", "ALIGNMENT", "RATE", + "RAF_ALG_ID", # utility functions + "random", "random_key", "random_nonce", "nonce_increment", @@ -895,4 +1122,18 @@ __all__ = [ "Encryptor", "Decryptor", "Mac", + # random-access encrypted files (thin wrappers; see aeg.raf for the high-level API) + "new_raf_ctx", + "raf_scratch_size", + "raf_create", + "raf_open", + "raf_read", + "raf_write", + "raf_truncate", + "raf_size", + "raf_sync", + "raf_close", + "raf_merkle_rebuild", + "raf_merkle_verify", + "raf_merkle_commitment", ] diff --git a/src/aeg/aegis128x4.py b/src/aeg/aegis128x4.py index 03f42e7..234d136 100644 --- a/src/aeg/aegis128x4.py +++ b/src/aeg/aegis128x4.py @@ -4,10 +4,12 @@ import errno import secrets +import sys from typing import Literal from ._loader import ffi from ._loader import lib as _lib +from .random import Random from .util import Buffer, new_aligned_struct, nonce_increment, wipe NAME = "AEGIS-128X4" #: Algorithm display name @@ -19,6 +21,15 @@ ALIGNMENT = 64 #: Required alignment for internal structures RATE = 64 #: Byte chunk size in internal processing +def random() -> Random: + """Return a new AEGIS stream Random instance using this cipher module. + + Seeded once from the OS, then deterministic: successive calls never + repeat an output block. See aeg.random.Random for the API. + """ + return Random(sys.modules[__name__]) + + def random_key() -> bytearray: """ Generate a secret key using cryptographically secure random bytes. @@ -868,6 +879,220 @@ def new_mac_state(): return new_aligned_struct("aegis128x4_mac_state", ALIGNMENT) +RAF_ALG_ID = 3 #: Algorithm identifier in RAF file headers + + +def _raise_errno(operation: str) -> None: + err_num = ffi.errno + err_name = errno.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"{operation} failed: {err_name}") + + +def new_raf_ctx(): + """Allocate and return a new aegis128x4_raf_ctx* with proper alignment.""" + return new_aligned_struct("aegis128x4_raf_ctx", ALIGNMENT) + + +def raf_scratch_size(chunk_size: int) -> int: + """Return the scratch buffer size required for the given chunk_size.""" + return _lib.aegis128x4_raf_scratch_size(chunk_size) + + +def raf_create(key: Buffer, ctx, io, rng, config) -> None: + """Create a new encrypted file, writing the header through io. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileExistsError: If the file exists and AEGIS_RAF_TRUNCATE was not set. + RuntimeError: If creation fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis128x4_raf_create(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + if ffi.errno == errno.EEXIST: + raise FileExistsError("file exists (use truncate to overwrite)") + _raise_errno("raf create") + + +def raf_open(key: Buffer, ctx, io, rng, config) -> None: + """Open an existing encrypted file, verifying the header MAC. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileNotFoundError: If the file does not exist. + ValueError: If the header is invalid or authentication fails (wrong key). + RuntimeError: If opening fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis128x4_raf_open(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + err_num = ffi.errno + if err_num == errno.ENOENT: + raise FileNotFoundError("file not found") + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf open") + + +def raf_read( + ctx, length: int, offset: int, *, into: Buffer | None = None +) -> bytearray | memoryview: + """Read and decrypt up to length bytes at offset. + + Returns: + The decrypted bytes, trimmed to the number actually read + (fewer at EOF), as bytearray if into not provided, + memoryview of into otherwise. + + Raises: + TypeError: If into is smaller than length. + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + if into is None: + out = bytearray(length) + else: + into = memoryview(into) + if into.nbytes < length: + raise TypeError("into length must be at least length") + out = into + bytes_read = ffi.new("size_t*") + rc = _lib.aegis128x4_raf_read( + ctx.ptr, ffi.from_buffer(out), bytes_read, length, offset + ) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf read") + n = bytes_read[0] + return out[:n] if into is None else memoryview(out)[:n] # type: ignore + + +def raf_write(ctx, data: Buffer, offset: int) -> int: + """Encrypt and write data at offset, extending the file as needed. + + Returns: + Number of bytes written. + + Raises: + RuntimeError: If the write fails (errno name included). + """ + data = memoryview(data) + bytes_written = ffi.new("size_t*") + rc = _lib.aegis128x4_raf_write( + ctx.ptr, bytes_written, _ptr(data), data.nbytes, offset + ) + if rc != 0: + _raise_errno("raf write") + return bytes_written[0] + + +def raf_truncate(ctx, size: int) -> None: + """Resize the file to size bytes (zero-filled when growing). + + Raises: + RuntimeError: If the truncate fails. + """ + rc = _lib.aegis128x4_raf_truncate(ctx.ptr, size) + if rc != 0: + _raise_errno("raf truncate") + + +def raf_size(ctx) -> int: + """Return the logical plaintext file size.""" + size = ffi.new("uint64_t*") + rc = _lib.aegis128x4_raf_get_size(ctx.ptr, size) + if rc != 0: + _raise_errno("raf get_size") + return size[0] + + +def raf_sync(ctx) -> None: + """Flush writes to the backing store. + + Raises: + RuntimeError: If the sync fails. + """ + rc = _lib.aegis128x4_raf_sync(ctx.ptr) + if rc != 0: + _raise_errno("raf sync") + + +def raf_close(ctx) -> None: + """Close the context, syncing and zeroizing key material.""" + _lib.aegis128x4_raf_close(ctx.ptr) + + +def raf_merkle_rebuild(ctx) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Raises: + RuntimeError: If Merkle is not enabled or rebuilding fails. + ValueError: If a chunk fails authentication. + """ + rc = _lib.aegis128x4_raf_merkle_rebuild(ctx.ptr) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf merkle_rebuild") + + +def raf_merkle_verify(ctx) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + + Raises: + RuntimeError: If Merkle is not enabled or verification fails. + """ + corrupted = ffi.new("uint64_t*") + rc = _lib.aegis128x4_raf_merkle_verify(ctx.ptr, corrupted) + if rc == 0: + return None + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + return corrupted[0] + _raise_errno("raf merkle_verify") + + +def raf_merkle_commitment(ctx, hash_len: int) -> bytes: + """Compute the Merkle root commitment hash. + + Returns: + The commitment hash as bytes of length hash_len. + + Raises: + RuntimeError: If Merkle is not enabled or the call fails. + """ + out = ffi.new(f"uint8_t[{hash_len}]") + rc = _lib.aegis128x4_raf_merkle_commitment(ctx.ptr, out, hash_len) + if rc != 0: + _raise_errno("raf merkle_commitment") + return bytes(ffi.buffer(out, hash_len)) + + __all__ = [ # constants "NAME", @@ -877,7 +1102,9 @@ __all__ = [ "MACBYTES_LONG", "ALIGNMENT", "RATE", + "RAF_ALG_ID", # utility functions + "random", "random_key", "random_nonce", "nonce_increment", @@ -895,4 +1122,18 @@ __all__ = [ "Encryptor", "Decryptor", "Mac", + # random-access encrypted files (thin wrappers; see aeg.raf for the high-level API) + "new_raf_ctx", + "raf_scratch_size", + "raf_create", + "raf_open", + "raf_read", + "raf_write", + "raf_truncate", + "raf_size", + "raf_sync", + "raf_close", + "raf_merkle_rebuild", + "raf_merkle_verify", + "raf_merkle_commitment", ] diff --git a/src/aeg/aegis256.py b/src/aeg/aegis256.py index 3041a13..3b9c6f2 100644 --- a/src/aeg/aegis256.py +++ b/src/aeg/aegis256.py @@ -4,10 +4,12 @@ import errno import secrets +import sys from typing import Literal from ._loader import ffi from ._loader import lib as _lib +from .random import Random from .util import Buffer, new_aligned_struct, nonce_increment, wipe NAME = "AEGIS-256" #: Algorithm display name @@ -19,6 +21,15 @@ ALIGNMENT = 64 #: Required alignment for internal structures RATE = 64 #: Byte chunk size in internal processing +def random() -> Random: + """Return a new AEGIS stream Random instance using this cipher module. + + Seeded once from the OS, then deterministic: successive calls never + repeat an output block. See aeg.random.Random for the API. + """ + return Random(sys.modules[__name__]) + + def random_key() -> bytearray: """ Generate a secret key using cryptographically secure random bytes. @@ -868,6 +879,220 @@ def new_mac_state(): return new_aligned_struct("aegis256_mac_state", ALIGNMENT) +RAF_ALG_ID = 4 #: Algorithm identifier in RAF file headers + + +def _raise_errno(operation: str) -> None: + err_num = ffi.errno + err_name = errno.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"{operation} failed: {err_name}") + + +def new_raf_ctx(): + """Allocate and return a new aegis256_raf_ctx* with proper alignment.""" + return new_aligned_struct("aegis256_raf_ctx", ALIGNMENT) + + +def raf_scratch_size(chunk_size: int) -> int: + """Return the scratch buffer size required for the given chunk_size.""" + return _lib.aegis256_raf_scratch_size(chunk_size) + + +def raf_create(key: Buffer, ctx, io, rng, config) -> None: + """Create a new encrypted file, writing the header through io. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileExistsError: If the file exists and AEGIS_RAF_TRUNCATE was not set. + RuntimeError: If creation fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis256_raf_create(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + if ffi.errno == errno.EEXIST: + raise FileExistsError("file exists (use truncate to overwrite)") + _raise_errno("raf create") + + +def raf_open(key: Buffer, ctx, io, rng, config) -> None: + """Open an existing encrypted file, verifying the header MAC. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileNotFoundError: If the file does not exist. + ValueError: If the header is invalid or authentication fails (wrong key). + RuntimeError: If opening fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis256_raf_open(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + err_num = ffi.errno + if err_num == errno.ENOENT: + raise FileNotFoundError("file not found") + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf open") + + +def raf_read( + ctx, length: int, offset: int, *, into: Buffer | None = None +) -> bytearray | memoryview: + """Read and decrypt up to length bytes at offset. + + Returns: + The decrypted bytes, trimmed to the number actually read + (fewer at EOF), as bytearray if into not provided, + memoryview of into otherwise. + + Raises: + TypeError: If into is smaller than length. + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + if into is None: + out = bytearray(length) + else: + into = memoryview(into) + if into.nbytes < length: + raise TypeError("into length must be at least length") + out = into + bytes_read = ffi.new("size_t*") + rc = _lib.aegis256_raf_read( + ctx.ptr, ffi.from_buffer(out), bytes_read, length, offset + ) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf read") + n = bytes_read[0] + return out[:n] if into is None else memoryview(out)[:n] # type: ignore + + +def raf_write(ctx, data: Buffer, offset: int) -> int: + """Encrypt and write data at offset, extending the file as needed. + + Returns: + Number of bytes written. + + Raises: + RuntimeError: If the write fails (errno name included). + """ + data = memoryview(data) + bytes_written = ffi.new("size_t*") + rc = _lib.aegis256_raf_write( + ctx.ptr, bytes_written, _ptr(data), data.nbytes, offset + ) + if rc != 0: + _raise_errno("raf write") + return bytes_written[0] + + +def raf_truncate(ctx, size: int) -> None: + """Resize the file to size bytes (zero-filled when growing). + + Raises: + RuntimeError: If the truncate fails. + """ + rc = _lib.aegis256_raf_truncate(ctx.ptr, size) + if rc != 0: + _raise_errno("raf truncate") + + +def raf_size(ctx) -> int: + """Return the logical plaintext file size.""" + size = ffi.new("uint64_t*") + rc = _lib.aegis256_raf_get_size(ctx.ptr, size) + if rc != 0: + _raise_errno("raf get_size") + return size[0] + + +def raf_sync(ctx) -> None: + """Flush writes to the backing store. + + Raises: + RuntimeError: If the sync fails. + """ + rc = _lib.aegis256_raf_sync(ctx.ptr) + if rc != 0: + _raise_errno("raf sync") + + +def raf_close(ctx) -> None: + """Close the context, syncing and zeroizing key material.""" + _lib.aegis256_raf_close(ctx.ptr) + + +def raf_merkle_rebuild(ctx) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Raises: + RuntimeError: If Merkle is not enabled or rebuilding fails. + ValueError: If a chunk fails authentication. + """ + rc = _lib.aegis256_raf_merkle_rebuild(ctx.ptr) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf merkle_rebuild") + + +def raf_merkle_verify(ctx) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + + Raises: + RuntimeError: If Merkle is not enabled or verification fails. + """ + corrupted = ffi.new("uint64_t*") + rc = _lib.aegis256_raf_merkle_verify(ctx.ptr, corrupted) + if rc == 0: + return None + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + return corrupted[0] + _raise_errno("raf merkle_verify") + + +def raf_merkle_commitment(ctx, hash_len: int) -> bytes: + """Compute the Merkle root commitment hash. + + Returns: + The commitment hash as bytes of length hash_len. + + Raises: + RuntimeError: If Merkle is not enabled or the call fails. + """ + out = ffi.new(f"uint8_t[{hash_len}]") + rc = _lib.aegis256_raf_merkle_commitment(ctx.ptr, out, hash_len) + if rc != 0: + _raise_errno("raf merkle_commitment") + return bytes(ffi.buffer(out, hash_len)) + + __all__ = [ # constants "NAME", @@ -877,7 +1102,9 @@ __all__ = [ "MACBYTES_LONG", "ALIGNMENT", "RATE", + "RAF_ALG_ID", # utility functions + "random", "random_key", "random_nonce", "nonce_increment", @@ -895,4 +1122,18 @@ __all__ = [ "Encryptor", "Decryptor", "Mac", + # random-access encrypted files (thin wrappers; see aeg.raf for the high-level API) + "new_raf_ctx", + "raf_scratch_size", + "raf_create", + "raf_open", + "raf_read", + "raf_write", + "raf_truncate", + "raf_size", + "raf_sync", + "raf_close", + "raf_merkle_rebuild", + "raf_merkle_verify", + "raf_merkle_commitment", ] diff --git a/src/aeg/aegis256x2.py b/src/aeg/aegis256x2.py index 3304f8b..620992d 100644 --- a/src/aeg/aegis256x2.py +++ b/src/aeg/aegis256x2.py @@ -4,10 +4,12 @@ import errno import secrets +import sys from typing import Literal from ._loader import ffi from ._loader import lib as _lib +from .random import Random from .util import Buffer, new_aligned_struct, nonce_increment, wipe NAME = "AEGIS-256X2" #: Algorithm display name @@ -19,6 +21,15 @@ ALIGNMENT = 64 #: Required alignment for internal structures RATE = 64 #: Byte chunk size in internal processing +def random() -> Random: + """Return a new AEGIS stream Random instance using this cipher module. + + Seeded once from the OS, then deterministic: successive calls never + repeat an output block. See aeg.random.Random for the API. + """ + return Random(sys.modules[__name__]) + + def random_key() -> bytearray: """ Generate a secret key using cryptographically secure random bytes. @@ -868,6 +879,220 @@ def new_mac_state(): return new_aligned_struct("aegis256x2_mac_state", ALIGNMENT) +RAF_ALG_ID = 5 #: Algorithm identifier in RAF file headers + + +def _raise_errno(operation: str) -> None: + err_num = ffi.errno + err_name = errno.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"{operation} failed: {err_name}") + + +def new_raf_ctx(): + """Allocate and return a new aegis256x2_raf_ctx* with proper alignment.""" + return new_aligned_struct("aegis256x2_raf_ctx", ALIGNMENT) + + +def raf_scratch_size(chunk_size: int) -> int: + """Return the scratch buffer size required for the given chunk_size.""" + return _lib.aegis256x2_raf_scratch_size(chunk_size) + + +def raf_create(key: Buffer, ctx, io, rng, config) -> None: + """Create a new encrypted file, writing the header through io. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileExistsError: If the file exists and AEGIS_RAF_TRUNCATE was not set. + RuntimeError: If creation fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis256x2_raf_create(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + if ffi.errno == errno.EEXIST: + raise FileExistsError("file exists (use truncate to overwrite)") + _raise_errno("raf create") + + +def raf_open(key: Buffer, ctx, io, rng, config) -> None: + """Open an existing encrypted file, verifying the header MAC. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileNotFoundError: If the file does not exist. + ValueError: If the header is invalid or authentication fails (wrong key). + RuntimeError: If opening fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis256x2_raf_open(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + err_num = ffi.errno + if err_num == errno.ENOENT: + raise FileNotFoundError("file not found") + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf open") + + +def raf_read( + ctx, length: int, offset: int, *, into: Buffer | None = None +) -> bytearray | memoryview: + """Read and decrypt up to length bytes at offset. + + Returns: + The decrypted bytes, trimmed to the number actually read + (fewer at EOF), as bytearray if into not provided, + memoryview of into otherwise. + + Raises: + TypeError: If into is smaller than length. + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + if into is None: + out = bytearray(length) + else: + into = memoryview(into) + if into.nbytes < length: + raise TypeError("into length must be at least length") + out = into + bytes_read = ffi.new("size_t*") + rc = _lib.aegis256x2_raf_read( + ctx.ptr, ffi.from_buffer(out), bytes_read, length, offset + ) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf read") + n = bytes_read[0] + return out[:n] if into is None else memoryview(out)[:n] # type: ignore + + +def raf_write(ctx, data: Buffer, offset: int) -> int: + """Encrypt and write data at offset, extending the file as needed. + + Returns: + Number of bytes written. + + Raises: + RuntimeError: If the write fails (errno name included). + """ + data = memoryview(data) + bytes_written = ffi.new("size_t*") + rc = _lib.aegis256x2_raf_write( + ctx.ptr, bytes_written, _ptr(data), data.nbytes, offset + ) + if rc != 0: + _raise_errno("raf write") + return bytes_written[0] + + +def raf_truncate(ctx, size: int) -> None: + """Resize the file to size bytes (zero-filled when growing). + + Raises: + RuntimeError: If the truncate fails. + """ + rc = _lib.aegis256x2_raf_truncate(ctx.ptr, size) + if rc != 0: + _raise_errno("raf truncate") + + +def raf_size(ctx) -> int: + """Return the logical plaintext file size.""" + size = ffi.new("uint64_t*") + rc = _lib.aegis256x2_raf_get_size(ctx.ptr, size) + if rc != 0: + _raise_errno("raf get_size") + return size[0] + + +def raf_sync(ctx) -> None: + """Flush writes to the backing store. + + Raises: + RuntimeError: If the sync fails. + """ + rc = _lib.aegis256x2_raf_sync(ctx.ptr) + if rc != 0: + _raise_errno("raf sync") + + +def raf_close(ctx) -> None: + """Close the context, syncing and zeroizing key material.""" + _lib.aegis256x2_raf_close(ctx.ptr) + + +def raf_merkle_rebuild(ctx) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Raises: + RuntimeError: If Merkle is not enabled or rebuilding fails. + ValueError: If a chunk fails authentication. + """ + rc = _lib.aegis256x2_raf_merkle_rebuild(ctx.ptr) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf merkle_rebuild") + + +def raf_merkle_verify(ctx) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + + Raises: + RuntimeError: If Merkle is not enabled or verification fails. + """ + corrupted = ffi.new("uint64_t*") + rc = _lib.aegis256x2_raf_merkle_verify(ctx.ptr, corrupted) + if rc == 0: + return None + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + return corrupted[0] + _raise_errno("raf merkle_verify") + + +def raf_merkle_commitment(ctx, hash_len: int) -> bytes: + """Compute the Merkle root commitment hash. + + Returns: + The commitment hash as bytes of length hash_len. + + Raises: + RuntimeError: If Merkle is not enabled or the call fails. + """ + out = ffi.new(f"uint8_t[{hash_len}]") + rc = _lib.aegis256x2_raf_merkle_commitment(ctx.ptr, out, hash_len) + if rc != 0: + _raise_errno("raf merkle_commitment") + return bytes(ffi.buffer(out, hash_len)) + + __all__ = [ # constants "NAME", @@ -877,7 +1102,9 @@ __all__ = [ "MACBYTES_LONG", "ALIGNMENT", "RATE", + "RAF_ALG_ID", # utility functions + "random", "random_key", "random_nonce", "nonce_increment", @@ -895,4 +1122,18 @@ __all__ = [ "Encryptor", "Decryptor", "Mac", + # random-access encrypted files (thin wrappers; see aeg.raf for the high-level API) + "new_raf_ctx", + "raf_scratch_size", + "raf_create", + "raf_open", + "raf_read", + "raf_write", + "raf_truncate", + "raf_size", + "raf_sync", + "raf_close", + "raf_merkle_rebuild", + "raf_merkle_verify", + "raf_merkle_commitment", ] diff --git a/src/aeg/aegis256x4.py b/src/aeg/aegis256x4.py index 5677b58..8126272 100644 --- a/src/aeg/aegis256x4.py +++ b/src/aeg/aegis256x4.py @@ -4,10 +4,12 @@ import errno import secrets +import sys from typing import Literal from ._loader import ffi from ._loader import lib as _lib +from .random import Random from .util import Buffer, new_aligned_struct, nonce_increment, wipe NAME = "AEGIS-256X4" #: Algorithm display name @@ -19,6 +21,15 @@ ALIGNMENT = 64 #: Required alignment for internal structures RATE = 64 #: Byte chunk size in internal processing +def random() -> Random: + """Return a new AEGIS stream Random instance using this cipher module. + + Seeded once from the OS, then deterministic: successive calls never + repeat an output block. See aeg.random.Random for the API. + """ + return Random(sys.modules[__name__]) + + def random_key() -> bytearray: """ Generate a secret key using cryptographically secure random bytes. @@ -868,6 +879,220 @@ def new_mac_state(): return new_aligned_struct("aegis256x4_mac_state", ALIGNMENT) +RAF_ALG_ID = 6 #: Algorithm identifier in RAF file headers + + +def _raise_errno(operation: str) -> None: + err_num = ffi.errno + err_name = errno.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"{operation} failed: {err_name}") + + +def new_raf_ctx(): + """Allocate and return a new aegis256x4_raf_ctx* with proper alignment.""" + return new_aligned_struct("aegis256x4_raf_ctx", ALIGNMENT) + + +def raf_scratch_size(chunk_size: int) -> int: + """Return the scratch buffer size required for the given chunk_size.""" + return _lib.aegis256x4_raf_scratch_size(chunk_size) + + +def raf_create(key: Buffer, ctx, io, rng, config) -> None: + """Create a new encrypted file, writing the header through io. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileExistsError: If the file exists and AEGIS_RAF_TRUNCATE was not set. + RuntimeError: If creation fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis256x4_raf_create(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + if ffi.errno == errno.EEXIST: + raise FileExistsError("file exists (use truncate to overwrite)") + _raise_errno("raf create") + + +def raf_open(key: Buffer, ctx, io, rng, config) -> None: + """Open an existing encrypted file, verifying the header MAC. + + Args: + key: Master key (KEYBYTES bytes). + ctx: Context from new_raf_ctx(). + io: aegis_raf_io* cdata (see aeg.raf). + rng: aegis_raf_rng* cdata (see aeg.random). + config: aegis_raf_config* cdata (see aeg.raf). + + Raises: + TypeError: If key length is invalid. + FileNotFoundError: If the file does not exist. + ValueError: If the header is invalid or authentication fails (wrong key). + RuntimeError: If opening fails. + """ + key = memoryview(key) + if key.nbytes != KEYBYTES: + raise TypeError(f"key length must be {KEYBYTES}") + rc = _lib.aegis256x4_raf_open(ctx.ptr, io, rng, config, _ptr(key)) + if rc != 0: + err_num = ffi.errno + if err_num == errno.ENOENT: + raise FileNotFoundError("file not found") + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf open") + + +def raf_read( + ctx, length: int, offset: int, *, into: Buffer | None = None +) -> bytearray | memoryview: + """Read and decrypt up to length bytes at offset. + + Returns: + The decrypted bytes, trimmed to the number actually read + (fewer at EOF), as bytearray if into not provided, + memoryview of into otherwise. + + Raises: + TypeError: If into is smaller than length. + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + if into is None: + out = bytearray(length) + else: + into = memoryview(into) + if into.nbytes < length: + raise TypeError("into length must be at least length") + out = into + bytes_read = ffi.new("size_t*") + rc = _lib.aegis256x4_raf_read( + ctx.ptr, ffi.from_buffer(out), bytes_read, length, offset + ) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf read") + n = bytes_read[0] + return out[:n] if into is None else memoryview(out)[:n] # type: ignore + + +def raf_write(ctx, data: Buffer, offset: int) -> int: + """Encrypt and write data at offset, extending the file as needed. + + Returns: + Number of bytes written. + + Raises: + RuntimeError: If the write fails (errno name included). + """ + data = memoryview(data) + bytes_written = ffi.new("size_t*") + rc = _lib.aegis256x4_raf_write( + ctx.ptr, bytes_written, _ptr(data), data.nbytes, offset + ) + if rc != 0: + _raise_errno("raf write") + return bytes_written[0] + + +def raf_truncate(ctx, size: int) -> None: + """Resize the file to size bytes (zero-filled when growing). + + Raises: + RuntimeError: If the truncate fails. + """ + rc = _lib.aegis256x4_raf_truncate(ctx.ptr, size) + if rc != 0: + _raise_errno("raf truncate") + + +def raf_size(ctx) -> int: + """Return the logical plaintext file size.""" + size = ffi.new("uint64_t*") + rc = _lib.aegis256x4_raf_get_size(ctx.ptr, size) + if rc != 0: + _raise_errno("raf get_size") + return size[0] + + +def raf_sync(ctx) -> None: + """Flush writes to the backing store. + + Raises: + RuntimeError: If the sync fails. + """ + rc = _lib.aegis256x4_raf_sync(ctx.ptr) + if rc != 0: + _raise_errno("raf sync") + + +def raf_close(ctx) -> None: + """Close the context, syncing and zeroizing key material.""" + _lib.aegis256x4_raf_close(ctx.ptr) + + +def raf_merkle_rebuild(ctx) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Raises: + RuntimeError: If Merkle is not enabled or rebuilding fails. + ValueError: If a chunk fails authentication. + """ + rc = _lib.aegis256x4_raf_merkle_rebuild(ctx.ptr) + if rc != 0: + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + raise ValueError("authentication failed") + _raise_errno("raf merkle_rebuild") + + +def raf_merkle_verify(ctx) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + + Raises: + RuntimeError: If Merkle is not enabled or verification fails. + """ + corrupted = ffi.new("uint64_t*") + rc = _lib.aegis256x4_raf_merkle_verify(ctx.ptr, corrupted) + if rc == 0: + return None + err_num = ffi.errno + if err_num in (errno.EINVAL, getattr(errno, "EBADMSG", -1)): + return corrupted[0] + _raise_errno("raf merkle_verify") + + +def raf_merkle_commitment(ctx, hash_len: int) -> bytes: + """Compute the Merkle root commitment hash. + + Returns: + The commitment hash as bytes of length hash_len. + + Raises: + RuntimeError: If Merkle is not enabled or the call fails. + """ + out = ffi.new(f"uint8_t[{hash_len}]") + rc = _lib.aegis256x4_raf_merkle_commitment(ctx.ptr, out, hash_len) + if rc != 0: + _raise_errno("raf merkle_commitment") + return bytes(ffi.buffer(out, hash_len)) + + __all__ = [ # constants "NAME", @@ -877,7 +1102,9 @@ __all__ = [ "MACBYTES_LONG", "ALIGNMENT", "RATE", + "RAF_ALG_ID", # utility functions + "random", "random_key", "random_nonce", "nonce_increment", @@ -895,4 +1122,18 @@ __all__ = [ "Encryptor", "Decryptor", "Mac", + # random-access encrypted files (thin wrappers; see aeg.raf for the high-level API) + "new_raf_ctx", + "raf_scratch_size", + "raf_create", + "raf_open", + "raf_read", + "raf_write", + "raf_truncate", + "raf_size", + "raf_sync", + "raf_close", + "raf_merkle_rebuild", + "raf_merkle_verify", + "raf_merkle_commitment", ] diff --git a/src/aeg/aegis_cdef.h b/src/aeg/aegis_cdef.h index ca4017b..6c4a394 100644 --- a/src/aeg/aegis_cdef.h +++ b/src/aeg/aegis_cdef.h @@ -475,3 +475,232 @@ int aegis256x4_mac_final(aegis256x4_mac_state *st_, uint8_t *mac, size_t maclen) int aegis256x4_mac_verify(aegis256x4_mac_state *st_, const uint8_t *mac, size_t maclen); void aegis256x4_mac_reset(aegis256x4_mac_state *st_); void aegis256x4_mac_state_clone(aegis256x4_mac_state *dst, const aegis256x4_mac_state *src); + +/* aegis_raf.h */ +typedef struct aegis_raf_scratch { uint8_t *buf; size_t len; } aegis_raf_scratch; +typedef struct aegis_raf_io { void *user; int (*read_at)(void *user, + uint8_t *buf, + size_t len, + uint64_t off); int (*write_at)(void *user, + const uint8_t *buf, + size_t len, + uint64_t off); int (*get_size)(void *user, + uint64_t *size); int (*set_size)(void *user, + uint64_t size); int (*sync)(void *user); } aegis_raf_io; +typedef struct aegis_raf_rng { void *user; int (*random)(void *user, uint8_t *out, size_t len); } aegis_raf_rng; +typedef struct aegis_raf_merkle_config { int (*hash_leaf)(void *user, + uint8_t *out, + size_t out_len, + const uint8_t *chunk, + size_t chunk_len, + uint64_t chunk_idx); int (*hash_parent)(void *user, + uint8_t *out, + size_t out_len, + const uint8_t *left, + const uint8_t *right, + uint32_t level, + uint64_t node_idx); int (*hash_empty)(void *user, + uint8_t *out, + size_t out_len, + uint32_t level, + uint64_t node_idx); int (*hash_commitment)(void *user, + uint8_t *out, + size_t out_len, + const uint8_t *structural_root, + const uint8_t *ctx, + size_t ctx_len, + uint64_t file_size); void *user; uint8_t *buf; size_t len; uint64_t max_chunks; uint32_t hash_len; } aegis_raf_merkle_config; +typedef struct aegis_raf_config { const aegis_raf_scratch *scratch; const aegis_raf_merkle_config *merkle; uint32_t chunk_size; uint8_t flags; } aegis_raf_config; +typedef struct aegis_raf_info { uint64_t file_size; uint32_t chunk_size; uint8_t alg_id; } aegis_raf_info; +typedef struct aegis128l_raf_ctx { uint8_t opaque[512]; } aegis128l_raf_ctx; +typedef struct aegis128x2_raf_ctx { uint8_t opaque[512]; } aegis128x2_raf_ctx; +typedef struct aegis128x4_raf_ctx { uint8_t opaque[512]; } aegis128x4_raf_ctx; +typedef struct aegis256_raf_ctx { uint8_t opaque[512]; } aegis256_raf_ctx; +typedef struct aegis256x2_raf_ctx { uint8_t opaque[512]; } aegis256x2_raf_ctx; +typedef struct aegis256x4_raf_ctx { uint8_t opaque[512]; } aegis256x4_raf_ctx; +size_t aegis_raf_chunk_min(void); +size_t aegis_raf_chunk_max(void); +size_t aegis_raf_header_size(void); +size_t aegis_raf_scratch_align(void); +int aegis_raf_probe(const aegis_raf_io *io, aegis_raf_info *info); +size_t aegis128l_raf_scratch_size(uint32_t chunk_size); +int aegis128l_raf_scratch_validate(const aegis_raf_scratch *scratch, uint32_t chunk_size); +size_t aegis128x2_raf_scratch_size(uint32_t chunk_size); +int aegis128x2_raf_scratch_validate(const aegis_raf_scratch *scratch, uint32_t chunk_size); +size_t aegis128x4_raf_scratch_size(uint32_t chunk_size); +int aegis128x4_raf_scratch_validate(const aegis_raf_scratch *scratch, uint32_t chunk_size); +size_t aegis256_raf_scratch_size(uint32_t chunk_size); +int aegis256_raf_scratch_validate(const aegis_raf_scratch *scratch, uint32_t chunk_size); +size_t aegis256x2_raf_scratch_size(uint32_t chunk_size); +int aegis256x2_raf_scratch_validate(const aegis_raf_scratch *scratch, uint32_t chunk_size); +size_t aegis256x4_raf_scratch_size(uint32_t chunk_size); +int aegis256x4_raf_scratch_validate(const aegis_raf_scratch *scratch, uint32_t chunk_size); +size_t aegis_raf_merkle_buffer_size(const aegis_raf_merkle_config *cfg); +int aegis_raf_derive_master_key(uint8_t *out, + size_t out_len, + const uint8_t *master_key, + size_t master_key_len, + const uint8_t *context, + size_t context_len); +int aegis128l_raf_create(aegis128l_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis128l_raf_open(aegis128l_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis128l_raf_read(aegis128l_raf_ctx *ctx, + uint8_t *out, + size_t *bytes_read, + size_t len, + uint64_t offset); +int aegis128l_raf_write(aegis128l_raf_ctx *ctx, + size_t *bytes_written, + const uint8_t *in, + size_t len, + uint64_t offset); +int aegis128l_raf_truncate(aegis128l_raf_ctx *ctx, uint64_t size); +int aegis128l_raf_get_size(const aegis128l_raf_ctx *ctx, uint64_t *size); +int aegis128l_raf_sync(aegis128l_raf_ctx *ctx); +void aegis128l_raf_close(aegis128l_raf_ctx *ctx); +int aegis128l_raf_merkle_rebuild(aegis128l_raf_ctx *ctx); +int aegis128l_raf_merkle_verify(aegis128l_raf_ctx *ctx, uint64_t *corrupted_chunk); +int aegis128l_raf_merkle_commitment(const aegis128l_raf_ctx *ctx, uint8_t *out, size_t out_len); +int aegis128x2_raf_create(aegis128x2_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis128x2_raf_open(aegis128x2_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis128x2_raf_read(aegis128x2_raf_ctx *ctx, + uint8_t *out, + size_t *bytes_read, + size_t len, + uint64_t offset); +int aegis128x2_raf_write(aegis128x2_raf_ctx *ctx, + size_t *bytes_written, + const uint8_t *in, + size_t len, + uint64_t offset); +int aegis128x2_raf_truncate(aegis128x2_raf_ctx *ctx, uint64_t size); +int aegis128x2_raf_get_size(const aegis128x2_raf_ctx *ctx, uint64_t *size); +int aegis128x2_raf_sync(aegis128x2_raf_ctx *ctx); +void aegis128x2_raf_close(aegis128x2_raf_ctx *ctx); +int aegis128x2_raf_merkle_rebuild(aegis128x2_raf_ctx *ctx); +int aegis128x2_raf_merkle_verify(aegis128x2_raf_ctx *ctx, uint64_t *corrupted_chunk); +int aegis128x2_raf_merkle_commitment(const aegis128x2_raf_ctx *ctx, uint8_t *out, size_t out_len); +int aegis128x4_raf_create(aegis128x4_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis128x4_raf_open(aegis128x4_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis128x4_raf_read(aegis128x4_raf_ctx *ctx, + uint8_t *out, + size_t *bytes_read, + size_t len, + uint64_t offset); +int aegis128x4_raf_write(aegis128x4_raf_ctx *ctx, + size_t *bytes_written, + const uint8_t *in, + size_t len, + uint64_t offset); +int aegis128x4_raf_truncate(aegis128x4_raf_ctx *ctx, uint64_t size); +int aegis128x4_raf_get_size(const aegis128x4_raf_ctx *ctx, uint64_t *size); +int aegis128x4_raf_sync(aegis128x4_raf_ctx *ctx); +void aegis128x4_raf_close(aegis128x4_raf_ctx *ctx); +int aegis128x4_raf_merkle_rebuild(aegis128x4_raf_ctx *ctx); +int aegis128x4_raf_merkle_verify(aegis128x4_raf_ctx *ctx, uint64_t *corrupted_chunk); +int aegis128x4_raf_merkle_commitment(const aegis128x4_raf_ctx *ctx, uint8_t *out, size_t out_len); +int aegis256_raf_create(aegis256_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis256_raf_open(aegis256_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis256_raf_read(aegis256_raf_ctx *ctx, + uint8_t *out, + size_t *bytes_read, + size_t len, + uint64_t offset); +int aegis256_raf_write(aegis256_raf_ctx *ctx, + size_t *bytes_written, + const uint8_t *in, + size_t len, + uint64_t offset); +int aegis256_raf_truncate(aegis256_raf_ctx *ctx, uint64_t size); +int aegis256_raf_get_size(const aegis256_raf_ctx *ctx, uint64_t *size); +int aegis256_raf_sync(aegis256_raf_ctx *ctx); +void aegis256_raf_close(aegis256_raf_ctx *ctx); +int aegis256_raf_merkle_rebuild(aegis256_raf_ctx *ctx); +int aegis256_raf_merkle_verify(aegis256_raf_ctx *ctx, uint64_t *corrupted_chunk); +int aegis256_raf_merkle_commitment(const aegis256_raf_ctx *ctx, uint8_t *out, size_t out_len); +int aegis256x2_raf_create(aegis256x2_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis256x2_raf_open(aegis256x2_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis256x2_raf_read(aegis256x2_raf_ctx *ctx, + uint8_t *out, + size_t *bytes_read, + size_t len, + uint64_t offset); +int aegis256x2_raf_write(aegis256x2_raf_ctx *ctx, + size_t *bytes_written, + const uint8_t *in, + size_t len, + uint64_t offset); +int aegis256x2_raf_truncate(aegis256x2_raf_ctx *ctx, uint64_t size); +int aegis256x2_raf_get_size(const aegis256x2_raf_ctx *ctx, uint64_t *size); +int aegis256x2_raf_sync(aegis256x2_raf_ctx *ctx); +void aegis256x2_raf_close(aegis256x2_raf_ctx *ctx); +int aegis256x2_raf_merkle_rebuild(aegis256x2_raf_ctx *ctx); +int aegis256x2_raf_merkle_verify(aegis256x2_raf_ctx *ctx, uint64_t *corrupted_chunk); +int aegis256x2_raf_merkle_commitment(const aegis256x2_raf_ctx *ctx, uint8_t *out, size_t out_len); +int aegis256x4_raf_create(aegis256x4_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis256x4_raf_open(aegis256x4_raf_ctx *ctx, + const aegis_raf_io *io, + const aegis_raf_rng *rng, + const aegis_raf_config *cfg, + const uint8_t *master_key); +int aegis256x4_raf_read(aegis256x4_raf_ctx *ctx, + uint8_t *out, + size_t *bytes_read, + size_t len, + uint64_t offset); +int aegis256x4_raf_write(aegis256x4_raf_ctx *ctx, + size_t *bytes_written, + const uint8_t *in, + size_t len, + uint64_t offset); +int aegis256x4_raf_truncate(aegis256x4_raf_ctx *ctx, uint64_t size); +int aegis256x4_raf_get_size(const aegis256x4_raf_ctx *ctx, uint64_t *size); +int aegis256x4_raf_sync(aegis256x4_raf_ctx *ctx); +void aegis256x4_raf_close(aegis256x4_raf_ctx *ctx); +int aegis256x4_raf_merkle_rebuild(aegis256x4_raf_ctx *ctx); +int aegis256x4_raf_merkle_verify(aegis256x4_raf_ctx *ctx, uint64_t *corrupted_chunk); +int aegis256x4_raf_merkle_commitment(const aegis256x4_raf_ctx *ctx, uint8_t *out, size_t out_len); diff --git a/src/aeg/raf.py b/src/aeg/raf.py new file mode 100644 index 0000000..b5bcbf8 --- /dev/null +++ b/src/aeg/raf.py @@ -0,0 +1,917 @@ +"""RAF (random-access encrypted file) high-level API. + +Provides pread/pwrite-style access to encrypted files. Files are divided into +fixed-size chunks, each independently encrypted with a fresh nonce, enabling +efficient random access without decrypting the whole file. + +Use create() / open() with a cipher module (from aeg.cipher()) and either a +path or a Storage instance. + +Portions of the storage/callback/Merkle/file-like machinery are adapted from +pyaegis (https://github.com/jedisct1/pyaegis) by Frank Denis, MIT license. +""" + +from __future__ import annotations + +import errno as errno_module +import hashlib +import os +import threading +from typing import NamedTuple, Protocol, runtime_checkable + +from . import cipher as _cipher_for_name +from . import random as _random +from ._loader import ffi, lib +from ._typing import Cipher +from .util import Buffer + +__all__ = [ + "CHUNK_MIN", + "CHUNK_MAX", + "HEADER_SIZE", + "SCRATCH_ALIGN", + "Storage", + "FileStorage", + "BytesIOStorage", + "MerkleHasher", + "SHA256MerkleHasher", + "RafInfo", + "Raf", + "create", + "open", + "probe", + "derive_master_key", +] + +CHUNK_MIN = int(lib.aegis_raf_chunk_min()) #: Minimum chunk size in bytes +CHUNK_MAX = int(lib.aegis_raf_chunk_max()) #: Maximum chunk size in bytes +HEADER_SIZE = int(lib.aegis_raf_header_size()) #: Encrypted file header size +SCRATCH_ALIGN = int(lib.aegis_raf_scratch_align()) #: Required scratch buffer alignment + +# From aegis_raf.h (preprocessor macros, not available through cdef) +_FLAG_CREATE = 0x01 +_FLAG_TRUNCATE = 0x02 +_MERKLE_HASH_MIN = 8 +_MERKLE_HASH_MAX = 64 + +@runtime_checkable +class Storage(Protocol): + """Backing storage for RAF files. Operations must complete fully or raise OSError.""" + + def read_at(self, buf: bytearray, offset: int) -> None: + """Read exactly len(buf) bytes at offset into buf.""" + ... + + def write_at(self, data: bytes, offset: int) -> None: + """Write exactly len(data) bytes at offset.""" + ... + + def get_size(self) -> int: + """Return current storage size in bytes.""" + ... + + def set_size(self, size: int) -> None: + """Resize storage (truncate or extend).""" + ... + + def sync(self) -> None: + """Flush writes to durable storage. May be a no-op.""" + ... + + +class FileStorage: + """File-based storage. + + Uses os.pread/os.pwrite where available (Unix); otherwise falls back to + a lock plus os.lseek/os.read/os.write (e.g. on Windows). + """ + + def __init__(self, path: str | os.PathLike, mode: str = "r+b"): + """Open a file for RAF storage. + + Args: + path: Path to the file. + mode: "r+b" to open existing, "w+b" or "x+b" to create. + """ + if mode == "r+b": + flags = os.O_RDWR + elif mode == "w+b": + flags = os.O_RDWR | os.O_CREAT | os.O_TRUNC + elif mode == "x+b": + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL + else: + raise ValueError(f"Unsupported mode: {mode}") + self._fd = os.open(path, flags, 0o644) + self._closed = False + self._lock = threading.Lock() + self._positional = hasattr(os, "pread") and hasattr(os, "pwrite") + + def read_at(self, buf: bytearray, offset: int) -> None: + """Read exactly len(buf) bytes. Raises OSError on short read.""" + view = memoryview(buf) + if self._positional: + done = 0 + while done < len(view): + chunk = os.pread(self._fd, len(view) - done, offset + done) + if not chunk: + raise OSError(f"Short read: expected {len(view)}, got {done}") + view[done : done + len(chunk)] = chunk + done += len(chunk) + else: + with self._lock: + os.lseek(self._fd, offset, os.SEEK_SET) + done = 0 + while done < len(view): + chunk = os.read(self._fd, len(view) - done) + if not chunk: + raise OSError(f"Short read: expected {len(view)}, got {done}") + view[done : done + len(chunk)] = chunk + done += len(chunk) + + def write_at(self, data: bytes, offset: int) -> None: + """Write exactly len(data) bytes.""" + view = memoryview(data) + if self._positional: + done = 0 + while done < len(view): + done += os.pwrite(self._fd, view[done:], offset + done) + else: + with self._lock: + os.lseek(self._fd, offset, os.SEEK_SET) + done = 0 + while done < len(view): + done += os.write(self._fd, view[done:]) + + def get_size(self) -> int: + """Return current file size.""" + return os.fstat(self._fd).st_size + + def set_size(self, size: int) -> None: + """Truncate or extend the file to the given size.""" + os.ftruncate(self._fd, size) + + def sync(self) -> None: + """Flush file data to disk.""" + os.fsync(self._fd) + + def close(self) -> None: + """Close the underlying file descriptor.""" + if not self._closed: + os.close(self._fd) + self._closed = True + + def __enter__(self) -> FileStorage: + return self + + def __exit__(self, *args) -> None: + self.close() + + +class BytesIOStorage: + """In-memory storage backed by a bytearray. Useful for tests.""" + + def __init__(self, initial: bytes = b""): + self._data = bytearray(initial) + + def read_at(self, buf: bytearray, offset: int) -> None: + """Read exactly len(buf) bytes at offset into buf.""" + end = offset + len(buf) + if end > len(self._data): + raise OSError( + f"Read past EOF: offset={offset}, len={len(buf)}, size={len(self._data)}" + ) + buf[:] = self._data[offset:end] + + def write_at(self, data: bytes, offset: int) -> None: + """Write exactly len(data) bytes at offset.""" + if not data: + return + end = offset + len(data) + if end > len(self._data): + self._data.extend(b"\x00" * (end - len(self._data))) + self._data[offset:end] = data + + def get_size(self) -> int: + """Return current buffer size.""" + return len(self._data) + + def set_size(self, size: int) -> None: + """Resize the buffer (truncate or extend with zeros).""" + current = len(self._data) + if size < current: + del self._data[size:] + elif size > current: + self._data.extend(b"\x00" * (size - current)) + + def sync(self) -> None: + """No-op for in-memory storage.""" + + def getvalue(self) -> bytes: + """Return the entire buffer contents.""" + return bytes(self._data) + + def __enter__(self) -> BytesIOStorage: + return self + + def __exit__(self, *args) -> None: + pass + + +@runtime_checkable +class MerkleHasher(Protocol): + """Protocol for Merkle tree hash functions.""" + + @property + def hash_len(self) -> int: ... + + def hash_leaf(self, chunk: bytes, chunk_len: int, chunk_idx: int) -> bytes: ... + def hash_parent( + self, left: bytes, right: bytes, level: int, node_idx: int + ) -> bytes: ... + def hash_empty(self, level: int, node_idx: int) -> bytes: ... + def hash_commitment( + self, structural_root: bytes, ctx: bytes, file_size: int + ) -> bytes: ... + + +class SHA256MerkleHasher: + """Default Merkle hasher using SHA-256 with domain-separated prefixes.""" + + hash_len = 32 + + def hash_leaf(self, chunk: bytes, chunk_len: int, chunk_idx: int) -> bytes: + h = hashlib.sha256() + h.update(b"\x00") + h.update(chunk_idx.to_bytes(8, "little")) + h.update(chunk[:chunk_len]) + return h.digest() + + def hash_parent( + self, left: bytes, right: bytes, level: int, node_idx: int + ) -> bytes: + h = hashlib.sha256() + h.update(b"\x01") + h.update(level.to_bytes(4, "little")) + h.update(node_idx.to_bytes(8, "little")) + h.update(left) + h.update(right) + return h.digest() + + def hash_empty(self, level: int, node_idx: int) -> bytes: + h = hashlib.sha256() + h.update(b"\x02") + h.update(level.to_bytes(4, "little")) + h.update(node_idx.to_bytes(8, "little")) + return h.digest() + + def hash_commitment( + self, structural_root: bytes, ctx: bytes, file_size: int + ) -> bytes: + h = hashlib.sha256() + h.update(b"\x03") + h.update(structural_root) + h.update(ctx) + h.update(file_size.to_bytes(8, "little")) + return h.digest() + + +# CFFI callbacks (module-level so they stay alive) + + +@ffi.callback("int(void*, uint8_t*, size_t, uint64_t)") +def _read_at_callback(user, buf, length, offset): + try: + storage = ffi.from_handle(user) + temp = bytearray(length) + storage.read_at(temp, offset) + ffi.buffer(buf, length)[:] = temp + return 0 + except Exception: + ffi.errno = errno_module.EIO + return -1 + + +@ffi.callback("int(void*, const uint8_t*, size_t, uint64_t)") +def _write_at_callback(user, buf, length, offset): + try: + storage = ffi.from_handle(user) + storage.write_at(bytes(ffi.buffer(buf, length)), offset) + return 0 + except Exception: + ffi.errno = errno_module.EIO + return -1 + + +@ffi.callback("int(void*, uint64_t*)") +def _get_size_callback(user, size_ptr): + try: + size_ptr[0] = ffi.from_handle(user).get_size() + return 0 + except Exception: + ffi.errno = errno_module.EIO + return -1 + + +@ffi.callback("int(void*, uint64_t)") +def _set_size_callback(user, size): + try: + ffi.from_handle(user).set_size(size) + return 0 + except Exception: + ffi.errno = errno_module.EIO + return -1 + + +@ffi.callback("int(void*)") +def _sync_callback(user): + try: + ffi.from_handle(user).sync() + return 0 + except Exception: + ffi.errno = errno_module.EIO + return -1 + + +@ffi.callback("int(void*, uint8_t*, size_t, const uint8_t*, size_t, uint64_t)") +def _hash_leaf_callback(user, out, out_len, chunk, chunk_len, chunk_idx): + try: + hasher = ffi.from_handle(user) + digest = hasher.hash_leaf( + bytes(ffi.buffer(chunk, chunk_len)), chunk_len, chunk_idx + ) + if len(digest) != out_len: + ffi.errno = errno_module.EINVAL + return -1 + ffi.memmove(out, digest, out_len) + return 0 + except Exception: + ffi.errno = errno_module.EINVAL + return -1 + + +@ffi.callback( + "int(void*, uint8_t*, size_t, const uint8_t*, const uint8_t*, uint32_t, uint64_t)" +) +def _hash_parent_callback(user, out, out_len, left, right, level, node_idx): + try: + hasher = ffi.from_handle(user) + digest = hasher.hash_parent( + bytes(ffi.buffer(left, out_len)), + bytes(ffi.buffer(right, out_len)), + level, + node_idx, + ) + if len(digest) != out_len: + ffi.errno = errno_module.EINVAL + return -1 + ffi.memmove(out, digest, out_len) + return 0 + except Exception: + ffi.errno = errno_module.EINVAL + return -1 + + +@ffi.callback("int(void*, uint8_t*, size_t, uint32_t, uint64_t)") +def _hash_empty_callback(user, out, out_len, level, node_idx): + try: + hasher = ffi.from_handle(user) + digest = hasher.hash_empty(level, node_idx) + if len(digest) != out_len: + ffi.errno = errno_module.EINVAL + return -1 + ffi.memmove(out, digest, out_len) + return 0 + except Exception: + ffi.errno = errno_module.EINVAL + return -1 + + +@ffi.callback( + "int(void*, uint8_t*, size_t, const uint8_t*, const uint8_t*, size_t, uint64_t)" +) +def _hash_commitment_callback( + user, out, out_len, structural_root, ctx, ctx_len, file_size +): + try: + hasher = ffi.from_handle(user) + root_bytes = bytes(ffi.buffer(structural_root, out_len)) + ctx_bytes = bytes(ffi.buffer(ctx, ctx_len)) if ctx != ffi.NULL else b"" + digest = hasher.hash_commitment(root_bytes, ctx_bytes, file_size) + if len(digest) != out_len: + ffi.errno = errno_module.EINVAL + return -1 + ffi.memmove(out, digest, out_len) + return 0 + except Exception: + ffi.errno = errno_module.EINVAL + return -1 + + +def _make_io(storage: Storage): + """Build an aegis_raf_io* cdata for a Storage. Returns (io, handle).""" + handle = ffi.new_handle(storage) + io = ffi.new("aegis_raf_io*") + io.user = handle + io.read_at = _read_at_callback + io.write_at = _write_at_callback + io.get_size = _get_size_callback + io.set_size = _set_size_callback + io.sync = _sync_callback + return io, handle + + +def _allocate_scratch(cipher: Cipher, chunk_size: int): + """Allocate an aligned aegis_raf_scratch*. Returns (raw, scratch).""" + size = cipher.raf_scratch_size(chunk_size) + raw = bytearray(size + SCRATCH_ALIGN) + raw_ptr = ffi.from_buffer(raw) + offset = (-int(ffi.cast("uintptr_t", raw_ptr))) & (SCRATCH_ALIGN - 1) + scratch = ffi.new("aegis_raf_scratch*") + scratch.buf = ffi.cast("uint8_t*", raw_ptr) + offset + scratch.len = size + return raw, scratch + + +def derive_master_key(master_key: Buffer, context: Buffer = b"") -> bytes: + """Derive a context-bound RAF master key from an application master key. + + Args: + master_key: A 16-byte or 32-byte application master key. + context: A public identifier for a file or file family + (at most 120 bytes for 16-byte keys, 72 for 32-byte keys). + + Returns: + A RAF-scoped key of the same length as master_key. + + Raises: + ValueError: If the key or context length is invalid. + RuntimeError: If the C KDF rejects validated inputs. + """ + master_key = memoryview(master_key) + context = memoryview(context) + key_size = master_key.nbytes + if key_size not in (16, 32): + raise ValueError(f"master_key must be 16 or 32 bytes, got {key_size}") + max_context = 120 if key_size == 16 else 72 + if context.nbytes > max_context: + raise ValueError( + f"context must be at most {max_context} bytes for a {key_size}-byte key" + ) + out = bytearray(key_size) + rc = lib.aegis_raf_derive_master_key( + ffi.from_buffer(out), + key_size, + bytes(master_key), + key_size, + bytes(context), + context.nbytes, + ) + if rc != 0: + err_num = ffi.errno + err_name = errno_module.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"derive master key failed: {err_name}") + return bytes(out) + + +class RafInfo(NamedTuple): + """Unverified file metadata from probe().""" + + alg_id: int + chunk_size: int + file_size: int + + +def probe(storage: Storage) -> RafInfo: + """Probe an encrypted file to determine its parameters. + + WARNING: the returned values are read from the header WITHOUT + cryptographic verification. Only use them for scratch sizing and + algorithm selection; open() verifies the header MAC with the key. + + Returns: + RafInfo(alg_id, chunk_size, file_size). + + Raises: + RuntimeError: Invalid header or I/O failure. + """ + io, _handle = _make_io(storage) + info = ffi.new("aegis_raf_info*") + if lib.aegis_raf_probe(io, info) != 0: + err_num = ffi.errno + err_name = errno_module.errorcode.get(err_num, f"errno_{err_num}") + raise RuntimeError(f"probe failed: {err_name}") + return RafInfo(int(info.alg_id), int(info.chunk_size), int(info.file_size)) + + +def _as_storage(storage_or_path, mode: str) -> tuple[Storage, bool]: + if isinstance(storage_or_path, (str, bytes, os.PathLike)): + return FileStorage(storage_or_path, mode), True + return storage_or_path, False + + +def _resolve_cipher(cipher) -> Cipher: + if isinstance(cipher, str): + return _cipher_for_name(cipher) + return cipher + + +def create( + storage_or_path, + key: Buffer, + cipher, + *, + chunk_size: int = 65536, + truncate: bool = False, + merkle: bool | MerkleHasher = False, + merkle_max_chunks: int = 16384, + rng: "_random.RandomSource | None" = None, +) -> "Raf": + """Create a new encrypted file. + + Args: + storage_or_path: A Storage instance or a path (opened "x+b", or "w+b" + when truncate=True). + key: Master key (cipher.KEYBYTES bytes). + cipher: A cipher module (from aeg.cipher()) or algorithm name. + chunk_size: Plaintext bytes per chunk (CHUNK_MIN..CHUNK_MAX, multiple of 16). + truncate: Overwrite an existing file. + merkle: Enable Merkle tree. True uses SHA256MerkleHasher, or pass a + custom MerkleHasher instance. + merkle_max_chunks: Maximum number of chunks the Merkle tree can track. + rng: Random source (see aeg.random). Default: a fresh aeg.random.Random + seeded for the same cipher module passed as cipher=; any callable(n) + returning bytes overrides. + + Raises: + TypeError: If the key length is invalid. + ValueError: If chunk_size or merkle parameters are invalid. + FileExistsError: If the file exists and truncate is False. + RuntimeError: If creation fails. + """ + cipher = _resolve_cipher(cipher) + if not (CHUNK_MIN <= chunk_size <= CHUNK_MAX): + raise ValueError( + f"chunk_size must be {CHUNK_MIN}-{CHUNK_MAX}, got {chunk_size}" + ) + if chunk_size % 16 != 0: + raise ValueError(f"chunk_size must be a multiple of 16, got {chunk_size}") + storage, owns = _as_storage(storage_or_path, "w+b" if truncate else "x+b") + return Raf( + storage, + key, + cipher, + owns_storage=owns, + creating=True, + chunk_size=chunk_size, + truncate=truncate, + merkle=merkle, + merkle_max_chunks=merkle_max_chunks, + rng=rng, + ) + + +def open( + storage_or_path, + key: Buffer, + cipher, + *, + merkle: bool | MerkleHasher = False, + merkle_max_chunks: int = 16384, + rng: "_random.RandomSource | None" = None, +) -> "Raf": + """Open an existing encrypted file. + + The header is probed to size internal buffers and its alg_id is checked + against cipher.RAF_ALG_ID; a mismatch raises ValueError (the algorithm is + never switched silently). + + Args: + storage_or_path: A Storage instance or a path (opened "r+b"). + key: Master key (cipher.KEYBYTES bytes). + cipher: A cipher module (from aeg.cipher()) or algorithm name. + merkle: Enable Merkle tree (call merkle_rebuild() before use). + merkle_max_chunks: Maximum number of chunks the Merkle tree can track. + rng: Random source (see aeg.random). Default: a fresh aeg.random.Random + seeded for the same cipher module passed as cipher=; any callable(n) + returning bytes overrides. + + Raises: + TypeError: If the key length is invalid. + ValueError: If the algorithm doesn't match cipher or authentication fails. + FileNotFoundError: If the file does not exist. + RuntimeError: If opening fails. + """ + cipher = _resolve_cipher(cipher) + storage, owns = _as_storage(storage_or_path, "r+b") + info = probe(storage) + if info.alg_id != cipher.RAF_ALG_ID: + raise ValueError( + f"file uses algorithm id {info.alg_id}, " + f"expected {cipher.RAF_ALG_ID} ({cipher.NAME})" + ) + return Raf( + storage, + key, + cipher, + owns_storage=owns, + creating=False, + chunk_size=info.chunk_size, + truncate=False, + merkle=merkle, + merkle_max_chunks=merkle_max_chunks, + rng=rng, + ) + + +class Raf: + """File-like random-access encrypted file. Use raf.create() / raf.open().""" + + def __init__( + self, + storage: Storage, + key: Buffer, + cipher: Cipher, + *, + owns_storage: bool, + creating: bool, + chunk_size: int, + truncate: bool, + merkle: bool | MerkleHasher, + merkle_max_chunks: int, + rng, + ): + self._cipher = cipher + self._storage = storage + self._owns_storage = owns_storage + self._closed = False + self._position = 0 + + self._ctx = cipher.new_raf_ctx() + self._io, self._storage_handle = _make_io(storage) + if rng is None: + rng = _random.Random(cipher) + self._rng, self._rng_keepalive = _random.as_raf_rng(rng) + self._raw_scratch, self._scratch = _allocate_scratch(cipher, chunk_size) + + self._config = ffi.new("aegis_raf_config*") + self._config.scratch = self._scratch + self._config.chunk_size = chunk_size + flags = 0 + if creating: + flags |= _FLAG_CREATE + if truncate: + flags |= _FLAG_TRUNCATE + self._config.flags = flags + self._config.merkle = ffi.NULL + + self._merkle_hasher = None + self._merkle_cfg = None + self._merkle_buf = None + self._merkle_hasher_handle = None + if merkle: + hasher = SHA256MerkleHasher() if merkle is True else merkle + if merkle_max_chunks <= 0: + raise ValueError("merkle_max_chunks must be > 0") + if not (_MERKLE_HASH_MIN <= hasher.hash_len <= _MERKLE_HASH_MAX): + raise ValueError( + f"hash_len must be between {_MERKLE_HASH_MIN} and " + f"{_MERKLE_HASH_MAX}, got {hasher.hash_len}" + ) + mcfg = ffi.new("aegis_raf_merkle_config*") + mcfg.hash_len = hasher.hash_len + mcfg.max_chunks = merkle_max_chunks + buf_size = lib.aegis_raf_merkle_buffer_size(mcfg) + if buf_size == 0 or buf_size >= int(ffi.cast("size_t", -1)): + raise ValueError("Merkle tree buffer size overflow") + self._merkle_buf = bytearray(buf_size) + self._merkle_hasher_handle = ffi.new_handle(hasher) + mcfg.hash_leaf = _hash_leaf_callback + mcfg.hash_parent = _hash_parent_callback + mcfg.hash_empty = _hash_empty_callback + mcfg.hash_commitment = _hash_commitment_callback + mcfg.user = self._merkle_hasher_handle + mcfg.buf = ffi.from_buffer(self._merkle_buf) + mcfg.len = buf_size + self._merkle_hasher = hasher + self._merkle_cfg = mcfg + self._config.merkle = mcfg + + if creating: + cipher.raf_create(key, self._ctx, self._io, self._rng, self._config) + else: + cipher.raf_open(key, self._ctx, self._io, self._rng, self._config) + + def _check_open(self) -> None: + if self._closed: + raise ValueError("I/O operation on closed file") + + def read(self, size: int = -1, offset: int | None = None) -> bytes: + """Read and decrypt bytes, advancing the current position. + + Args: + size: Number of bytes to read (-1 for all remaining). + offset: Position to read from (None uses current position). + + Returns: + Decrypted bytes (fewer than requested at EOF). + + Raises: + ValueError: If authentication fails (corruption or tampering). + RuntimeError: If the read fails. + """ + self._check_open() + if offset is None: + offset = self._position + elif offset < 0: + raise ValueError(f"offset must be non-negative, got {offset}") + if size < 0: + size = max(0, self.size - offset) + if size == 0: + return b"" + out = self._cipher.raf_read(self._ctx, size, offset) + self._position = offset + len(out) + return bytes(out) + + def pread(self, size: int, offset: int) -> bytes: + """Read at offset without updating the position (like os.pread).""" + self._check_open() + if offset < 0: + raise ValueError(f"offset must be non-negative, got {offset}") + if size < 0: + raise ValueError(f"size must be non-negative, got {size}") + if size == 0: + return b"" + return bytes(self._cipher.raf_read(self._ctx, size, offset)) + + def read_into(self, buf: Buffer, offset: int | None = None) -> int: + """Read and decrypt into buf, advancing the position. + + Returns: + Number of bytes actually read. + """ + self._check_open() + buf = memoryview(buf) + if offset is None: + offset = self._position + elif offset < 0: + raise ValueError(f"offset must be non-negative, got {offset}") + if buf.nbytes == 0: + return 0 + out = self._cipher.raf_read(self._ctx, buf.nbytes, offset, into=buf) + n = len(out) + self._position = offset + n + return n + + def write(self, data: Buffer, offset: int | None = None) -> int: + """Encrypt and write bytes, advancing the current position. + + Returns: + Number of bytes written (always len(data) on success). + + Raises: + ValueError: If the write exceeds merkle_max_chunks. + RuntimeError: If the write fails. + """ + self._check_open() + data = memoryview(data) + if offset is None: + offset = self._position + elif offset < 0: + raise ValueError(f"offset must be non-negative, got {offset}") + if data.nbytes == 0: + return 0 + try: + n = self._cipher.raf_write(self._ctx, data, offset) + except RuntimeError as e: + if "EOVERFLOW" in str(e) and self._merkle_cfg is not None: + new_end = offset + data.nbytes + if new_end <= 0xFFFFFFFFFFFFFFFF: + chunk_size = self._config.chunk_size + new_chunks = (new_end + chunk_size - 1) // chunk_size + if new_chunks > self._merkle_cfg.max_chunks: + raise ValueError( + f"write exceeds merkle_max_chunks " + f"({self._merkle_cfg.max_chunks}); " + "open the file with a larger merkle_max_chunks" + ) from e + raise + self._position = offset + n + return n + + def pwrite(self, data: Buffer, offset: int) -> int: + """Write at offset without updating the position (like os.pwrite).""" + pos = self._position + try: + return self.write(data, offset) + finally: + self._position = pos + + def truncate(self, size: int | None = None) -> int: + """Resize the file (default: current position). Returns the new size.""" + self._check_open() + if size is None: + size = self._position + self._cipher.raf_truncate(self._ctx, size) + return size + + def seek(self, offset: int, whence: int = 0) -> int: + """Move the file position (whence: 0=absolute, 1=relative, 2=from end).""" + self._check_open() + if whence == 0: + new_pos = offset + elif whence == 1: + new_pos = self._position + offset + elif whence == 2: + new_pos = self.size + offset + else: + raise ValueError(f"invalid whence: {whence}") + if new_pos < 0: + raise ValueError(f"negative seek position: {new_pos}") + self._position = new_pos + return new_pos + + def tell(self) -> int: + """Return the current position.""" + return self._position + + @property + def size(self) -> int: + """Logical plaintext file size.""" + self._check_open() + return self._cipher.raf_size(self._ctx) + + def sync(self) -> None: + """Flush writes to backing storage.""" + self._check_open() + self._cipher.raf_sync(self._ctx) + + def close(self) -> None: + """Close the file, syncing and zeroizing key material.""" + if not self._closed: + self._closed = True + self._cipher.raf_close(self._ctx) + if self._owns_storage and hasattr(self._storage, "close"): + self._storage.close() + + @property + def closed(self) -> bool: + """True if the file has been closed.""" + return self._closed + + def __enter__(self) -> "Raf": + return self + + def __exit__(self, *args) -> None: + self.close() + + def _check_merkle_enabled(self) -> None: + if self._merkle_cfg is None: + raise ValueError("Merkle tree is not enabled on this file") + + def merkle_rebuild(self) -> None: + """Rebuild the Merkle tree by reading and re-hashing every chunk. + + Must be called after opening an existing file with merkle enabled. + """ + self._check_open() + self._check_merkle_enabled() + self._cipher.raf_merkle_rebuild(self._ctx) + + def merkle_verify(self) -> int | None: + """Verify every chunk's hash against the Merkle tree. + + Returns: + None if all chunks match, otherwise the index of the first + corrupted chunk. + """ + self._check_open() + self._check_merkle_enabled() + return self._cipher.raf_merkle_verify(self._ctx) + + @property + def root_hash(self) -> bytes | None: + """Current Merkle root commitment hash, or None if Merkle is not enabled.""" + self._check_open() + if self._merkle_cfg is None: + return None + return self._cipher.raf_merkle_commitment( + self._ctx, self._merkle_hasher.hash_len + ) + + def verify_root(self, expected: Buffer) -> None: + """Rebuild the Merkle tree and verify the root matches expected. + + Raises: + ValueError: If the root mismatches or expected has the wrong length. + """ + self._check_open() + self._check_merkle_enabled() + expected = memoryview(expected) + if expected.nbytes != self._merkle_hasher.hash_len: + raise ValueError( + f"expected must be {self._merkle_hasher.hash_len} bytes, " + f"got {expected.nbytes}" + ) + self.merkle_rebuild() + if self.root_hash != bytes(expected): + raise ValueError("authentication failed: merkle root mismatch") diff --git a/src/aeg/random.py b/src/aeg/random.py new file mode 100644 index 0000000..62605e1 --- /dev/null +++ b/src/aeg/random.py @@ -0,0 +1,113 @@ +"""Pluggable random sources for aeg, built on AEGIS streams. + +``Random`` is seeded once from the OS (secrets), then deterministic: each call +produces AEGIS keystream under the current key/nonce and increments the nonce +afterwards (little-endian), so no output block is ever produced twice. +Modeled on quantumpipe's ChaCha20-based Random class. + +Note: within the ``aeg`` package namespace this submodule shadows the stdlib +``random`` module; import it as ``from aeg import random`` (or ``aeg.random``). +""" + +import errno +import secrets +from typing import Protocol + +from ._loader import ffi +from .util import Buffer, nonce_increment + +__all__ = ["RandomSource", "Random", "system_random", "as_raf_rng"] + + +class RandomSource(Protocol): + """Callable returning n cryptographically secure random bytes.""" + + def __call__(self, n: int) -> bytes: ... + + +class Random: + """AEGIS stream RNG: seeded once from the OS, then deterministic. + + Each call produces keystream under the current key/nonce and increments + the nonce afterwards, so no output block is ever produced twice. + + The primary way to create one is the cipher module's own random() + helper, e.g. ``aeg.cipher("AEGIS-128X2").random()``. + """ + + def __init__(self, cipher, *, seed: Buffer | None = None): + """Create a Random using the given cipher module. + + Args: + cipher: Cipher module (from aeg.cipher()). + seed: Optional nonce || key bytes (NONCEBYTES + KEYBYTES long). + Primarily for tests; default seeds from the OS via secrets. + + Raises: + TypeError: If seed length is invalid. + """ + self._cipher = cipher + if seed is None: + seed = secrets.token_bytes(cipher.NONCEBYTES + cipher.KEYBYTES) + seed = memoryview(seed) + if seed.nbytes != cipher.NONCEBYTES + cipher.KEYBYTES: + raise TypeError( + f"seed length must be {cipher.NONCEBYTES + cipher.KEYBYTES} " + f"(nonce || key) for {cipher.NAME}" + ) + self._nonce = bytearray(seed[: cipher.NONCEBYTES]) + self._key = seed[cipher.NONCEBYTES :] + + def bytes(self, n: int) -> bytearray: + """Return n random bytes as a bytearray.""" + out = self._cipher.stream(self._key, self._nonce, n) + nonce_increment(self._nonce) + return out + + def into(self, buf: Buffer) -> None: + """Fill buf in place with random bytes.""" + mv = memoryview(buf).cast("B") + self._cipher.stream(self._key, self._nonce, into=mv) + nonce_increment(self._nonce) + + def __call__(self, n: int) -> bytearray: + """Alias for bytes(n); satisfies the RandomSource protocol.""" + return self.bytes(n) + + +def system_random(n: int) -> bytes: + """Random source backed directly by secrets.token_bytes.""" + return secrets.token_bytes(n) + + +@ffi.callback("int(void*, uint8_t*, size_t)") +def _random_callback(user, out, length): + try: + source = ffi.from_handle(user) + data = source(length) + if len(data) != length: + ffi.errno = errno.EINVAL + return -1 + ffi.memmove(out, data, length) + return 0 + except Exception: + ffi.errno = errno.EIO + return -1 + + +def as_raf_rng(source: RandomSource): + """Build an aegis_raf_rng* cdata wrapping a Python random source. + + Args: + source: Callable returning n random bytes, e.g. a Random instance + or system_random. + + Returns: + Tuple of (rng, keepalive). Keep ``keepalive`` alive for as long as + the rng may be called by C code. + """ + handle = ffi.new_handle(source) + rng = ffi.new("aegis_raf_rng*") + rng.user = handle + rng.random = _random_callback + return rng, (handle, _random_callback) diff --git a/tests/test_fuzz_raf_merkle.py b/tests/test_fuzz_raf_merkle.py new file mode 100644 index 0000000..3271f26 --- /dev/null +++ b/tests/test_fuzz_raf_merkle.py @@ -0,0 +1,1140 @@ +"""Fuzz tests for RAF and Merkle tree features. + +Adapted from pyaegis (github.com/jedisct1/pyaegis) tests by Frank Denis. + +Exercises random operation sequences, edge cases, and consistency invariants +to verify reliability of the encrypted random-access file and Merkle tree. +""" + +import hashlib +import os +import random + +import pytest + +from aeg import aegis128l, aegis128x2, aegis128x4, aegis256, aegis256x2, aegis256x4 +from aeg import raf +from aeg.raf import BytesIOStorage, FileStorage + +CIPHERS = [aegis128l, aegis128x2, aegis128x4, aegis256, aegis256x2, aegis256x4] + +# Subset used for heavier fuzz loops, as in pyaegis +FUZZ_CIPHERS = [aegis128l, aegis256] + + +def cipher_id(c): + return c.NAME + + +class RefFile: + """Reference in-memory plaintext file for oracle comparison.""" + + def __init__(self): + self._data = bytearray() + self._pos = 0 + + @property + def size(self): + return len(self._data) + + def write(self, data, offset=None): + if offset is None: + offset = self._pos + end = offset + len(data) + if end > len(self._data): + self._data.extend(b"\x00" * (end - len(self._data))) + self._data[offset : offset + len(data)] = data + self._pos = end + return len(data) + + def pwrite(self, data, offset): + end = offset + len(data) + if end > len(self._data): + self._data.extend(b"\x00" * (end - len(self._data))) + self._data[offset : offset + len(data)] = data + return len(data) + + def read(self, size=-1, offset=None): + if offset is None: + offset = self._pos + if size < 0: + size = max(0, len(self._data) - offset) + result = bytes(self._data[offset : offset + size]) + self._pos = offset + len(result) + return result + + def pread(self, size, offset): + return bytes(self._data[offset : offset + size]) + + def truncate(self, size=None): + if size is None: + size = self._pos + if size < len(self._data): + self._data = self._data[:size] + else: + self._data.extend(b"\x00" * (size - len(self._data))) + return size + + def seek(self, offset, whence=0): + if whence == 0: + self._pos = offset + elif whence == 1: + self._pos += offset + elif whence == 2: + self._pos = len(self._data) + offset + self._pos = max(0, self._pos) + return self._pos + + def tell(self): + return self._pos + + +class Blake2bMerkleHasher: + """Custom Merkle hasher using BLAKE2b for testing.""" + + hash_len = 32 + + def hash_leaf(self, chunk: bytes, chunk_len: int, chunk_idx: int) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x00") + h.update(chunk_idx.to_bytes(8, "little")) + h.update(chunk[:chunk_len]) + return h.digest() + + def hash_parent( + self, left: bytes, right: bytes, level: int, node_idx: int + ) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x01") + h.update(level.to_bytes(4, "little")) + h.update(node_idx.to_bytes(8, "little")) + h.update(left) + h.update(right) + return h.digest() + + def hash_empty(self, level: int, node_idx: int) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x02") + h.update(level.to_bytes(4, "little")) + h.update(node_idx.to_bytes(8, "little")) + return h.digest() + + def hash_commitment( + self, structural_root: bytes, ctx: bytes, file_size: int + ) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x03") + h.update(structural_root) + h.update(ctx) + h.update(file_size.to_bytes(8, "little")) + return h.digest() + + +class TestFuzzRAFRandomOps: + """Random operation sequences compared against a reference oracle.""" + + @pytest.mark.parametrize("seed", range(20)) + @pytest.mark.parametrize("c", FUZZ_CIPHERS, ids=cipher_id) + def test_random_ops_no_merkle(self, seed, c): + """Random write/read/seek/truncate ops match reference.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = c.random_key() + chunk_size = rng.choice([1024, 2048, 4096]) + ref = RefFile() + + with raf.create(storage, key, cipher=c, chunk_size=chunk_size) as f: + for _ in range(50): + op = rng.choice( + [ + "write", + "write", + "pwrite", + "read", + "pread", + "seek", + "truncate", + "read_all", + "size", + ] + ) + self._do_op(rng, f, ref, op) + + @pytest.mark.parametrize("seed", range(20)) + def test_random_ops_with_merkle(self, seed): + """Random ops with merkle=True; root stays consistent.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = rng.choice([1024, 2048]) + ref = RefFile() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=256, + ) as f: + for _ in range(40): + op = rng.choice( + [ + "write", + "write", + "pwrite", + "read", + "pread", + "seek", + "truncate", + "read_all", + "size", + ] + ) + self._do_op(rng, f, ref, op, max_data=chunk_size * 2) + + root = f.root_hash + assert root is not None and len(root) == 32 + + # Reopen and verify + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=256 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + # Data still matches reference + assert f.pread(ref.size, 0) == bytes(ref._data) + + def _do_op(self, rng, f, ref, op, max_data=4096): + if op == "write": + data = os.urandom(rng.randint(0, max_data)) + # Sometimes write at current pos, sometimes at explicit offset + if rng.random() < 0.3 and ref.size > 0: + offset = rng.randint(0, ref.size) + f.write(data, offset=offset) + ref.write(data, offset=offset) + else: + f.write(data) + ref.write(data) + + elif op == "pwrite": + data = os.urandom(rng.randint(0, max_data)) + offset = rng.randint(0, max(1, ref.size)) + f.pwrite(data, offset) + ref.pwrite(data, offset) + + elif op == "read": + size = rng.randint(0, max_data) + got = f.read(size) + expected = ref.read(size) + assert got == expected, ( + f"read({size}): got {len(got)} bytes, expected {len(expected)}" + ) + + elif op == "pread": + if ref.size > 0: + offset = rng.randint(0, ref.size - 1) + size = rng.randint(0, ref.size - offset + 100) + else: + offset = 0 + size = rng.randint(0, 100) + got = f.pread(size, offset) + expected = ref.pread(size, offset) + assert got == expected + + elif op == "read_all": + offset = rng.randint(0, max(1, ref.size)) + f.seek(offset) + ref.seek(offset) + got = f.read(-1) + expected = ref.read(-1) + assert got == expected + + elif op == "seek": + whence = rng.choice([0, 1, 2]) + if whence == 0: + offset = rng.randint(0, max(1, ref.size + 100)) + elif whence == 1: + # Avoid negative resulting position + offset = rng.randint(-ref.tell(), ref.size + 50) + else: + offset = rng.randint(-ref.size, 0) if ref.size > 0 else 0 + new_pos = f.seek(offset, whence) + ref_pos = ref.seek(offset, whence) + assert new_pos == ref_pos, ( + f"seek({offset}, {whence}): {new_pos} != {ref_pos}" + ) + + elif op == "truncate": + new_size = rng.randint(0, max(1, ref.size + 100)) + got = f.truncate(new_size) + expected = ref.truncate(new_size) + assert got == expected + + elif op == "size": + assert f.size == ref.size + + +class TestFuzzMerkleConsistency: + """Merkle tree consistency under random operations.""" + + @pytest.mark.parametrize("seed", range(10)) + @pytest.mark.parametrize("c", FUZZ_CIPHERS, ids=cipher_id) + def test_merkle_rebuild_always_matches(self, seed, c): + """After random writes, close/reopen/rebuild always reproduces root.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = c.random_key() + chunk_size = 1024 + + with raf.create( + storage, + key, + cipher=c, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=128, + ) as f: + # Do random writes + for _ in range(rng.randint(5, 20)): + data = os.urandom(rng.randint(1, chunk_size * 3)) + if rng.random() < 0.5 and f.size > 0: + offset = rng.randint(0, f.size) + f.pwrite(data, offset) + else: + f.write(data) + root = f.root_hash + + with raf.open(storage, key, cipher=c, merkle=True, merkle_max_chunks=128) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + + @pytest.mark.parametrize("seed", range(10)) + def test_verify_root_after_random_ops(self, seed): + """verify_root succeeds after random ops.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=128, + ) as f: + for _ in range(rng.randint(3, 15)): + data = os.urandom(rng.randint(1, 3000)) + f.write(data) + + # Also some truncations + if f.size > 0 and rng.random() < 0.3: + f.truncate(rng.randint(0, f.size)) + + root = f.root_hash + + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=128 + ) as f: + f.verify_root(root) + + @pytest.mark.parametrize("seed", range(10)) + def test_merkle_root_deterministic_across_write_patterns(self, seed): + """Different write patterns for same content produce same root after rebuild.""" + rng = random.Random(seed) + key = aegis128l.random_key() + total_size = rng.randint(100, 4000) + full_data = os.urandom(total_size) + + # Pattern 1: single write + s1 = BytesIOStorage() + with raf.create( + s1, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=64, + ) as f: + f.write(full_data) + root1 = f.root_hash + + # Reopen and rebuild - must match + with raf.open( + s1, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root1 + + # Pattern 2: chunked writes to same file (truncate and rewrite) + with raf.open( + s1, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + f.truncate(0) + pos = 0 + while pos < total_size: + chunk = rng.randint(1, min(500, total_size - pos)) + f.write(full_data[pos : pos + chunk]) + pos += chunk + root2 = f.root_hash + + # Pattern 3: pwrite in random order to same file + with raf.open( + s1, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + f.truncate(0) + f.write(b"\x00" * total_size) + offsets = list(range(0, total_size, 100)) + rng.shuffle(offsets) + for off in offsets: + end = min(off + 100, total_size) + f.pwrite(full_data[off:end], off) + root3 = f.root_hash + + assert root1 == root2 + assert root1 == root3 + + +class TestFuzzMerkleTampering: + """Detect tampering in ciphertext with Merkle verification.""" + + @pytest.mark.parametrize("seed", range(15)) + def test_tamper_random_byte_detected(self, seed): + """Flipping a random byte in the ciphertext is detected by rebuild or verify.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=64, + ) as f: + data = os.urandom(rng.randint(100, 5000)) + f.write(data) + original_root = f.root_hash + + raw = storage._data + # Skip the 92-byte header to tamper only chunk data + if len(raw) > 100: + tamper_pos = rng.randint(92, len(raw) - 1) + raw[tamper_pos] ^= rng.randint(1, 255) + + # Either rebuild fails (auth error) or verify detects mismatch + try: + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + result = f.merkle_verify() + # If rebuild succeeded, verify should find mismatch + # OR the root changed + assert result is not None or f.root_hash != original_root + except (ValueError, RuntimeError): + pass # Expected: tampered chunk fails authentication + + @pytest.mark.parametrize("seed", range(10)) + def test_tamper_multiple_bytes_detected(self, seed): + """Flipping multiple random bytes is detected.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=64, + ) as f: + f.write(os.urandom(rng.randint(2000, 8000))) + original_root = f.root_hash + + raw = storage._data + # Tamper 3-10 random bytes + n_tampers = rng.randint(3, 10) + for _ in range(n_tampers): + if len(raw) > 100: + pos = rng.randint(92, len(raw) - 1) + raw[pos] ^= rng.randint(1, 255) + + try: + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + result = f.merkle_verify() + assert result is not None or f.root_hash != original_root + except (ValueError, RuntimeError): + pass + + +class TestFuzzChunkBoundaries: + """Test operations at exact chunk boundaries and crossing them.""" + + @pytest.mark.parametrize("chunk_size", [1024, 2048]) + @pytest.mark.parametrize("seed", range(5)) + def test_write_at_chunk_boundaries(self, chunk_size, seed): + """Writes that start/end/cross chunk boundaries.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + ref = RefFile() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=64, + ) as f: + for _i in range(20): + # Choose offsets near chunk boundaries + chunk_idx = rng.randint(0, 5) + boundary = chunk_idx * chunk_size + offset = boundary + rng.choice([-2, -1, 0, 1, 2, chunk_size // 2]) + offset = max(0, offset) + size = rng.choice( + [ + 1, + chunk_size - 1, + chunk_size, + chunk_size + 1, + chunk_size * 2, + rng.randint(1, chunk_size * 3), + ] + ) + data = os.urandom(size) + + f.pwrite(data, offset) + ref.pwrite(data, offset) + + # Verify full content matches + assert f.size == ref.size + assert f.pread(f.size, 0) == bytes(ref._data) + + root = f.root_hash + + # Verify merkle + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + + @pytest.mark.parametrize("seed", range(5)) + def test_read_across_chunk_boundaries(self, seed): + """Reads that span multiple chunks.""" + rng = random.Random(seed) + chunk_size = 1024 + storage = BytesIOStorage() + key = aegis128l.random_key() + + total = chunk_size * 5 + data = os.urandom(total) + + with raf.create(storage, key, cipher=aegis128l, chunk_size=chunk_size) as f: + f.write(data) + + # Random reads crossing boundaries + for _ in range(30): + offset = rng.randint(0, total - 1) + size = rng.randint(1, total - offset) + got = f.pread(size, offset) + assert got == data[offset : offset + size] + + +class TestFuzzTruncate: + """Fuzz truncation combined with writes and merkle.""" + + @pytest.mark.parametrize("seed", range(10)) + def test_truncate_and_rewrite(self, seed): + """Truncate to various sizes then rewrite; data and merkle stay consistent.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + ref = RefFile() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=128, + ) as f: + # Initial write + data = os.urandom(rng.randint(500, 5000)) + f.write(data) + ref.write(data) + + for _ in range(15): + op = rng.choice(["truncate", "write", "pwrite"]) + if op == "truncate": + new_size = rng.randint(0, ref.size + 500) + f.truncate(new_size) + ref.truncate(new_size) + elif op == "write": + d = os.urandom(rng.randint(0, 2000)) + f.write(d) + ref.write(d) + elif op == "pwrite": + d = os.urandom(rng.randint(0, 1000)) + off = rng.randint(0, max(1, ref.size)) + f.pwrite(d, off) + ref.pwrite(d, off) + + assert f.size == ref.size + if ref.size > 0: + assert f.pread(ref.size, 0) == bytes(ref._data) + + root = f.root_hash + + # Reopen and verify + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=128 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + + +class TestFuzzEmptyAndTiny: + """Edge cases with empty files and tiny writes.""" + + def test_empty_file_merkle(self): + """Create empty file with merkle, root_hash is valid.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + root = f.root_hash + assert root is not None + assert len(root) == 32 + + def test_single_byte_write_merkle(self): + """Single byte write with merkle.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"\x42") + root = f.root_hash + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + assert f.pread(1, 0) == b"\x42" + + def test_write_then_truncate_to_zero(self): + """Write data then truncate to zero; merkle should be consistent.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=64, + ) as f: + f.write(os.urandom(3000)) + f.truncate(0) + assert f.size == 0 + root = f.root_hash + + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=64 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + + @pytest.mark.parametrize("size", [1, 15, 16, 17, 1023, 1024, 1025]) + def test_exact_sizes_roundtrip(self, size): + """Files of exact sizes (around alignment) round-trip correctly.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + data = os.urandom(size) + + with raf.create( + storage, key, cipher=aegis128l, chunk_size=1024, merkle=True + ) as f: + f.write(data) + root = f.root_hash + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.pread(size, 0) == data + + +class TestFuzzMultipleReopenCycles: + """Test repeated open/write/close/reopen cycles.""" + + @pytest.mark.parametrize("seed", range(5)) + def test_multiple_reopen_append(self, seed): + """Open/write/close/reopen cycle many times; data accumulates correctly.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + ref = RefFile() + + # Initial create + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=256, + ) as f: + d = os.urandom(rng.randint(100, 500)) + f.write(d) + ref.write(d) + + for _cycle in range(8): + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=256 + ) as f: + f.merkle_rebuild() + # Sometimes append, sometimes overwrite + if rng.random() < 0.6: + d = os.urandom(rng.randint(50, 300)) + offset = f.size + f.pwrite(d, offset) + ref.pwrite(d, offset) + else: + if f.size > 0: + offset = rng.randint(0, f.size - 1) + d = os.urandom(rng.randint(1, 200)) + f.pwrite(d, offset) + ref.pwrite(d, offset) + + # Final verify + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=256 + ) as f: + f.merkle_rebuild() + assert f.merkle_verify() is None + assert f.pread(ref.size, 0) == bytes(ref._data) + + +class TestFuzzAllVariants: + """Test all 6 AEGIS variants with random operations.""" + + @pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) + @pytest.mark.parametrize("seed", range(3)) + def test_variant_random_ops_merkle(self, c, seed): + """Each variant handles random ops + merkle correctly.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = c.random_key() + ref = RefFile() + + with raf.create( + storage, key, cipher=c, chunk_size=1024, merkle=True, merkle_max_chunks=128 + ) as f: + for _ in range(20): + data = os.urandom(rng.randint(1, 2000)) + if rng.random() < 0.4 and ref.size > 0: + off = rng.randint(0, ref.size) + f.pwrite(data, off) + ref.pwrite(data, off) + else: + f.write(data) + ref.write(data) + root = f.root_hash + + with raf.open(storage, key, cipher=c, merkle=True, merkle_max_chunks=128) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + if ref.size > 0: + assert f.pread(ref.size, 0) == bytes(ref._data) + + +class TestFuzzCustomHasher: + """Fuzz with custom Merkle hashers.""" + + @pytest.mark.parametrize("seed", range(5)) + def test_blake2b_random_ops(self, seed): + """Custom Blake2b hasher under random operations.""" + rng = random.Random(seed) + hasher = Blake2bMerkleHasher() + storage = BytesIOStorage() + key = aegis128l.random_key() + ref = RefFile() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=hasher, + merkle_max_chunks=128, + ) as f: + for _ in range(25): + data = os.urandom(rng.randint(1, 2000)) + f.write(data) + ref.write(data) + root = f.root_hash + + with raf.open( + storage, key, cipher=aegis128l, merkle=hasher, merkle_max_chunks=128 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + assert f.pread(ref.size, 0) == bytes(ref._data) + + def test_sha256_vs_blake2b_differ(self): + """SHA256 and Blake2b hashers produce different roots for same data.""" + key = aegis128l.random_key() + data = os.urandom(2000) + + sha_storage = BytesIOStorage() + with raf.create( + sha_storage, key, cipher=aegis128l, chunk_size=1024, merkle=True + ) as f: + f.write(data) + sha_root = f.root_hash + + b2_storage = BytesIOStorage() + with raf.create( + b2_storage, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=Blake2bMerkleHasher(), + ) as f: + f.write(data) + b2_root = f.root_hash + + assert sha_root != b2_root + + +class TestFuzzWrongKey: + """Fuzz: opening files with wrong keys.""" + + @pytest.mark.parametrize("seed", range(10)) + def test_wrong_key_rejected(self, seed): + """Random data written, then reopened with wrong key fails.""" + rng = random.Random(seed) + c = rng.choice(CIPHERS) + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c, chunk_size=1024) as f: + f.write(os.urandom(rng.randint(100, 3000))) + + wrong_key = c.random_key() + assert wrong_key != key # Negligible collision probability + with pytest.raises(ValueError, match="authentication failed"): + raf.open(storage, wrong_key, cipher=c) + + +class TestFuzzFileStorage: + """Fuzz test RAF with real file I/O.""" + + @pytest.mark.parametrize("seed", range(5)) + def test_file_storage_random_ops(self, seed, tmp_path): + """Random ops on file-backed storage produce correct results.""" + rng = random.Random(seed) + path = tmp_path / f"fuzz_{seed}.aegis" + key = aegis128l.random_key() + ref = RefFile() + + with FileStorage(path, "w+b") as fs: + with raf.create( + fs, + key, + cipher=aegis128l, + chunk_size=1024, + merkle=True, + merkle_max_chunks=128, + ) as f: + for _ in range(20): + data = os.urandom(rng.randint(1, 1500)) + if rng.random() < 0.3 and ref.size > 0: + off = rng.randint(0, ref.size) + f.pwrite(data, off) + ref.pwrite(data, off) + else: + f.write(data) + ref.write(data) + root = f.root_hash + + # Reopen from file and verify + with FileStorage(path, "r+b") as fs: + with raf.open( + fs, + key, + cipher=aegis128l, + merkle=True, + merkle_max_chunks=128, + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + if ref.size > 0: + assert f.pread(ref.size, 0) == bytes(ref._data) + + +class TestFuzzRafOpen: + """Fuzz probe + explicit-cipher open with random data. + + pyaegis tested raf_open auto-detection; our raf.open() requires the + cipher, so we check probe() reports the right alg_id instead. + """ + + @pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) + def test_probe_reports_variant(self, c): + """probe() correctly reports each variant after random writes.""" + storage = BytesIOStorage() + key = c.random_key() + data = os.urandom(random.randint(100, 3000)) + + with raf.create(storage, key, cipher=c, chunk_size=1024, merkle=True) as f: + f.write(data) + root = f.root_hash + + assert raf.probe(storage).alg_id == c.RAF_ALG_ID + + with raf.open(storage, key, cipher=c, merkle=True) as f: + f.verify_root(root) + assert f.pread(len(data), 0) == data + + +class TestFuzzMerkleMaxChunks: + """Test merkle_max_chunks boundary.""" + + def test_write_exactly_max_chunks(self): + """Writing exactly max_chunks chunks is OK.""" + chunk_size = 1024 + max_chunks = 4 + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=max_chunks, + ) as f: + # Write exactly max_chunks * chunk_size bytes + data = os.urandom(chunk_size * max_chunks) + f.write(data) + assert f.root_hash is not None + + def test_write_exceeds_max_chunks(self): + """Writing past max_chunks raises ValueError.""" + chunk_size = 1024 + max_chunks = 4 + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=max_chunks, + ) as f: + data = os.urandom(chunk_size * (max_chunks + 1)) + with pytest.raises(ValueError, match="merkle_max_chunks"): + f.write(data) + + @pytest.mark.parametrize("max_chunks", [1, 2, 3, 8]) + def test_boundary_max_chunks(self, max_chunks): + """Boundary test: fill exactly to max, verify merkle still works.""" + chunk_size = 1024 + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=max_chunks, + ) as f: + data = os.urandom(chunk_size * max_chunks) + f.write(data) + root = f.root_hash + + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=max_chunks + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + + +class TestFuzzReadInto: + """Fuzz test read_into with random patterns.""" + + @pytest.mark.parametrize("seed", range(5)) + def test_read_into_matches_read(self, seed): + """read_into produces same result as read for same offset/size.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + data = os.urandom(rng.randint(500, 5000)) + + with raf.create(storage, key, cipher=aegis128l, chunk_size=1024) as f: + f.write(data) + + for _ in range(30): + offset = rng.randint(0, len(data)) + size = rng.randint(0, len(data) - offset + 100) + size = min(size, len(data)) + + f.seek(offset) + read_result = f.read(size) + + buf = bytearray(size) + f.seek(offset) + n = f.read_into(buf, offset) + + assert bytes(buf[:n]) == read_result + + +class TestFuzzSeekTell: + """Fuzz seek/tell consistency.""" + + @pytest.mark.parametrize("seed", range(5)) + def test_seek_tell_consistency(self, seed): + """seek returns same value as subsequent tell.""" + rng = random.Random(seed) + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, chunk_size=1024) as f: + f.write(os.urandom(rng.randint(100, 3000))) + + for _ in range(50): + whence = rng.choice([0, 1, 2]) + if whence == 0: + off = rng.randint(0, f.size + 100) + elif whence == 1: + off = rng.randint(-f.tell(), f.size + 50) + else: + off = rng.randint(-f.size, 50) + + pos = f.seek(off, whence) + assert f.tell() == pos + + +class TestFuzzLargeFile: + """Test with larger data sizes.""" + + def test_large_sequential_writes_merkle(self): + """100KB of data in sequential writes with merkle verification.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + total = 100 * 1024 + chunk_size = 4096 + ref = bytearray() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=256, + ) as f: + written = 0 + while written < total: + piece = os.urandom(random.randint(1, 8192)) + f.write(piece) + ref.extend(piece) + written += len(piece) + root = f.root_hash + + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=256 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + assert f.pread(len(ref), 0) == bytes(ref) + + def test_large_random_access_merkle(self): + """Random reads/writes across 100KB file with merkle.""" + rng = random.Random(42) + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = 4096 + ref = RefFile() + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=256, + ) as f: + # Initial fill + initial = os.urandom(50 * 1024) + f.write(initial) + ref.write(initial) + + # Random overwrites + for _ in range(100): + off = rng.randint(0, ref.size - 1) + d = os.urandom(rng.randint(1, 4096)) + f.pwrite(d, off) + ref.pwrite(d, off) + + # Verify content + assert f.size == ref.size + # Read in chunks to verify + for off in range(0, ref.size, chunk_size): + size = min(chunk_size, ref.size - off) + assert f.pread(size, off) == ref.pread(size, off) + + root = f.root_hash + + with raf.open( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=256 + ) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + + +class TestFuzzConcurrentStorageOps: + """Test that storage state is consistent across RAF operations.""" + + def test_probe_after_random_writes(self): + """raf.probe returns consistent info after random writes.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = 2048 + + with raf.create(storage, key, cipher=aegis128l, chunk_size=chunk_size) as f: + for _ in range(10): + f.write(os.urandom(random.randint(100, 3000))) + expected_size = f.size + + _alg_id, probed_chunk, probed_size = raf.probe(storage) + assert probed_chunk == chunk_size + assert probed_size == expected_size diff --git a/tests/test_raf.py b/tests/test_raf.py new file mode 100644 index 0000000..faf9618 --- /dev/null +++ b/tests/test_raf.py @@ -0,0 +1,358 @@ +import pytest + +from aeg import aegis128l, aegis128x2, aegis128x4, aegis256, aegis256x2, aegis256x4 +from aeg import raf +from aeg import random as aeg_random + +CIPHERS = [aegis128l, aegis128x2, aegis128x4, aegis256, aegis256x2, aegis256x4] + +CHUNK = 1024 + + +def cipher_id(c): + return c.NAME + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_roundtrip_bytesio(c): + key = c.random_key() + data = bytes(range(256)) * 20 # spans multiple chunks + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + assert f.write(data) == len(data) + assert f.size == len(data) + with raf.open(st, key, c) as f: + assert f.read() == data + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_roundtrip_filestorage(c, tmp_path): + key = c.random_key() + data = b"file-backed" * 500 + path = tmp_path / "test.raf" + with raf.create(path, key, c, chunk_size=CHUNK) as f: + f.write(data) + with raf.open(path, key, c) as f: + assert f.read() == data + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_random_access(c): + key = c.random_key() + st = raf.BytesIOStorage() + data = bytearray(4097) + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(data) + # unaligned writes across chunk boundaries + f.pwrite(b"ABCDEF", 1000) # crosses chunk 0/1 boundary at 1024 + f.pwrite(b"z", 4096) # last byte + data[1000:1006] = b"ABCDEF" + data[4096:4097] = b"z" + assert f.pread(6, 1000) == b"ABCDEF" + assert f.pread(4097, 0) == bytes(data) + assert f.tell() == len(data) # pwrite/pread don't move the position + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_seek_tell_read_into(c): + key = c.random_key() + st = raf.BytesIOStorage() + data = b"0123456789" * 100 + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(data) + assert f.tell() == len(data) + assert f.seek(10) == 10 + assert f.read(5) == data[10:15] + assert f.tell() == 15 + assert f.seek(-5, 1) == 10 + assert f.seek(0, 2) == len(data) + buf = bytearray(20) + assert f.read_into(buf, 100) == 20 + assert bytes(buf) == data[100:120] + with pytest.raises(ValueError): + f.seek(-1) + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_truncate(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(b"x" * 3000) + assert f.truncate(1500) == 1500 + assert f.size == 1500 + assert f.pread(1500, 0) == b"x" * 1500 + f.truncate(2500) # grow, zero-filled + assert f.size == 2500 + assert f.pread(1000, 1500) == b"\x00" * 1000 + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_wrong_key(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(b"secret") + with pytest.raises(ValueError, match="authentication failed"): + raf.open(st, c.random_key(), c) + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_tampered_chunk(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(b"y" * 2048) + # flip a ciphertext byte inside the first chunk record (after the nonce) + st._data[raf.HEADER_SIZE + c.NONCEBYTES + 10] ^= 1 + with raf.open(st, key, c) as f: + with pytest.raises(ValueError, match="authentication failed"): + f.read() + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_probe_and_cipher_mismatch(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(b"data") + info = raf.probe(st) + assert info.alg_id == c.RAF_ALG_ID + assert info.chunk_size == CHUNK + assert info.file_size == 4 + other = aegis256 if c is not aegis256 else aegis128l + with pytest.raises(ValueError, match="algorithm id"): + raf.open(st, key, other) + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_create_existing(c, tmp_path): + key = c.random_key() + path = tmp_path / "exists.raf" + raf.create(path, key, c, chunk_size=CHUNK).close() + with pytest.raises(FileExistsError): + raf.create(path, key, c, chunk_size=CHUNK) + with raf.create(path, key, c, chunk_size=CHUNK, truncate=True) as f: + assert f.size == 0 + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_open_missing(c, tmp_path): + with pytest.raises(FileNotFoundError): + raf.open(tmp_path / "missing.raf", c.random_key(), c) + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_merkle(c): + key = c.random_key() + st = raf.BytesIOStorage() + data = b"m" * 5000 + with raf.create(st, key, c, chunk_size=CHUNK, merkle=True) as f: + f.write(data) + root1 = f.root_hash + assert root1 is not None and len(root1) == 32 + assert f.merkle_verify() is None + f.pwrite(b"X", 0) + root2 = f.root_hash + assert root1 != root2 + assert f.merkle_verify() is None + with raf.open(st, key, c, merkle=True) as f: + f.verify_root(root2) # rebuilds and compares + assert f.root_hash == root2 + with pytest.raises(ValueError, match="authentication failed"): + f.verify_root(root1) + with pytest.raises(ValueError): + f.verify_root(b"short") + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_merkle_detects_corruption(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK, merkle=True) as f: + f.write(b"z" * 3000) + f.merkle_rebuild() + record = c.NONCEBYTES + CHUNK + 16 + # corrupt chunk 2's ciphertext in the backing store + st._data[raf.HEADER_SIZE + 2 * record + c.NONCEBYTES + 5] ^= 1 + assert f.merkle_verify() == 2 + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_merkle_overflow(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create( + st, key, c, chunk_size=CHUNK, merkle=True, merkle_max_chunks=2 + ) as f: + f.write(b"a" * 2048) + with pytest.raises(ValueError, match="merkle_max_chunks"): + f.write(b"b") + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_merkle_disabled(c): + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + assert f.root_hash is None + with pytest.raises(ValueError, match="not enabled"): + f.merkle_verify() + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_custom_rng(c): + counter = 0 + + def deterministic(n: int) -> bytes: + nonlocal counter + out = bytes((counter + i) % 256 for i in range(n)) + counter += n + return out + + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK, rng=deterministic) as f: + f.write(b"rng test") + with raf.open(st, key, c) as f: + assert f.read() == b"rng test" + assert counter > 0 + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_default_rng_matches_cipher(c): + """raf.create/open without rng use a Random seeded for the same cipher.""" + key = c.random_key() + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + assert f._rng_keepalive[0] # handle alive + f.write(b"default rng") + with raf.open(st, key, c) as f: + assert f.read() == b"default rng" + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_random_per_cipher(c): + """Random can be constructed with each cipher module.""" + r1 = aeg_random.Random(c) + r2 = c.random() + out = r1.bytes(100) + assert len(out) == 100 + assert any(out) # not all zeros + assert len(r2.bytes(50)) == 50 + # Random instance satisfies the RandomSource callable protocol + assert callable(r1) + rng_cdata, keepalive = aeg_random.as_raf_rng(r1) + assert rng_cdata.random is not None and keepalive + + +def test_random_determinism_with_seed(): + """Two Random instances seeded identically produce identical output.""" + c = aegis256x4 + seed = bytes(range(c.NONCEBYTES + c.KEYBYTES)) + r1 = aeg_random.Random(c, seed=seed) + r2 = aeg_random.Random(c, seed=seed) + assert r1.bytes(64) == r2.bytes(64) + assert r1.bytes(1000) == r2.bytes(1000) + with pytest.raises(TypeError, match="seed length"): + aeg_random.Random(c, seed=b"short") + + +def test_random_into_equivalence(): + """into() fills in place and matches bytes() from an identically-seeded RNG.""" + c = aegis256x4 + seed = bytes(c.NONCEBYTES + c.KEYBYTES) + r1 = aeg_random.Random(c, seed=seed) + r2 = aeg_random.Random(c, seed=seed) + buf = bytearray(128) + r1.into(buf) + assert bytes(buf) == bytes(r2.bytes(128)) + # buffers of other buffer-protocol types work too + import array + + arr = array.array("b", bytes(32)) + r1.into(arr) + assert any(arr.tobytes()) + + +def test_random_no_repeat(): + """Successive calls never repeat a block (nonce increments).""" + r = aeg_random.Random(aegis128x2) + blocks = {bytes(r.bytes(64)) for _ in range(1000)} + assert len(blocks) == 1000 + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +def test_cipher_module_random_helper(c): + """Each cipher module provides a random() helper bound to itself.""" + r = c.random() + assert isinstance(r, aeg_random.Random) + assert r._cipher is c + assert len(r.bytes(32)) == 32 + buf = bytearray(48) + r.into(buf) + assert any(buf) + + +def test_random_module(): + assert len(aeg_random.system_random(16)) == 16 + # any callable source can be bridged to the C RNG struct + rng, keepalive = aeg_random.as_raf_rng(aeg_random.system_random) + assert rng.random is not None and keepalive + + def bad(n: int) -> bytes: + return b"too short" + + rng2, _ = aeg_random.as_raf_rng(bad) + out = aeg_random.ffi.new("uint8_t[8]") + assert rng2.random(rng2.user, out, 8) == -1 + + +def test_derive_master_key(): + mk16 = bytes(range(16)) + mk32 = bytes(range(32)) + assert len(raf.derive_master_key(mk16)) == 16 + assert len(raf.derive_master_key(mk32)) == 32 + # context-bound: different contexts give different keys + assert raf.derive_master_key(mk32, b"a") != raf.derive_master_key(mk32, b"b") + # empty context still derives a scoped subkey + assert raf.derive_master_key(mk32) != mk32 + with pytest.raises(ValueError): + raf.derive_master_key(b"\x00" * 24) + with pytest.raises(ValueError): + raf.derive_master_key(mk16, b"c" * 121) + with pytest.raises(ValueError): + raf.derive_master_key(mk32, b"c" * 73) + # derived keys work end-to-end + c = aegis256x4 + key = raf.derive_master_key(mk32, b"my-file") + st = raf.BytesIOStorage() + with raf.create(st, key, c, chunk_size=CHUNK) as f: + f.write(b"derived") + with raf.open(st, key, c) as f: + assert f.read() == b"derived" + + +def test_invalid_chunk_size(): + c = aegis256x4 + with pytest.raises(ValueError, match="chunk_size"): + raf.create(raf.BytesIOStorage(), c.random_key(), c, chunk_size=512) + with pytest.raises(ValueError, match="multiple of 16"): + raf.create(raf.BytesIOStorage(), c.random_key(), c, chunk_size=1032) + + +def test_bad_key_length(): + c = aegis256x4 + with pytest.raises(TypeError, match="key length"): + raf.create(raf.BytesIOStorage(), b"\x00" * 16, c, chunk_size=CHUNK) + + +def test_closed_file(): + c = aegis256x4 + st = raf.BytesIOStorage() + f = raf.create(st, c.random_key(), c, chunk_size=CHUNK) + f.close() + assert f.closed + with pytest.raises(ValueError, match="closed"): + f.read() + f.close() # idempotent diff --git a/tests/test_raf_pyaegis.py b/tests/test_raf_pyaegis.py new file mode 100644 index 0000000..aecd2b4 --- /dev/null +++ b/tests/test_raf_pyaegis.py @@ -0,0 +1,1113 @@ +"""Tests for RAF (Random Access Format) encrypted file API. + +Adapted from pyaegis (github.com/jedisct1/pyaegis) tests by Frank Denis. + +Exception mapping from pyaegis: RAFAuthenticationError -> ValueError, +RAFConfigError -> ValueError/TypeError, RAFError/RAFIOError -> RuntimeError. +Their auto-detect raf_open() does not apply: our raf.open() requires the +cipher explicitly and validates it against the probed header. +""" + +import hashlib + +import pytest + +from aeg import aegis128l, aegis128x2, aegis128x4, aegis256, aegis256x2, aegis256x4 +from aeg import raf +from aeg.raf import ( + BytesIOStorage, + FileStorage, + MerkleHasher, + SHA256MerkleHasher, +) + +CIPHERS = [aegis128l, aegis128x2, aegis128x4, aegis256, aegis256x2, aegis256x4] + + +def cipher_id(c): + return c.NAME + + +class TestRafDeriveMasterKey: + """Tests for context-bound RAF key derivation.""" + + def test_128_bit_known_answer(self): + master_key = bytes(range(16)) + + derived = raf.derive_master_key(master_key, b"test-context") + + assert derived == bytes.fromhex("fb80072c5a6f1cddc6e97b35ed1f3bf3") + + def test_256_bit_known_answer(self): + master_key = bytes(range(32)) + + derived = raf.derive_master_key(master_key, b"test-context") + + assert derived == bytes.fromhex( + "fee2d3cc58c69d8f43fd7b4e33eaec0053539685c7e284e6e12ee9c4f423d136" + ) + + def test_empty_context_is_domain_separated(self): + master_key = bytes(range(16)) + + derived = raf.derive_master_key(master_key) + + assert derived == bytes.fromhex("9b8e6ddb09c9eb0137888ca2a366fdd0") + assert derived != master_key + + @pytest.mark.parametrize(("key_size", "max_context_size"), [(16, 120), (32, 72)]) + def test_context_length_limits(self, key_size, max_context_size): + master_key = bytes(range(key_size)) + + assert ( + len(raf.derive_master_key(master_key, b"x" * max_context_size)) == key_size + ) + with pytest.raises(ValueError, match=f"at most {max_context_size} bytes"): + raf.derive_master_key(master_key, b"x" * (max_context_size + 1)) + + @pytest.mark.parametrize("key_size", [0, 15, 17, 31, 33]) + def test_invalid_key_length(self, key_size): + with pytest.raises(ValueError, match="master_key must be 16 or 32 bytes"): + raf.derive_master_key(b"x" * key_size) + + @pytest.mark.parametrize( + ("master_key", "context"), + [("not-bytes", b"context"), (b"x" * 16, "not-bytes")], + ) + def test_requires_bytes(self, master_key, context): + # pyaegis checked types explicitly; we raise TypeError from memoryview() + with pytest.raises(TypeError): + raf.derive_master_key(master_key, context) + + def test_derived_key_opens_only_with_same_context(self): + storage = BytesIOStorage() + master_key = bytes(range(16)) + key = raf.derive_master_key(master_key, b"context-a") + + with raf.create(storage, key, cipher="AEGIS-128L") as f: + f.write(b"context-bound data") + + same_key = raf.derive_master_key(master_key, b"context-a") + with raf.open(storage, same_key, cipher="AEGIS-128L") as f: + assert f.read() == b"context-bound data" + + wrong_key = raf.derive_master_key(master_key, b"context-b") + with pytest.raises(ValueError, match="authentication failed"): + raf.open(storage, wrong_key, cipher="AEGIS-128L") + + +class TestBytesIOStorage: + """Tests for BytesIOStorage backend.""" + + def test_read_write_basic(self): + """Test basic read/write operations.""" + storage = BytesIOStorage() + storage.write_at(b"hello", 0) + buf = bytearray(5) + storage.read_at(buf, 0) + assert buf == b"hello" + + def test_read_past_eof_raises(self): + """Test that reading past EOF raises IOError.""" + storage = BytesIOStorage(b"short") + buf = bytearray(10) + with pytest.raises(IOError): + storage.read_at(buf, 0) + + def test_write_extends_buffer(self): + """Test that writing past end extends the buffer.""" + storage = BytesIOStorage() + storage.write_at(b"hello", 10) + assert storage.get_size() == 15 + assert storage.getvalue()[:10] == b"\x00" * 10 + assert storage.getvalue()[10:] == b"hello" + + def test_set_size_truncate(self): + """Test truncating via set_size.""" + storage = BytesIOStorage(b"hello world") + storage.set_size(5) + assert storage.getvalue() == b"hello" + + def test_set_size_extend(self): + """Test extending via set_size.""" + storage = BytesIOStorage(b"hi") + storage.set_size(5) + assert storage.getvalue() == b"hi\x00\x00\x00" + + def test_context_manager(self): + """Test context manager support.""" + with BytesIOStorage(b"test") as storage: + buf = bytearray(4) + storage.read_at(buf, 0) + assert buf == b"test" + + def test_getvalue_returns_bytes(self): + """getvalue() returns immutable bytes, not bytearray.""" + storage = BytesIOStorage(b"test") + result = storage.getvalue() + assert isinstance(result, bytes) + assert not isinstance(result, bytearray) + + def test_initial_data_is_copied(self): + """Mutating initial_data after construction has no effect.""" + data = bytearray(b"hello") + storage = BytesIOStorage(data) + data[0] = 0xFF + assert storage.getvalue() == b"hello" + + def test_read_write_at_exact_boundary(self): + """Write then read at the exact end of existing data.""" + storage = BytesIOStorage(b"hello") + storage.write_at(b"world", 5) + assert storage.get_size() == 10 + buf = bytearray(5) + storage.read_at(buf, 5) + assert buf == b"world" + + def test_overwrite_same_region(self): + """Repeatedly overwriting the same region keeps final value.""" + storage = BytesIOStorage(b"\x00" * 10) + for i in range(100): + storage.write_at(bytes([i % 256]), 5) + assert storage.get_size() == 10 + buf = bytearray(1) + storage.read_at(buf, 5) + assert buf[0] == 99 + + def test_overwrite_preserves_surrounding(self): + """Overwriting a middle region doesn't corrupt surrounding data.""" + storage = BytesIOStorage(b"AABBBBCC") + storage.write_at(b"XXXX", 2) + assert storage.getvalue() == b"AAXXXXCC" + + def test_zero_length_read(self): + """Reading zero bytes succeeds even on empty storage.""" + storage = BytesIOStorage() + buf = bytearray(0) + storage.read_at(buf, 0) + assert buf == b"" + + def test_zero_length_write(self): + """Writing zero bytes doesn't change size.""" + storage = BytesIOStorage() + storage.write_at(b"", 0) + assert storage.get_size() == 0 + + def test_zero_length_write_at_offset(self): + """Writing zero bytes at an offset doesn't extend.""" + storage = BytesIOStorage(b"hi") + storage.write_at(b"", 100) + assert storage.get_size() == 2 + + def test_set_size_to_zero(self): + """Truncating to zero clears everything.""" + storage = BytesIOStorage(b"X" * 10000) + storage.set_size(0) + assert storage.get_size() == 0 + assert storage.getvalue() == b"" + + def test_set_size_same(self): + """set_size to current size is a no-op.""" + storage = BytesIOStorage(b"hello") + storage.set_size(5) + assert storage.getvalue() == b"hello" + + def test_sparse_write_large_gap(self): + """Writing past a large gap fills with zeros.""" + storage = BytesIOStorage() + storage.write_at(b"end", 10000) + assert storage.get_size() == 10003 + assert storage.getvalue()[:10000] == b"\x00" * 10000 + assert storage.getvalue()[10000:] == b"end" + + def test_read_past_eof_various(self): + """Read past EOF at different positions.""" + storage = BytesIOStorage(b"abc") + with pytest.raises(OSError): + storage.read_at(bytearray(1), 3) + with pytest.raises(OSError): + storage.read_at(bytearray(4), 0) + with pytest.raises(OSError): + storage.read_at(bytearray(2), 2) + + def test_read_at_offset_zero_on_empty(self): + """Reading any bytes from empty storage raises.""" + storage = BytesIOStorage() + with pytest.raises(OSError): + storage.read_at(bytearray(1), 0) + + def test_sync_is_noop(self): + """sync() doesn't raise or modify data.""" + storage = BytesIOStorage(b"data") + storage.sync() + storage.sync() + assert storage.getvalue() == b"data" + + def test_truncate_then_extend_zeros(self): + """Truncate then extend fills new region with zeros, not old data.""" + storage = BytesIOStorage(b"ABCDEFGHIJ") + storage.set_size(3) + storage.set_size(10) + assert storage.getvalue() == b"ABC" + b"\x00" * 7 + + def test_write_overlapping_existing(self): + """Write that partially overlaps existing data and extends.""" + storage = BytesIOStorage(b"hello") + storage.write_at(b"WORLD!", 3) + assert storage.getvalue() == b"helWORLD!" + assert storage.get_size() == 9 + + def test_sequential_adjacent_writes(self): + """Adjacent writes produce contiguous data.""" + storage = BytesIOStorage() + storage.write_at(b"AAA", 0) + storage.write_at(b"BBB", 3) + storage.write_at(b"CCC", 6) + assert storage.getvalue() == b"AAABBBCCC" + + def test_get_size_empty(self): + """Empty storage has size 0.""" + assert BytesIOStorage().get_size() == 0 + + def test_get_size_with_initial_data(self): + """Size matches initial data length.""" + assert BytesIOStorage(b"hello").get_size() == 5 + + +class TestFileStorage: + """Tests for FileStorage backend.""" + + def test_read_write_basic(self, tmp_path): + """Test basic file read/write operations.""" + path = tmp_path / "test.bin" + with FileStorage(path, "w+b") as storage: + storage.write_at(b"hello", 0) + buf = bytearray(5) + storage.read_at(buf, 0) + assert buf == b"hello" + + def test_read_short_raises(self, tmp_path): + """Test that short read raises IOError.""" + path = tmp_path / "test.bin" + with FileStorage(path, "w+b") as storage: + storage.write_at(b"hi", 0) + buf = bytearray(10) + with pytest.raises(IOError, match="Short read"): + storage.read_at(buf, 0) + + def test_file_size(self, tmp_path): + """Test get_size and set_size.""" + path = tmp_path / "test.bin" + with FileStorage(path, "w+b") as storage: + storage.write_at(b"hello world", 0) + assert storage.get_size() == 11 + storage.set_size(5) + assert storage.get_size() == 5 + + def test_invalid_mode_raises(self, tmp_path): + """Test that invalid mode raises ValueError.""" + path = tmp_path / "test.bin" + with pytest.raises(ValueError, match="Unsupported mode"): + FileStorage(path, "rb") + + +@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) +class TestRafVariants: + """Per-variant tests, merged from pyaegis's six AegisRaf* classes.""" + + def test_create_write_read_basic(self, c): + """Test creating a file, writing, closing, reopening, and reading.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"Hello, World!") + + with raf.open(storage, key, cipher=c) as f: + assert f.read() == b"Hello, World!" + + def test_create_with_file_storage(self, c, tmp_path): + """Test with FileStorage backend.""" + path = tmp_path / "test.raf" + key = c.random_key() + + with FileStorage(path, "w+b") as storage: + with raf.create(storage, key, cipher=c) as f: + f.write(b"Hello, World!") + + with FileStorage(path, "r+b") as storage: + with raf.open(storage, key, cipher=c) as f: + assert f.read() == b"Hello, World!" + + def test_random_access(self, c): + """Test random access read/write.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"AAAAAAAAAA") + f.seek(5) + f.write(b"BBBBB") + f.seek(0) + assert f.read() == b"AAAAABBBBB" + + def test_pread_pwrite(self, c): + """Test pread/pwrite don't update position.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"0123456789") + assert f.tell() == 10 + + # pread doesn't update position + assert f.pread(3, 0) == b"012" + assert f.tell() == 10 + + # pwrite doesn't update position and preserves existing data + f.pwrite(b"ABC", 5) + assert f.tell() == 10 + + f.seek(0) + assert f.read() == b"01234ABC89" + + def test_read_into(self, c): + """Test read_into with pre-allocated buffer.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"Hello, World!") + + with raf.open(storage, key, cipher=c) as f: + buf = bytearray(5) + n = f.read_into(buf) + assert n == 5 + assert buf == b"Hello" + assert f.tell() == 5 + + def test_seek_operations(self, c): + """Test seek with different whence values.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"0123456789") + + assert f.seek(0) == 0 + assert f.seek(5, 0) == 5 + assert f.seek(2, 1) == 7 + assert f.seek(-3, 2) == 7 + assert f.seek(0, 2) == 10 + + def test_truncate(self, c): + """Test file truncation.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"Hello, World!") + assert f.size == 13 + f.truncate(5) + assert f.size == 5 + f.seek(0) + assert f.read() == b"Hello" + + def test_truncate_uses_position(self, c): + """Test truncate with no argument uses current position.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"Hello, World!") + f.seek(5) + f.truncate() + assert f.size == 5 + + def test_wrong_key_fails(self, c): + """Test that wrong key fails authentication.""" + storage = BytesIOStorage() + key1 = c.random_key() + key2 = c.random_key() + + with raf.create(storage, key1, cipher=c) as f: + f.write(b"secret") + + with pytest.raises(ValueError, match="authentication failed"): + raf.open(storage, key2, cipher=c) + + def test_invalid_key_size_rejected(self, c): + """Test that invalid key size is rejected.""" + storage = BytesIOStorage() + + with pytest.raises(TypeError, match="key length"): + raf.create(storage, b"short", cipher=c) + + def test_invalid_chunk_size_rejected(self, c): + """Test invalid chunk_size is rejected.""" + storage = BytesIOStorage() + key = c.random_key() + + with pytest.raises(ValueError, match="chunk_size must be"): + raf.create(storage, key, cipher=c, chunk_size=100) + + with pytest.raises(ValueError, match="multiple of 16"): + raf.create(BytesIOStorage(), key, cipher=c, chunk_size=1025) + + def test_truncate_overwrite(self, c): + """Test create with truncate=True overwrites existing file.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"original content") + + with raf.create(storage, key, cipher=c, truncate=True) as f: + f.write(b"new") + + with raf.open(storage, key, cipher=c) as f: + assert f.read() == b"new" + + def test_empty_file(self, c): + """Test empty file operations.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + assert f.size == 0 + assert f.read() == b"" + + def test_single_byte(self, c): + """Test single byte read/write.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"X") + f.seek(0) + assert f.read() == b"X" + + def test_large_file(self, c): + """Test large file operations.""" + storage = BytesIOStorage() + key = c.random_key() + data = b"X" * 100000 + + with raf.create(storage, key, cipher=c) as f: + f.write(data) + + with raf.open(storage, key, cipher=c) as f: + assert f.read() == data + + def test_custom_chunk_size(self, c): + """Test with custom chunk size.""" + storage = BytesIOStorage() + key = c.random_key() + chunk_size = 4096 + + with raf.create(storage, key, cipher=c, chunk_size=chunk_size) as f: + data = b"A" * (chunk_size * 2 + 100) + f.write(data) + + with raf.open(storage, key, cipher=c) as f: + assert f.read() == data + + def test_negative_offset_rejected(self, c): + """Test that negative offsets raise ValueError.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"test") + + with pytest.raises(ValueError, match="non-negative"): + f.read(10, offset=-1) + with pytest.raises(ValueError, match="non-negative"): + f.pread(10, -1) + with pytest.raises(ValueError, match="non-negative"): + f.read_into(bytearray(10), offset=-1) + with pytest.raises(ValueError, match="non-negative"): + f.write(b"x", offset=-1) + with pytest.raises(ValueError, match="non-negative"): + f.pwrite(b"x", -1) + + def test_negative_size_rejected(self, c): + """Test that negative size in pread raises ValueError.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"test") + with pytest.raises(ValueError, match="non-negative"): + f.pread(-5, 0) + + def test_context_manager(self, c): + """Test context manager properly closes file.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"test") + assert not f.closed + + assert f.closed + + def test_operations_on_closed_file_fail(self, c): + """Test that operations on closed file raise ValueError.""" + storage = BytesIOStorage() + key = c.random_key() + + f = raf.create(storage, key, cipher=c) + f.write(b"test") + f.close() + + with pytest.raises(ValueError, match="closed"): + f.read() + with pytest.raises(ValueError, match="closed"): + f.write(b"x") + with pytest.raises(ValueError, match="closed"): + f.seek(0) + with pytest.raises(ValueError, match="closed"): + _ = f.size + + def test_key_sizes(self, c): + """Test that key sizes are correct.""" + expected = 16 if "128" in c.NAME else 32 + assert c.KEYBYTES == expected + assert c.NONCEBYTES == expected + + +class TestRafProbe: + """Tests for raf.probe function.""" + + def test_probe_default_chunk_size(self): + """Test probing an AEGIS-128L file.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + f.write(b"test data") + + alg_id, chunk_size, file_size = raf.probe(storage) + assert alg_id == aegis128l.RAF_ALG_ID + assert chunk_size == 65536 + assert file_size == 9 + + def test_probe_custom_chunk_size(self): + """Test probing an AEGIS-256 file with custom chunk size.""" + storage = BytesIOStorage() + key = aegis256.random_key() + + with raf.create(storage, key, cipher=aegis256, chunk_size=4096) as f: + f.write(b"hello") + + alg_id, chunk_size, _ = raf.probe(storage) + assert alg_id == aegis256.RAF_ALG_ID + assert chunk_size == 4096 + + def test_probe_invalid_file_fails(self): + """Test that probing invalid file fails.""" + storage = BytesIOStorage(b"not a valid RAF file") + + with pytest.raises(RuntimeError, match="probe failed"): + raf.probe(storage) + + +class TestRafOpen: + """Tests for raf.open with explicit cipher (pyaegis auto-detect doesn't apply).""" + + @pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) + def test_probe_reports_cipher_and_open_reads(self, c): + """probe() reports the cipher's alg_id; open() with that cipher reads.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c) as f: + f.write(b"test variant") + + info = raf.probe(storage) + assert info.alg_id == c.RAF_ALG_ID + + with raf.open(storage, key, cipher=c) as f: + assert f.read() == b"test variant" + + def test_wrong_key_fails(self): + """Test raf.open with wrong key fails.""" + storage = BytesIOStorage() + key1 = aegis128l.random_key() + key2 = aegis128l.random_key() + + with raf.create(storage, key1, cipher=aegis128l) as f: + f.write(b"secret") + + with pytest.raises(ValueError, match="authentication failed"): + raf.open(storage, key2, cipher=aegis128l) + + +class TestAlgorithmMismatch: + """Test opening file with wrong cipher.""" + + def test_wrong_cipher_rejected(self): + """Test that opening with the wrong cipher is rejected.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + f.write(b"test") + + with pytest.raises(ValueError, match="uses algorithm"): + raf.open(storage, aegis256.random_key(), cipher=aegis256) + + +class TestChunkBoundary: + """Tests for operations at chunk boundaries.""" + + def test_exact_chunk_write(self): + """Test writing exactly one chunk.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = 4096 + + with raf.create(storage, key, cipher=aegis128l, chunk_size=chunk_size) as f: + data = b"X" * chunk_size + f.write(data) + + with raf.open(storage, key, cipher=aegis128l) as f: + assert f.read() == data + + def test_cross_chunk_read(self): + """Test read spanning multiple chunks.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = 4096 + + with raf.create(storage, key, cipher=aegis128l, chunk_size=chunk_size) as f: + data = b"ABCD" * chunk_size + f.write(data) + + with raf.open(storage, key, cipher=aegis128l) as f: + # Read spanning chunk boundary + partial = f.pread(100, chunk_size - 50) + assert len(partial) == 100 + + def test_cross_chunk_write(self): + """Test write spanning multiple chunks (overwriting).""" + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = 4096 + + with raf.create(storage, key, cipher=aegis128l, chunk_size=chunk_size) as f: + # Write initial data spanning two chunks + f.write(b"0" * (chunk_size + 100)) + # Overwrite across chunk boundary + f.pwrite(b"X" * 100, chunk_size - 50) + + with raf.open(storage, key, cipher=aegis128l) as f: + data = f.read() + assert len(data) == chunk_size + 100 + # First part unchanged + assert data[: chunk_size - 50] == b"0" * (chunk_size - 50) + # Overwritten part + assert data[chunk_size - 50 : chunk_size + 50] == b"X" * 100 + # Trailing part unchanged + assert data[chunk_size + 50 :] == b"0" * 50 + + +class TestSync: + """Tests for sync operation.""" + + def test_sync_basic(self, tmp_path): + """Test sync flushes to storage.""" + path = tmp_path / "test.raf" + key = aegis128l.random_key() + + with FileStorage(path, "w+b") as storage: + with raf.create(storage, key, cipher=aegis128l) as f: + f.write(b"data to sync") + f.sync() + + +class TestReadAtEOF: + """Tests for reading at/past EOF.""" + + def test_read_at_eof_returns_empty(self): + """Test reading at EOF returns empty bytes.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + f.write(b"hello") + + with raf.open(storage, key, cipher=aegis128l) as f: + f.seek(0, 2) + assert f.read() == b"" + + def test_read_partial_at_eof(self): + """Test reading more bytes than available returns partial.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + f.write(b"hello") + + with raf.open(storage, key, cipher=aegis128l) as f: + data = f.pread(100, 0) + assert data == b"hello" + + +class Blake2bMerkleHasher: + """Custom Merkle hasher using BLAKE2b for testing.""" + + hash_len = 32 + + def hash_leaf(self, chunk: bytes, chunk_len: int, chunk_idx: int) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x00") + h.update(chunk_idx.to_bytes(8, "little")) + h.update(chunk[:chunk_len]) + return h.digest() + + def hash_parent( + self, left: bytes, right: bytes, level: int, node_idx: int + ) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x01") + h.update(level.to_bytes(4, "little")) + h.update(node_idx.to_bytes(8, "little")) + h.update(left) + h.update(right) + return h.digest() + + def hash_empty(self, level: int, node_idx: int) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x02") + h.update(level.to_bytes(4, "little")) + h.update(node_idx.to_bytes(8, "little")) + return h.digest() + + def hash_commitment( + self, structural_root: bytes, ctx: bytes, file_size: int + ) -> bytes: + h = hashlib.blake2b(digest_size=32) + h.update(b"\x03") + h.update(structural_root) + h.update(ctx) + h.update(file_size.to_bytes(8, "little")) + return h.digest() + + +class TestMerkle: + """Tests for Merkle tree support in RAF.""" + + def test_merkle_true_roundtrip(self): + """merkle=True round-trip: root_hash is 32 bytes.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"Hello, Merkle!") + root = f.root_hash + assert root is not None + assert len(root) == 32 + + def test_root_hash_changes_on_write(self): + """Root hash changes after additional write.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"first") + root1 = f.root_hash + f.write(b"second") + root2 = f.root_hash + assert root1 != root2 + + def test_root_hash_deterministic(self): + """Same file rewritten with same data produces same root commitment.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + data = b"deterministic content" + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(data) + root1 = f.root_hash + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.merkle_rebuild() + root2 = f.root_hash + + assert root1 == root2 + + def test_rebuild_reproduces_root(self): + """Close and reopen: rebuild reproduces the same root hash.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"data for rebuild test") + original_root = f.root_hash + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.merkle_rebuild() + assert f.root_hash == original_root + + def test_verify_clean_file(self): + """Verify returns None for an untampered file.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"clean data") + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.merkle_rebuild() + assert f.merkle_verify() is None + + def test_verify_root_happy_path(self): + """verify_root succeeds with correct root.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"verify root test") + root = f.root_hash + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.verify_root(root) + + def test_verify_root_wrong_root(self): + """verify_root raises ValueError with wrong root.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"some data") + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + with pytest.raises(ValueError, match="merkle root mismatch"): + f.verify_root(b"\x00" * 32) + + def test_rebuild_fails_on_tampered_ciphertext(self): + """Tamper with raw storage; rebuild fails authentication.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"X" * 1000) + + raw = storage._data + tamper_offset = 100 + if tamper_offset < len(raw): + raw[tamper_offset] ^= 0xFF + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + with pytest.raises((ValueError, RuntimeError)): + f.merkle_rebuild() + + def test_verify_detects_hash_mismatch(self): + """Corrupt in-memory tree buffer; verify returns chunk index.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"data to verify") + f._merkle_buf[0] ^= 0xFF + result = f.merkle_verify() + assert result is not None + assert isinstance(result, int) + + def test_merkle_max_chunks_zero_rejected(self): + """merkle_max_chunks=0 raises ValueError.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with pytest.raises(ValueError, match="must be > 0"): + raf.create(storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=0) + + def test_overflow_protection(self): + """Absurdly large merkle_max_chunks raises ValueError.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with pytest.raises(ValueError, match="overflow"): + raf.create( + storage, key, cipher=aegis128l, merkle=True, merkle_max_chunks=2**62 + ) + + def test_rebuild_without_merkle_raises(self): + """merkle_rebuild on non-merkle file raises ValueError.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + with pytest.raises(ValueError, match="not enabled"): + f.merkle_rebuild() + + def test_verify_without_merkle_raises(self): + """merkle_verify on non-merkle file raises ValueError.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + with pytest.raises(ValueError, match="not enabled"): + f.merkle_verify() + + def test_root_hash_none_without_merkle(self): + """root_hash returns None when merkle is not enabled.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l) as f: + assert f.root_hash is None + + def test_custom_hasher_wrong_digest_length(self): + """Hasher returning wrong digest length causes write to fail.""" + + class BadHasher: + hash_len = 32 + + def hash_leaf(self, chunk, chunk_len, chunk_idx): + return b"\x00" * 16 # Wrong: 16 instead of 32 + + def hash_parent(self, left, right, level, node_idx): + return b"\x00" * 16 + + def hash_empty(self, level, node_idx): + return b"\x00" * 16 + + def hash_commitment(self, structural_root, ctx, file_size): + return b"\x00" * 16 + + storage = BytesIOStorage() + key = aegis128l.random_key() + + with pytest.raises((RuntimeError, ValueError)): + with raf.create(storage, key, cipher=aegis128l, merkle=BadHasher()) as f: + f.write(b"test data") + + def test_merkle_eoverflow_suggests_fix(self): + """Writing past merkle_max_chunks raises ValueError with hint.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + chunk_size = 1024 + + with raf.create( + storage, + key, + cipher=aegis128l, + chunk_size=chunk_size, + merkle=True, + merkle_max_chunks=2, + ) as f: + data = b"X" * (chunk_size * 3) + with pytest.raises(ValueError, match="merkle_max_chunks"): + f.write(data) + + @pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) + def test_merkle_all_variants_roundtrip(self, c): + """merkle=True works with all 6 RAF variants.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c, merkle=True) as f: + f.write(b"variant test data") + root = f.root_hash + assert root is not None + assert len(root) == 32 + + @pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) + def test_merkle_all_variants_rebuild(self, c): + """merkle_rebuild reproduces root across all variants.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c, merkle=True) as f: + f.write(b"rebuild variant test") + original_root = f.root_hash + + with raf.open(storage, key, cipher=c, merkle=True) as f: + f.merkle_rebuild() + assert f.root_hash == original_root + + @pytest.mark.parametrize("c", CIPHERS, ids=cipher_id) + def test_merkle_all_variants_verify(self, c): + """merkle_verify returns None for clean file across all variants.""" + storage = BytesIOStorage() + key = c.random_key() + + with raf.create(storage, key, cipher=c, merkle=True) as f: + f.write(b"verify variant test") + + with raf.open(storage, key, cipher=c, merkle=True) as f: + f.merkle_rebuild() + assert f.merkle_verify() is None + + def test_custom_hasher_blake2b(self): + """Custom BLAKE2b hasher works correctly.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + hasher = Blake2bMerkleHasher() + + with raf.create(storage, key, cipher=aegis128l, merkle=hasher) as f: + f.write(b"blake2b test data") + root = f.root_hash + assert root is not None + assert len(root) == 32 + + with raf.open(storage, key, cipher=aegis128l, merkle=hasher) as f: + f.merkle_rebuild() + assert f.root_hash == root + assert f.merkle_verify() is None + + def test_verify_root_wrong_length(self): + """verify_root with wrong length raises ValueError.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"test") + with pytest.raises(ValueError, match="must be 32 bytes"): + f.verify_root(b"\x00" * 16) + + def test_raf_open_with_merkle(self): + """raf.open passes merkle params correctly.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + f.write(b"raf_open merkle test") + root = f.root_hash + + with raf.open(storage, key, cipher=aegis128l, merkle=True) as f: + f.verify_root(root) + + def test_sha256_merkle_hasher_is_merkle_hasher(self): + """SHA256MerkleHasher satisfies MerkleHasher protocol.""" + assert isinstance(SHA256MerkleHasher(), MerkleHasher) + + def test_merkle_methods_on_closed_file(self): + """All merkle methods raise ValueError on closed file.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + f = raf.create(storage, key, cipher=aegis128l, merkle=True) + f.write(b"test data") + f.close() + + with pytest.raises(ValueError, match="closed"): + f.merkle_rebuild() + with pytest.raises(ValueError, match="closed"): + f.merkle_verify() + with pytest.raises(ValueError, match="closed"): + f.verify_root(b"\x00" * 32) + with pytest.raises(ValueError, match="closed"): + _ = f.root_hash + + def test_eoverflow_with_absurd_offset_no_merkle_hint(self): + """EOVERFLOW from absurd offset doesn't mention merkle_max_chunks.""" + storage = BytesIOStorage() + key = aegis128l.random_key() + + with raf.create(storage, key, cipher=aegis128l, merkle=True) as f: + with pytest.raises(RuntimeError, match="EOVERFLOW"): + f.write(b"x", offset=2**64 - 1) diff --git a/tools/generate.py b/tools/generate.py index 48344db..b0400e3 100755 --- a/tools/generate.py +++ b/tools/generate.py @@ -88,6 +88,7 @@ def generate_cdef(include_dir: pathlib.Path) -> str: "aegis256.h", "aegis256x2.h", "aegis256x4.h", + "aegis_raf.h", ] for header_name in headers: @@ -105,9 +106,9 @@ def generate_cdef(include_dir: pathlib.Path) -> str: def extract_constants( - common_h_path: pathlib.Path, header_path: pathlib.Path + common_h_path: pathlib.Path, header_path: pathlib.Path, raf_h_path: pathlib.Path ) -> Dict[str, int]: - """Extract constants from common.h (ALIGNMENT, RATE) and main header (KEYBYTES, NPUBBYTES, ABYTES_*).""" + """Extract constants from common.h (ALIGNMENT, RATE), main header (KEYBYTES, NPUBBYTES, ABYTES_*), and aegis_raf.h (RAF_ALG_ID).""" constants = {} # Extract from common.h @@ -136,6 +137,14 @@ def extract_constants( raise ValueError(f"Could not extract {const_name} from {header_path}") constants[const_name] = int(match.group(1)) + # Extract RAF algorithm id from aegis_raf.h + raf_content = raf_h_path.read_text(encoding="utf-8") + raf_pattern = rf"^\s*#define\s+AEGIS_RAF_ALG_{variant[5:].upper()}\s+(\d+)" + raf_match = re.search(raf_pattern, raf_content, re.MULTILINE) + if not raf_match: + raise ValueError(f"Could not extract RAF_ALG_ID from {raf_h_path}") + constants["RAF_ALG_ID"] = int(raf_match.group(1)) + return constants @@ -155,6 +164,7 @@ def extract_all_constants( for variant in variants: common_h = libaegis_src_dir / variant / f"{variant}_common.h" header_h = include_dir / f"{variant}.h" + raf_h = include_dir / "aegis_raf.h" if not common_h.exists(): print(f"Warning: {common_h} not found, skipping {variant}", file=sys.stderr) @@ -165,7 +175,7 @@ def extract_all_constants( continue try: - constants[variant] = extract_constants(common_h, header_h) + constants[variant] = extract_constants(common_h, header_h, raf_h) except Exception as e: print(f"Error extracting constants from {variant}: {e}", file=sys.stderr) @@ -215,6 +225,11 @@ def generate_variant(template_src: str, variant: str, constants: Dict[str, int]) f"MACBYTES_LONG = {constants['ABYTES_MAX']}", s, ) + s = re.sub( + r"RAF_ALG_ID = \d+", + f"RAF_ALG_ID = {constants['RAF_ALG_ID']}", + s, + ) return s @@ -262,6 +277,11 @@ def generate_python_modules( f"MACBYTES_LONG = {const_dict['ABYTES_MAX']}", new_content, ) + new_content = re.sub( + r"RAF_ALG_ID = \d+", + f"RAF_ALG_ID = {const_dict['RAF_ALG_ID']}", + new_content, + ) else: new_content = generate_variant(template_src, variant, const_dict)