aeg.raf: StreamStorage adapter; accept io.BytesIO directly
StreamStorage wraps any seek-based file-like object using the same lock + seek/read/write pattern as FileStorage's fallback for platforms without pread/pwrite. create()/open() auto-wrap io.BytesIO, so in-memory use no longer requires the explicit BytesIOStorage wrapper. The caller keeps ownership of the stream (Raf does not close it).
This commit is contained in:
1 parent
f2a21856b3
commit
32954c4eba
3 files changed
+86
-3
No files matched your search
@@ -135,7 +135,7 @@ Seeded once from the OS, then a deterministic AEGIS keystream: each call increme
|
||||
|
||||
### Random-access encrypted files (aeg.raf)
|
||||
|
||||
pread/pwrite-style access to files split into independently encrypted chunks, with optional Merkle tree integrity. Storage is pluggable (file path, aeg.raf.FileStorage, aeg.raf.BytesIOStorage, or a custom Storage).
|
||||
pread/pwrite-style access to files split into independently encrypted chunks, with optional Merkle tree integrity. Storage is pluggable: a file path, io.BytesIO, aeg.raf.FileStorage / BytesIOStorage / StreamStorage, or a custom Storage.
|
||||
|
||||
- raf.create(path_or_storage, key, cipher, chunk_size=65536, merkle=False) / raf.open(...)
|
||||
- f.read(size, offset), f.pread(size, offset), f.write(data, offset), f.pwrite(data, offset)
|
||||
|
||||
+65
-2
@@ -4,8 +4,8 @@ Provides pread/pwrite-style access to encrypted files. Files are divided into
|
||||
fixed-size chunks, each independently encrypted with a fresh nonce, enabling
|
||||
efficient random access without decrypting the whole file.
|
||||
|
||||
Use create() / open() with a cipher module (from aeg.cipher()) and either a
|
||||
path or a Storage instance.
|
||||
Use create() / open() with a cipher module (from aeg.cipher()) and a path,
|
||||
an io.BytesIO, or a Storage instance.
|
||||
|
||||
Portions of the storage/callback/Merkle/file-like machinery are adapted from
|
||||
pyaegis (https://github.com/jedisct1/pyaegis) by Frank Denis, MIT license.
|
||||
@@ -15,6 +15,7 @@ from __future__ import annotations
|
||||
|
||||
import errno as errno_module
|
||||
import hashlib
|
||||
import io as io_module
|
||||
import os
|
||||
import threading
|
||||
from typing import NamedTuple, Protocol, runtime_checkable
|
||||
@@ -33,6 +34,7 @@ __all__ = [
|
||||
"Storage",
|
||||
"FileStorage",
|
||||
"BytesIOStorage",
|
||||
"StreamStorage",
|
||||
"MerkleHasher",
|
||||
"SHA256MerkleHasher",
|
||||
"RafInfo",
|
||||
@@ -218,6 +220,65 @@ class BytesIOStorage:
|
||||
pass
|
||||
|
||||
|
||||
class StreamStorage:
|
||||
"""Storage adapter for seek-based file-like objects (io.BytesIO, open files).
|
||||
|
||||
Uses a lock plus seek/read/write, mirroring FileStorage's fallback path
|
||||
for platforms without os.pread/os.pwrite. The stream is not closed by
|
||||
Raf; closing it remains the caller's responsibility.
|
||||
"""
|
||||
|
||||
def __init__(self, stream):
|
||||
self._stream = stream
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def read_at(self, buf: bytearray, offset: int) -> None:
|
||||
"""Read exactly len(buf) bytes at offset into buf."""
|
||||
with self._lock:
|
||||
self._stream.seek(offset)
|
||||
view = memoryview(buf)
|
||||
done = 0
|
||||
while done < len(view):
|
||||
chunk = self._stream.read(len(view) - done)
|
||||
if not chunk:
|
||||
raise OSError(f"Short read: expected {len(view)}, got {done}")
|
||||
view[done : done + len(chunk)] = chunk
|
||||
done += len(chunk)
|
||||
|
||||
def write_at(self, data: bytes, offset: int) -> None:
|
||||
"""Write exactly len(data) bytes at offset."""
|
||||
with self._lock:
|
||||
self._stream.seek(offset)
|
||||
view = memoryview(data)
|
||||
done = 0
|
||||
while done < len(view):
|
||||
done += self._stream.write(view[done:])
|
||||
|
||||
def get_size(self) -> int:
|
||||
"""Return current stream size."""
|
||||
with self._lock:
|
||||
pos = self._stream.tell()
|
||||
try:
|
||||
return self._stream.seek(0, os.SEEK_END)
|
||||
finally:
|
||||
self._stream.seek(pos)
|
||||
|
||||
def set_size(self, size: int) -> None:
|
||||
"""Resize the stream (truncate or extend with zeros)."""
|
||||
self._stream.truncate(size)
|
||||
|
||||
def sync(self) -> None:
|
||||
"""Flush the stream if it supports flushing."""
|
||||
if hasattr(self._stream, "flush"):
|
||||
self._stream.flush()
|
||||
|
||||
def __enter__(self) -> StreamStorage:
|
||||
return self
|
||||
|
||||
def __exit__(self, *args) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class MerkleHasher(Protocol):
|
||||
"""Protocol for Merkle tree hash functions."""
|
||||
@@ -507,6 +568,8 @@ def probe(storage: Storage) -> RafInfo:
|
||||
def _as_storage(storage_or_path, mode: str) -> tuple[Storage, bool]:
|
||||
if isinstance(storage_or_path, (str, bytes, os.PathLike)):
|
||||
return FileStorage(storage_or_path, mode), True
|
||||
if isinstance(storage_or_path, io_module.BytesIO):
|
||||
return StreamStorage(storage_or_path), False
|
||||
return storage_or_path, False
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,26 @@ def test_roundtrip_bytesio(c):
|
||||
assert f.read() == data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id)
|
||||
def test_roundtrip_io_bytesio(c):
|
||||
"""A bare io.BytesIO is auto-wrapped in StreamStorage."""
|
||||
import io
|
||||
|
||||
key = c.random_key()
|
||||
data = bytes(range(256)) * 20
|
||||
bio = io.BytesIO()
|
||||
with raf.create(bio, key, c, chunk_size=CHUNK) as f:
|
||||
f.write(data)
|
||||
f.pwrite(b"XY", 1000) # unaligned write across a chunk boundary
|
||||
data = data[:1000] + b"XY" + data[1002:]
|
||||
f.truncate(2000)
|
||||
with raf.open(bio, key, c) as f:
|
||||
assert f.size == 2000
|
||||
assert f.pread(6, 999) == data[999:1005]
|
||||
assert f.read() == data[:2000]
|
||||
bio.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("c", CIPHERS, ids=cipher_id)
|
||||
def test_roundtrip_filestorage(c, tmp_path):
|
||||
key = c.random_key()
|
||||
|
||||
Reference in new issue
Block a user