Encrypted events with JSON underneath in RailsEventStore
One of the most wanted features in RES is the encryption of event payloads — full or partial. This allows storing sensitive information and is often necessary for compliance in particular domains.
With Postgres backing the storage, it is natural to choose JSON/B as the data type for the event payload. It is the default. And a choice that further allows structural searches within event payloads with JSON functions and operators — useful and effective for digging deeper or doing some data science from raw facts.
Unfortunately, encrypted payloads and JSON don't work together out of the box:
JSON text exchanged between systems that are not part of a closed ecosystem MUST be encoded using UTF-8
the ciphertext may contain byte sequences that are invalid in UTF-8
The solution is to encode encrypted parts, or the complete payloads, into a JSON-safe representation: from binary to ASCII chars. Depending on the chosen encoder, the encrypted payload would grow in size:
| Format | Alphabet | Overhead |
|---|---|---|
| Base85 | 85 characters | ~25% |
| Base64 | 64 characters | ~33% |
| Base32 | 32 characters | ~60% |
| Hex | 16 characters | ~100% |
How to combine all that in RES?
Let's start with a failing example, replicating the issue. An in-memory RES instance that serializes event payloads to JSON with encryption mapper hooked in:
require "bundler/inline"
require "json"
require "openssl"
require "base64"
gemfile { gem "ruby_event_store" }
class TicketHolderEmailProvided < RubyEventStore::Event
def self.encryption_schema =
{
email: ->(data) do
data.fetch(:user_id)
end
}
end
class InMemoryEncryptionKeyRepository
DEFAULT_CIPHER = "aes-256-cbc".freeze
def initialize
@keys = {}
end
def key_of(
identifier,
cipher: DEFAULT_CIPHER
)
@keys[[identifier, cipher]]
end
def create(
identifier,
cipher: DEFAULT_CIPHER
)
@keys[
[identifier, cipher]
] = RubyEventStore::Mappers::EncryptionKey.new(
cipher: cipher,
key: random_key(cipher)
)
end
def forget(identifier)
@keys =
@keys.reject do |(id, _)|
id.eql?(identifier)
end
end
private
def random_key(cipher)
crypto = OpenSSL::Cipher.new(cipher)
crypto.encrypt
crypto.random_key
end
end
key_repository =
InMemoryEncryptionKeyRepository.new
key_repository.create(user_id = 2137)
res =
RubyEventStore::Client.new(
repository:
RubyEventStore::InMemoryRepository.new(
serializer: JSON
),
mapper:
RubyEventStore::Mappers::EncryptionMapper.new(
key_repository
)
)
res.append(
TicketHolderEmailProvided.new(
data: {
email: "kaka@dudu.me",
user_id: user_id
}
)
)
require "bundler/inline"
require "json"
require "openssl"
require "base64"
gemfile { gem "ruby_event_store" }
class TicketHolderEmailProvided < RubyEventStore::Event
def self.encryption_schema = { email: ->(data) { data.fetch(:user_id) } }
end
class InMemoryEncryptionKeyRepository
DEFAULT_CIPHER = "aes-256-cbc".freeze
def initialize
@keys = {}
end
def key_of(identifier, cipher: DEFAULT_CIPHER)
@keys[[identifier, cipher]]
end
def create(identifier, cipher: DEFAULT_CIPHER)
@keys[[identifier, cipher]] = RubyEventStore::Mappers::EncryptionKey.new(
cipher: cipher,
key: random_key(cipher)
)
end
def forget(identifier)
@keys = @keys.reject { |(id, _)| id.eql?(identifier) }
end
private
def random_key(cipher)
crypto = OpenSSL::Cipher.new(cipher)
crypto.encrypt
crypto.random_key
end
end
key_repository = InMemoryEncryptionKeyRepository.new
key_repository.create(user_id = 2137)
res =
RubyEventStore::Client.new(
repository: RubyEventStore::InMemoryRepository.new(serializer: JSON),
mapper: RubyEventStore::Mappers::EncryptionMapper.new(key_repository)
)
res.append(
TicketHolderEmailProvided.new(
data: {
email: "kaka@dudu.me",
user_id: user_id
}
)
)
This predictably fails on invalid byte sequences in UTF-8:
/Users/mostlyobvious/.gem/ruby/4.0.1/gems/json-2.18.1/lib/json/common.rb:956:in 'JSON::Ext::Generator::State.generate': "\xF2" from ASCII-8BIT to UTF-8 (JSON::GeneratorError)
Invalid object: "\xF2\x19\ekX\x93\x81\x03j\xE4\x95\xF1G\xBB\x1A:\a\x91\x18\xFCV\xA9\xA3Qx\xDC/\x0F\x8A#l\x8E"
As a first step to fix it, let's inline the encryption mapper to see what transformations it consists of. Plus the cure for symbol-vs-string madness in PreserveTypes:
RubyEventStore::Client.new(
repository:
RubyEventStore::InMemoryRepository.new(
serializer: JSON
),
mapper:
Class
.new(
RubyEventStore::Mappers::PipelineMapper
) do
def initialize(key_repository)
super(
RubyEventStore::Mappers::Pipeline.new(
RubyEventStore::Mappers::Transformation::Encryption.new(
key_repository,
serializer:
RubyEventStore::Serializers::YAML,
forgotten_data:
RubyEventStore::Mappers::ForgottenData.new
),
RubyEventStore::Mappers::Transformation::PreserveTypes.new.register(
Symbol,
serializer: ->(v) do
v.to_s
end,
deserializer: ->(v) do
v.to_sym
end
),
RubyEventStore::Mappers::Transformation::SymbolizeMetadataKeys.new
)
)
end
end
.new(key_repository)
)
RubyEventStore::Client.new(
repository: RubyEventStore::InMemoryRepository.new(serializer: JSON),
mapper:
Class
.new(RubyEventStore::Mappers::PipelineMapper) do
def initialize(key_repository)
super(
RubyEventStore::Mappers::Pipeline.new(
RubyEventStore::Mappers::Transformation::Encryption.new(
key_repository,
serializer: RubyEventStore::Serializers::YAML,
forgotten_data: RubyEventStore::Mappers::ForgottenData.new
),
RubyEventStore::Mappers::Transformation::PreserveTypes.new.register(
Symbol,
serializer: ->(v) { v.to_s },
deserializer: ->(v) { v.to_sym }
),
RubyEventStore::Mappers::Transformation::SymbolizeMetadataKeys.new
)
)
end
end
.new(key_repository)
)
With that in place, let's insert a transformation that encodes the encrypted payload into base64. It's the encoder available in the standard library.
The transformation is bi-directional and we need it right after encryption in the list:
- to encode ciphertext right before passing to the storage serializer
- to decode base64 text into ciphertext just before decryption when loading a deserialized event from storage
For the record, not only the encrypted data fields need encoding. There's also the initialization vector for the cipher that is stored in the metadata in raw bytes:
#<RubyEventStore::Record:0x0000000124a17730
@data={email: "\x16\xE8v\xD6\xA1Y\xE1\xED\xF0\xC2#\xBE\xEF\xAF\xAC%\xE6\xB5[o\xF2k\b\x97\xAC\xD4\x8Bj\xD9p\xCA?", user_id: 2137},
@event_id="4a21667a-20a5-4be3-93be-b69025d2fcdf",
@event_type="TicketHolderEmailProvided",
@metadata=
{correlation_id: "594128fa-c8bc-44b8-b13e-8f8d21ce6825",
encryption: {email: {cipher: "aes-256-cbc", iv: "R.c\xD1F\x86\x87\x9B\x00\x95\xE7O\x10$\x8B\xF7", identifier: 2137}}},
@serialized_records={},
@timestamp=2026-04-01 10:55:01.380733 UTC,
@valid_at=2026-04-01 10:55:01.380733 UTC>
Here's the transformation to encode binary in its entirety. The record containing event data and metadata is processed according to the encryption schema, which describes what fields are expected to be a ciphertext. Each of these fields has an additional description in the metadata that needs to be taken care of too. On dump there's encoding. On load we're decoding.
RubyEventStore::Client.new(
repository:
RubyEventStore::InMemoryRepository.new(
serializer: JSON
),
mapper:
Class
.new(
RubyEventStore::Mappers::PipelineMapper
) do
def initialize(key_repository)
super(
RubyEventStore::Mappers::Pipeline.new(
RubyEventStore::Mappers::Transformation::Encryption.new(
key_repository,
serializer:
RubyEventStore::Serializers::YAML,
forgotten_data:
RubyEventStore::Mappers::ForgottenData.new
),
Class
.new do
def dump(record)
RubyEventStore::Record.new(
event_id:
record.event_id,
event_type:
record.event_type,
data:
process_data(
deep_dup(
record.data
),
encryption_schema(
Object.const_get(
record.event_type
)
),
Base64.method(
:strict_encode64
)
),
metadata:
process_metadata(
deep_dup(
record.metadata
),
encryption_schema(
Object.const_get(
record.event_type
)
),
Base64.method(
:strict_encode64
)
),
timestamp:
record.timestamp,
valid_at:
record.valid_at
)
end
def load(record)
RubyEventStore::Record.new(
event_id:
record.event_id,
event_type:
record.event_type,
data:
process_data(
deep_dup(
record.data
),
encryption_schema(
Object.const_get(
record.event_type
)
),
Base64.method(
:strict_decode64
)
),
metadata:
process_metadata(
deep_dup(
record.metadata
),
encryption_schema(
Object.const_get(
record.event_type
)
),
Base64.method(
:strict_decode64
)
),
timestamp:
record.timestamp,
valid_at:
record.valid_at
)
end
private
def encryption_schema(
event_class
)
if event_class.respond_to?(
:encryption_schema
)
event_class.encryption_schema
else
{}
end
end
def deep_dup(hash)
duplicate = hash.dup
duplicate.each do |k, v|
duplicate[k] = (
if v.instance_of?(
Hash
)
deep_dup(v)
else
v
end
)
end
end
def process_data(
data,
schema,
method
)
schema.reduce(
data
) do |acc, (key, _)|
acc[
key
] = method.call(
data[key]
) if data.has_key?(
key
)
acc
end
end
def process_metadata(
metadata,
schema,
method
)
schema.reduce(
metadata
) do |acc, (key, _)|
acc[:encryption][
key
][
:iv
] = method.call(
metadata[
:encryption
][
key
][
:iv
]
) if metadata[
:encryption
].has_key?(key)
acc
end
end
end
.new,
RubyEventStore::Mappers::Transformation::PreserveTypes.new.register(
Symbol,
serializer: ->(v) do
v.to_s
end,
deserializer: ->(v) do
v.to_sym
end
),
RubyEventStore::Mappers::Transformation::SymbolizeMetadataKeys.new
)
)
end
end
.new(key_repository)
)
RubyEventStore::Client.new(
repository: RubyEventStore::InMemoryRepository.new(serializer: JSON),
mapper:
Class
.new(RubyEventStore::Mappers::PipelineMapper) do
def initialize(key_repository)
super(
RubyEventStore::Mappers::Pipeline.new(
RubyEventStore::Mappers::Transformation::Encryption.new(
key_repository,
serializer: RubyEventStore::Serializers::YAML,
forgotten_data: RubyEventStore::Mappers::ForgottenData.new
),
Class
.new do
def dump(record)
RubyEventStore::Record.new(
event_id: record.event_id,
event_type: record.event_type,
data:
process_data(
deep_dup(record.data),
encryption_schema(
Object.const_get(record.event_type)
),
Base64.method(:strict_encode64)
),
metadata:
process_metadata(
deep_dup(record.metadata),
encryption_schema(
Object.const_get(record.event_type)
),
Base64.method(:strict_encode64)
),
timestamp: record.timestamp,
valid_at: record.valid_at
)
end
def load(record)
RubyEventStore::Record.new(
event_id: record.event_id,
event_type: record.event_type,
data:
process_data(
deep_dup(record.data),
encryption_schema(
Object.const_get(record.event_type)
),
Base64.method(:strict_decode64)
),
metadata:
process_metadata(
deep_dup(record.metadata),
encryption_schema(
Object.const_get(record.event_type)
),
Base64.method(:strict_decode64)
),
timestamp: record.timestamp,
valid_at: record.valid_at
)
end
private
def encryption_schema(event_class)
if event_class.respond_to?(:encryption_schema)
event_class.encryption_schema
else
{}
end
end
def deep_dup(hash)
duplicate = hash.dup
duplicate.each do |k, v|
duplicate[k] = v.instance_of?(Hash) ? deep_dup(v) : v
end
end
def process_data(data, schema, method)
schema.reduce(data) do |acc, (key, _)|
acc[key] = method.call(data[key]) if data.has_key?(key)
acc
end
end
def process_metadata(metadata, schema, method)
schema.reduce(metadata) do |acc, (key, _)|
acc[:encryption][key][:iv] = method.call(
metadata[:encryption][key][:iv]
) if metadata[:encryption].has_key?(key)
acc
end
end
end
.new,
RubyEventStore::Mappers::Transformation::PreserveTypes.new.register(
Symbol,
serializer: ->(v) { v.to_s },
deserializer: ->(v) { v.to_sym }
),
RubyEventStore::Mappers::Transformation::SymbolizeMetadataKeys.new
)
)
end
end
.new(key_repository)
)
I don't like how inlining a mapper just to add one transformation made the whole thing awfully verbose and exposed deep namespaces. That's the ugly side of it.
What I wanted to show is that, despite the verbosity, RES is quite malleable when it needs to be adapted to a very specific use case.
I hope this post helps at least one person — Juna, who kindly asked for it on RES Discord and Discussions.
Happy hacking!