A fast and flexible unicode library, fully configurable at build time.
const uucode = @import("uucode");
var cp: u21 = undefined; // A u21 is the type for a Unicode code point
//////////////////////
// `get` properties
cp = 0x2200; // β
uucode.get(.general_category, cp) // .symbol_math
cp = 0x03C2; // Ο
uucode.get(.simple_uppercase_mapping, cp) // U+03A3 == Ξ£
cp = 0x21C1; // β
uucode.get(.name, cp) // "RIGHTWARDS HARPOON WITH BARB DOWNWARDS"
// Many of the []const u21 fields need a single item buffer passed to `with`:
var buffer: [1]u21 = undefined;
cp = 0x00DF; // Γ
uucode.get(.uppercase_mapping, cp).with(&buffer, cp) // "SS"
//////////////////////
// `getAll` to get a group of properties for a code point together.
cp = 0x03C2; // Ο
// The first argument is the name/index of the table.
const data = uucode.getAll("0", cp);
data.simple_uppercase_mapping // U+03A3 == Ξ£
data.general_category // .letter_lowercase
//////////////////////
// utf8.Iterator
var it = uucode.utf8.Iterator.init("ππ
π»πΊ");
it.next(); // 0x1F600
it.i; // 4 (bytes into the utf8 string)
it.peek(); // 0x1F605
it.next(); // 0x1F605
it.next(); // 0x1F63B
it.next(); // 0x1F47A
//////////////////////
// grapheme.Iterator / grapheme.utf8Iterator
var it = uucode.grapheme.utf8Iterator("π©π½βππ¨ππ¨π»βπΌ")
// (which is equivalent to:)
var it = uucode.grapheme.Iterator(uccode.utf8.Iterator).init(.init("π©π½βππ¨ππ¨π»βπΌ"));
// `nextCodePoint` advances one code point at a time, indicating a new grapheme
// with `is_break = true`.
it.nextCodePoint(); // { .code_point = 0x1F469; .is_break = false } // π©
it.i; // 4 (bytes into the utf8 string)
it.peekCodePoint(); // { .code_point = 0x1F3FD; .is_break = false } // π½
it.nextCodePoint(); // { .code_point = 0x1F3FD; .is_break = false } // π½
it.nextCodePoint(); // { .code_point = 0x200D; .is_break = false } // Zero width joiner
it.nextCodePoint(); // { .code_point = 0x1F680; .is_break = true } // π
// `nextGrapheme` advances until the start of the next grapheme cluster
const result = it.nextGrapheme(); // { .start = 15; .end = 23 }
it.i; // "π©π½βππ¨π".len
str[result.?.start..result.?.end]; // "π¨π"
const result = it.peekGrapheme();
str[result.?.start..result.?.end]; // "π¨π»βπΌ"
//////////////////////
// grapheme.isBreak
var break_state: uucode.grapheme.BreakState = .default;
var cp1: u21 = 0x1F469; // π©
var cp2: u21 = 0x1F3FD; // π½
uucode.grapheme.isBreak(cp1, cp2, &break_state); // false
cp1 = cp2;
cp2 = 0x200D; // Zero width joiner
uucode.grapheme.isBreak(cp1, cp2, &break_state); // false
cp1 = cp2;
cp2 = 0x1F680; // π
// The combined grapheme cluster is π©π½βπ (woman astronaut)
uucode.grapheme.isBreak(cp1, cp2, &break_state); // false
cp1 = cp2;
cp2 = 0x1F468; // π¨
uucode.grapheme.isBreak(cp1, cp2, &break_state); // true
//////////////////////
// x.grapheme.wcwidth{,Next,Remaining} / x.grapheme.utf8Wcwidth
const str = "oΜπ¨π»ββ€οΈβπ¨πΏ_";
var it = uucode.grapheme.utf8Iterator(str);
// Requires the `wcwidth` builtin extension (see below)
uucode.x.grapheme.wcwidth(it); // 1 for 'oΜ'
uucode.x.grapheme.wcwidthNext(&it); // 1 for 'oΜ'
const result = it.peekGrapheme();
str[result.?.start..result.?.end]; // "π¨π»ββ€οΈβπ¨πΏ"
uucode.x.grapheme.wcwidthRemaining(&it); // 3 for "π¨π»ββ€οΈβπ¨πΏ_"
uucode.x.grapheme.utf8Wcwidth(str); // 4 for the whole string
//////////////////////
// TypeOf / TypeOfAll / hasField
uucode.TypeOf(.general_category) // uucode.types.GeneralCategory
uucode.TypeOfAll("0") // @TypeOf(uucode.getAll("0"))
uucode.hasField("is_emoji") // true if `is_emoji` is in one of your tablesSee src/config.zig for the names of all fields.
Only include the Unicode fields you actually use:
// In `build.zig`:
if (b.lazyDependency("uucode", .{
.target = target,
.optimize = optimize,
.fields = @as([]const []const u8, &.{
"name",
"general_category",
"case_folding_simple",
"is_alphabetic",
// ...
}),
})) |dep| {
step.root_module.addImport("uucode", dep.module("uucode"));
}If you forget to add a field you're using, you'll get an error such as:
src/root.zig:36:29: error: enum 'get.FieldEnum__enum_8678' has no member named 'is_not_a_configured_field'
try testing.expect(get(.is_not_a_configured_field, 65));
~^~~~~~~~~~~~~~~~~~~~~~~~~
Fields can be split into multiple tables using fields_0 through fields_9, to optimize how fields are stored and accessed (with no code changes needed).
// In `build.zig`:
if (b.lazyDependency("uucode", .{
.target = target,
.optimize = optimize,
.fields_0 = @as([]const []const u8, &.{
"general_category",
"case_folding_simple",
"is_alphabetic",
}),
.fields_1 = @as([]const []const u8, &.{
// ...
}),
.fields_2 = @as([]const []const u8, &.{
// ...
}),
// ... `fields_3` to `fields_9`
})) |dep| {
step.root_module.addImport("uucode", dep.module("uucode"));
}uucode includes builtin extensions that add derived properties. Use extensions or extensions_0 through extensions_9 to include them:
// In `build.zig`:
if (b.lazyDependency("uucode", .{
.target = target,
.optimize = optimize,
.extensions = @as([]const []const u8, &.{
"wcwidth",
}),
.fields = @as([]const []const u8, &.{
// Make sure to also include the extension's fields here:
"wcwidth_standalone",
"wcwidth_zero_in_grapheme",
...
"general_category",
}),
})) |dep| {
step.root_module.addImport("uucode", dep.module("uucode"));
}
// In your code:
uucode.get(.wcwidth_standalone, 0x26F5) // β΅ == 2See src/x/config.x.zig for the full list of builtin extensions.
///////////////////////////////////////////////////////////
// In `build.zig`:
b.dependency("uucode", .{
.target = target,
.optimize = optimize,
.build_config_path = b.path("src/build/uucode_config.zig"),
// Alternatively, use a string literal:
//.@"build_config.zig" = "..."
})
///////////////////////////////////////////////////////////
// In `src/build/uucode_config.zig`:
const std = @import("std");
const config = @import("config.zig");
// Use `config.x.zig` for builtin extensions:
const config_x = @import("config.x.zig");
const d = config.default;
const wcwidth = config_x.wcwidth;
// Or build your own extension:
const emoji_odd_or_even = config.Extension{
.inputs = &.{"is_emoji"},
.compute = &computeEmojiOddOrEven,
.fields = &.{
.{ .name = "emoji_odd_or_even", .type = EmojiOddOrEven },
},
};
fn computeEmojiOddOrEven(
allocator: std.mem.Allocator,
cp: u21,
data: anytype,
backing: anytype,
tracking: anytype,
) std.mem.Allocator.Error!void {
// allocator is an ArenaAllocator, so don't worry about freeing
_ = allocator;
// backing and tracking are only used for slice types (see
// src/build/test_build_config.zig for examples).
_ = backing;
_ = tracking;
if (!data.is_emoji) {
data.emoji_odd_or_even = .not_emoji;
} else if (cp % 2 == 0) {
data.emoji_odd_or_even = .even_emoji;
} else {
data.emoji_odd_or_even = .odd_emoji;
}
}
// Types must be marked `pub`
pub const EmojiOddOrEven = enum(u2) {
not_emoji,
even_emoji,
odd_emoji,
};
// Configure tables with the `tables` declaration.
// The only required field is `fields`, and the rest have reasonable defaults.
pub const tables = [_]config.Table{
.{
// Optional name, to be able to `getAll("foo")` rather than e.g.
// `getAll("0")`
.name = "foo",
// A two stage table can be slightly faster if the data is small. The
// default `.auto` will pick a reasonable value, but to get the
// absolute best performance run benchmarks with `.two` or `.three`
// on realistic data.
.stages = .three,
// The default `.auto` value decide whether the final data stage struct
// should be a `packed struct` (.@"packed") or a regular Zig `struct`.
.packing = .unpacked,
.extensions = &.{
emoji_odd_or_even,
wcwidth,
},
.fields = &.{
// Don't forget to include the extension's fields here.
emoji_odd_or_even.field("emoji_odd_or_even"),
wcwidth.field("wcwidth_standalone"),
wcwidth.field("wcwidth_zero_in_grapheme"),
// See `src/config.zig` for everything that can be overriden.
// In this example, we're embedding 15 bytes into the `stage3` data,
// and only names longer than that need to use the `backing` slice.
d.field("name").override(.{
.embedded_len = 15,
.max_offset = 986096, // run once to get the correct number
}),
d.field("general_category"),
d.field("block"),
// ...
},
},
};
// Turn on debug logging:
pub const log_level = .debug;
///////////////////////////////////////////////////////////
// In your code:
const uucode = @import("uucode");
uucode.get(.wcwidth_standalone, 0x26F5) // β΅ == 2
uucode.get(.emoji_odd_or_even, 0x1F34B) // π == .odd_emoji
For more examples of advanced configuration and extensions, see src/build/test_build_config.zig.
The architecture works in a few layers:
- Layer 1 - src/build/Ucd.zig: Parses the Unicode Character Database (UCD).
- Layer 2 - src/build/tables.zig: Generates table data written to a zig file.
- Layer 3 - src/root.zig: Exposes methods to fetch information from the built tables.
Other important files:
- src/config.zig: Includes configuration for all fields and types for table and extension configuration.
- src/types.zig: Includes special types for handling slice, optional, union, and "shift" types (u21 fields that often map to themselves).
- src/get.zig: Comptime heavy code to look up properties given a code point.
- src/utf8.zig: Extracts code points from utf-8 text.
- src/grapheme.zig: Includes grapheme break logic and a grapheme iterator.
uucode has a number of special types to handle certain unicode fields.
These fields are configured in src/config.zig with the Field struct:
pub const Field = struct {
name: [:0]const u8,
type: type,
// For Shift + Slice fields
cp_packing: CpPacking = .direct,
shift_low: isize = 0,
shift_high: isize = 0,
// For Slice fields
max_len: usize = 0,
max_offset: usize = 0,
embedded_len: usize = 0,
// For PackedOptional fields
min_value: isize = 0,
max_value: isize = 0,
...
};The special types are as follows.
This allows for packing an optional field, by fitting the "null" as the maxInt of the IntFittingRange between the configured min_value and max_value + 1.
This allows for including u21 fields without bloating the data required to store the field. A number of unicode fields default to mapping to themselves (e.g. simple_uppercase_mapping) or some other code point that might not be too far away in value. The Shift type internally stores the difference between the current code point and the u21 value, and at the time of get, adds this difference back to get the actual value.
To use a Shift field, the following need to be set:
.cp_packing = .shift.shift_low.shift_high
Temporarily configure shift_low to -@as(isize, config.max_code_point) and shift_high to config.max_code_point, then run the build to get the actual values.
error: Config for field 'uppercase_mapping_first_char' does not match actual. Set .shift_low = -64190, // change from -1114111
error: Config for field 'uppercase_mapping_first_char' does not match actual. Set .shift_high = 42561, // change from 1114111
thread 38963456 panic: Table config doesn't match actual. See above for details
This allows for including union fields in packed tables, or with shift fields. No special configuration is needed, unless there is a shift field, then the above shift_low and shift_high need to be set.
This allows for including slice fields ([]const T). Unsupported for packed tables. This can also include a shift for when the slice is u21, used only when the length is 1.
A Slice field stores its data as a union in either embedded in a fix-sized array (embedded: [embedded_len]T), in the shift field (for u21 slice of length 1), or as an offset: Offset pointing to a "backing" buffer.
To use a Slice field, the following need to be set:
.max_len: The maximum length of the slice across all code points..max_offset: The maximum offset in theoffsetfield (technically, the length of the backing buffer after iterating over all code points)..embedded_len: The length of the embedded array, defaulting to zero..shift_low: Needs to be configured as inShiftabove foru21element types..shift_high: Needs to be configured as inShiftabove foru21element types.
When using embedded_len > 0, the slice will be stored in the embedded array if it fits, otherwise in the backing buffer.
uucode began out of work on the Ghostty terminal on an issue to upgrade dependencies, where the experience modifying zg gave the confidence to build a fresh new library.
uucode builds upon the Unicode performance work done in Ghostty, as outlined in this excellent Devlog. The 3-stage lookup tables, as mentioned in that Devlog, come from this article.
uucode is available under an MIT License. See ./LICENSE.md for the license text and an index of licenses for code used in the repo.
See ./RESOURCES.md for a list of resources used to build uucode.