greyson/playpen
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
playpen/tools/gen-emoji.py
T
Greyson Parrelli 7414c89fe6 Add notifications.
2026-08-26 23:24:14 -04:00

525 lines
20 KiB
Python
Executable File
Raw Permalink Blame History

#!/usr/bin/env python3
"""Regenerate src/emoji.zig from Unicode and CLDR data.
tools/gen-emoji.py # fetch the data, write src/emoji.zig
tools/gen-emoji.py --check # fail if the checked-in file is stale
tools/gen-emoji.py --emoji-test PATH --annotations PATH --derived PATH
The table it writes is committed, so this runs by hand rather than as part of
the build: a build that needs the network to compile is a build that fails on a
train. Re-run it when a new Unicode version lands, or when you want to add a
synonym to SYNONYMS below.
Three inputs, all canonical:
emoji-test.txt Unicode's own ordering, grouped and subgrouped, with
the fully-qualified form of every RGI emoji. This is
what every emoji keyboard is laid out from.
annotations/en.xml CLDR's English keywords per emoji — the synonyms that
make search work ("grin" finding the grinning face).
annotationsDerived/en.xml
The same for sequences CLDR derives rather than names
outright, which is most of the ZWJ ones.
Skin-tone variants are left out. Unicode lists 3944 fully-qualified emoji and
just over half of those are the same gesture five more times; a grid of them is
harder to look through, not more complete. Every base glyph is present, which
is the same choice GTK's own emoji chooser, iOS and Slack all make.
So is anything newer than MAX_VERSION. Color emoji fonts trail Unicode by a
year or two, and a glyph the font has never heard of draws as a hex-digit box —
which in a picker reads as a bug rather than as a font that needs updating. At
the time of writing, Noto Color Emoji could draw all but 7 of the E17.0
additions and everything older. Raise MAX_VERSION when fonts have caught up;
`--max-version 99` turns the cutoff off entirely.
"""
import argparse
import collections
import os
import re
import sys
import urllib.request
# "latest" rather than a pinned version: the per-version directories under
# /Public/emoji/ stop being published once a version ships, so a pinned URL is
# one that 404s later. What version "latest" turned out to be is recorded in the
# generated file's header, and MAX_VERSION below is what actually decides the
# contents.
EMOJI_TEST = "https://unicode.org/Public/emoji/latest/emoji-test.txt"
ANNOTATIONS = "https://raw.githubusercontent.com/unicode-org/cldr/main/common/annotations/en.xml"
DERIVED = "https://raw.githubusercontent.com/unicode-org/cldr/main/common/annotationsDerived/en.xml"
# Terms no data file will ever give you: what a glyph means to someone labeling
# a terminal tab. CLDR knows a rocket is a "space ship"; it does not know it is
# what you reach for when the tab is a deploy.
SYNONYMS = {
"🚀": "deploy ship launch release",
"📦": "release bundle package ship artifact",
"🏷️": "version tag release",
"🎉": "ship shipped release celebrate tada",
"🐛": "bug issue defect regression",
"🩹": "hotfix patch bandaid",
"🔥": "hot lit onfire urgent",
"💥": "crash boom broke",
"💀": "dead deprecated killed",
"🦖": "legacy ancient",
"⚡": "zap fast perf quick",
"⏱️": "benchmark perf timing latency",
"🧪": "test experiment trial",
"🧫": "test lab",
"🔬": "inspect investigate research",
"🔍": "search find grep lookup",
"👀": "review look watch eyes",
"✅": "pass passing green done ok",
"❌": "fail failing red broken",
"⚠️": "warn warning caution",
"🚧": "wip work in progress unfinished",
"🏗️": "wip building scaffolding",
"🧹": "cleanup refactor tidy sweep",
"♻️": "refactor reuse recycle",
"🔄": "sync retry refresh reload",
"🔀": "shuffle random merge",
"⚙️": "settings config gear options",
"🔒": "secure private locked",
"🔓": "public unlocked open",
"🔑": "auth key password access token secret",
"🛡️": "security hardening defense",
"📈": "metrics growth up analytics",
"📉": "metrics down regression analytics",
"📊": "metrics analytics stats dashboard",
"🔔": "alert notification ping",
"🔕": "mute silence snooze",
"💻": "dev code local laptop",
"🖥️": "server desktop box host",
"🗄️": "database storage archive",
"🐳": "docker container whale",
"🐧": "linux tux",
"🦀": "rust cargo",
"🐍": "python",
"🐫": "perl camel",
"☕": "java coffee jvm",
"💎": "ruby gem",
"🐘": "postgres php elephant memory",
"🍎": "apple mac macos",
"🪟": "windows",
"🤖": "bot agent ai automation claude",
"🧑‍💻": "dev developer engineer coding",
"📝": "todo note notes scratch",
"🗑️": "delete trash remove drop",
"🌈": "pride rainbow",
"🎯": "goal target focus",
"🧭": "navigate direction bearings",
"🚦": "ci status pipeline signal",
"🏁": "done finished race",
"🧊": "freeze frozen cold pinned",
"🕸️": "stale abandoned cobweb",
"🧠": "think smart reasoning",
}
# Nothing in a keyword list should be a word you cannot type. Everything else is
# kept, including the non-ASCII names of places, because a term only has to
# match to be worth carrying.
STRIP = re.compile(r"[\"\\|,:;()\[\]{}!?“”]+")
SKIN_TONES = range(0x1F3FB, 0x1F400)
# Newest emoji version to include. See the note at the top of this file.
MAX_VERSION = "16.0"
def version_tuple(text):
""""E16.0" or "16" as something comparable. Unknown sorts newest."""
try:
return tuple(int(p) for p in text.lstrip("Ee").split("."))
except ValueError:
return (999,)
def read(source):
"""Contents of a URL or a path, whichever `source` looks like."""
if source.startswith(("http://", "https://")):
with urllib.request.urlopen(source, timeout=60) as response:
return response.read().decode("utf-8")
with open(source, encoding="utf-8") as handle:
return handle.read()
def parse_emoji_test(text, max_version):
"""Unicode's list, in Unicode's order: (glyph, group, subgroup, name)."""
out = []
skipped = []
group = subgroup = ""
version = "unknown"
for line in text.splitlines():
if line.startswith("# Version:"):
version = line.split(":", 1)[1].strip()
continue
if line.startswith("# group:"):
group = line.split(":", 1)[1].strip()
continue
if line.startswith("# subgroup:"):
subgroup = line.split(":", 1)[1].strip()
continue
if not line.strip() or line.startswith("#"):
continue
codepoints, rest = line.split(";", 1)
status, _, comment = rest.partition("#")
if status.strip() != "fully-qualified":
continue
points = [int(c, 16) for c in codepoints.split()]
if any(p in SKIN_TONES for p in points):
continue
# The comment is "<glyph> E<version> <name>": the version each sequence
# was introduced in, which is what the font-coverage cutoff reads.
parts = comment.strip().split(" ", 2)
introduced = parts[1] if len(parts) > 1 else "E0"
name = parts[2] if len(parts) > 2 else ""
if version_tuple(introduced) > version_tuple(max_version):
skipped.append(introduced)
continue
out.append(("".join(chr(p) for p in points), group, subgroup, name))
return version, out, collections.Counter(skipped)
def parse_annotations(text):
"""CLDR's keywords and short name per emoji, merged into one dict."""
keywords = {}
names = {}
for match in re.finditer(
r'<annotation cp="([^"]*)"(?P<tts> type="tts")?>(.*?)</annotation>',
text,
re.DOTALL,
):
cp, tts, body = match.group(1), match.group("tts"), match.group(3)
body = (
body.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", '"')
)
if tts:
names[cp] = body.strip()
else:
keywords.setdefault(cp, []).extend(p.strip() for p in body.split("|"))
return keywords, names
def escape(text):
"""`text` as a Zig string literal body.
Only the display name needs this. Keywords go through `tokenize`, which
drops every character that would have to be escaped in the first place.
"""
return text.replace("\\", "\\\\").replace('"', '\\"')
def tokenize(*phrases):
"""Lowercase words from `phrases`, deduplicated, in first-seen order."""
seen = []
for phrase in phrases:
if not phrase:
continue
cleaned = STRIP.sub(" ", phrase.replace("-", " ").replace("_", " "))
for word in cleaned.lower().split():
# A bare "&" survives group names like "Smileys & Emotion".
if word == "&":
continue
if word not in seen:
seen.append(word)
return seen
def build(emoji_test, annotations, derived, max_version):
version, entries, skipped = parse_emoji_test(emoji_test, max_version)
keywords, names = parse_annotations(annotations)
derived_keywords, derived_names = parse_annotations(derived)
for source, into in ((derived_keywords, keywords), (derived_names, names)):
for cp, value in source.items():
if cp not in into:
into[cp] = value
rows = []
for glyph, group, subgroup, name in entries:
# CLDR keys on the emoji without its presentation selector as often as
# with it, so try both before giving up and using Unicode's own name.
bare = glyph.replace("\ufe0f", "")
short = names.get(glyph) or names.get(bare) or name
words = tokenize(
short,
" ".join(keywords.get(glyph, keywords.get(bare, []))),
# The group and subgroup make whole shelves reachable by name:
# "flags", "fruit", "arrow", "zodiac".
subgroup,
group,
SYNONYMS.get(glyph, ""),
)
rows.append((glyph, group, escape(short), " ".join(words)))
return version, rows, skipped
def render(version, rows, max_version):
out = []
out.append(
HEADER
% {"version": version, "count": len(rows), "max_version": max_version}
)
group = None
for glyph, row_group, short, words in rows:
if row_group != group:
group = row_group
if out[-1].endswith("},\n"):
out.append("\n")
out.append(" // ---- %s %s\n" % (group, "-" * max(3, 60 - len(group))))
out.append(
' .{ .glyph = "%s", .name = "%s", .keywords = "%s" },\n'
% (glyph, short, words)
)
out.append("};\n")
out.append(TESTS)
return "".join(out)
HEADER = '''//! Every emoji a tab can wear in place of its icon, and the search that finds
//! them.
//!
//! Generated — run `tools/gen-emoji.py` rather than editing this file. It reads
//! Unicode's `emoji-test.txt` (currently version %(version)s) for the set and its
//! ordering, and CLDR's English annotations for the keywords, so the picker is
//! laid out and searchable the same way every other emoji keyboard is.
//!
//! %(count)d entries: the RGI set up to emoji %(max_version)s, minus skin-tone variants.
//! Unicode lists nearly twice this many fully-qualified sequences and the
//! difference is almost entirely the same gesture in five tones, which makes a
//! grid longer to look through without making it more complete. The version
//! cutoff is there because color emoji fonts trail Unicode, and a glyph the
//! font has never heard of draws as a hex-digit box; the generator's header
//! explains how to raise it.
//!
//! Keywords are CLDR's, plus the group and subgroup a glyph belongs to — so
//! "fruit", "arrow" and "flags" each bring back a whole shelf — plus a table of
//! synonyms in the generator for the words a terminal user would actually type:
//! a rocket answers to "deploy", a bandage to "hotfix", a whale to "docker".
const std = @import("std");
pub const Emoji = struct {
/// The glyph itself, NUL-terminated so it can go straight into a label.
glyph: [:0]const u8,
/// CLDR's short name — "red apple", "flag: Kenya" — as the picker shows it
/// in a tooltip. NUL-terminated for the same reason as the glyph, and kept
/// apart from `keywords` because a name is one phrase and a keyword list is
/// twenty words: readable in a tooltip, and unreadable in one.
name: [:0]const u8,
/// Space-separated search terms, lowercase, the name's own words among them.
keywords: []const u8,
};
/// Whether `emoji` should show for `query`.
///
/// Every whitespace-separated term has to match somewhere, which is what makes
/// "red circle" and "circle red" both land on the same glyph while "red" alone
/// still brings back the whole family. An empty query matches everything, so
/// the unfiltered grid falls out of the same path as a filtered one.
pub fn matches(emoji: Emoji, query: []const u8) bool {
var terms = std.mem.tokenizeAny(u8, query, " \\t");
while (terms.next()) |term| {
if (std.ascii.indexOfIgnoreCase(emoji.keywords, term) == null) return false;
}
return true;
}
/// The table's own copy of `glyph`, or null if it isn't one of ours.
///
/// What comes back is static, so a caller can hold it for as long as it likes
/// with nothing to free — which is what lets a glyph read out of a config file
/// be handed to something that wants a pointer it can keep. A glyph the table
/// doesn't have reads as no choice at all: the picker is where these come from,
/// so anything else is a hand-edited file naming something this build can't
/// draw at the size the row wants.
pub fn lookup(glyph: []const u8) ?[:0]const u8 {
for (table) |entry| {
if (std.mem.eql(u8, entry.glyph, glyph)) return entry.glyph;
}
return null;
}
/// Every emoji the picker offers, in Unicode's order.
pub const table = [_]Emoji{
'''
TESTS = '''
// -------------------------------------------------------------------------
// Tests
//
// The generator is what keeps the table right; these are the properties the
// picker depends on it having. They run against whatever is checked in, so a
// bad regeneration fails here rather than in the dialog.
test "every entry is usable" {
for (table) |entry| {
try std.testing.expect(entry.glyph.len > 0);
try std.testing.expect(std.unicode.utf8ValidateSlice(entry.glyph));
// Long enough for the longest RGI sequence, short enough that nothing
// here is quietly a whole word.
try std.testing.expect(entry.glyph.len <= 40);
try std.testing.expect(entry.name.len > 0);
try std.testing.expect(std.unicode.utf8ValidateSlice(entry.name));
try std.testing.expect(entry.keywords.len > 0);
for (entry.keywords) |c| try std.testing.expect(!std.ascii.isUpper(c));
}
}
test "glyphs are distinct" {
// A child's position in the picker's grid is how the dialog names the entry
// it shows, and `indexOf` maps the other way by comparing glyphs. Both stop
// being true if a glyph appears twice.
for (table, 0..) |entry, i| {
for (table[i + 1 ..]) |other| {
try std.testing.expect(!std.mem.eql(u8, entry.glyph, other.glyph));
}
}
}
test "the whole set is here" {
// Unicode 17 has 1914 RGI emoji once skin-tone variants are folded away.
// A table that has drifted far from that has lost a group.
try std.testing.expect(table.len > 1800);
}
/// The one entry a search test leans on, looked up rather than indexed so that
/// regenerating the table doesn't rewrite the tests.
fn find(glyph: []const u8) Emoji {
for (table) |entry| {
if (std.mem.eql(u8, entry.glyph, glyph)) return entry;
}
unreachable;
}
test "an empty query matches everything" {
for (table) |entry| {
try std.testing.expect(matches(entry, ""));
try std.testing.expect(matches(entry, " "));
}
}
test "a term matches part of a keyword, in any case" {
const rocket = find("\\u{1F680}");
try std.testing.expect(matches(rocket, "rocket"));
try std.testing.expect(matches(rocket, "ROCKET"));
try std.testing.expect(matches(rocket, "Rock"));
try std.testing.expect(!matches(rocket, "banana"));
}
test "every term has to match, in any order" {
const green = find("\\u{1F7E2}");
try std.testing.expect(matches(green, "green circle"));
try std.testing.expect(matches(green, "circle green"));
try std.testing.expect(matches(green, " green circle "));
// The second term is what rules the other circles out.
const red = find("\\u{1F534}");
try std.testing.expect(matches(red, "circle"));
try std.testing.expect(!matches(red, "green circle"));
// And a term matching nothing rules out a glyph the rest of the query hit.
try std.testing.expect(!matches(green, "green circle sideways"));
}
test "names are the phrase, not the keyword list" {
// What separates the two fields: the tooltip stays short enough to read.
for (table) |entry| try std.testing.expect(entry.name.len <= 64);
try std.testing.expectEqualStrings("red apple", find("\\u{1F34E}").name);
try std.testing.expectEqualStrings("rocket", find("\\u{1F680}").name);
}
test "CLDR keywords reach a glyph its name would not" {
try std.testing.expect(matches(find("\\u{1F600}"), "grin"));
try std.testing.expect(matches(find("\\u{1F60A}"), "blush"));
try std.testing.expect(matches(find("\\u{1F4A9}"), "poop"));
}
test "a group or subgroup brings back its whole shelf" {
try std.testing.expect(matches(find("\\u{1F34E}"), "fruit"));
try std.testing.expect(matches(find("\\u{1F1FA}\\u{1F1F8}"), "flags"));
try std.testing.expect(matches(find("\\u{2B06}\\u{FE0F}"), "arrow"));
}
test "synonyms reach the glyph the data would not" {
try std.testing.expect(matches(find("\\u{1F680}"), "deploy"));
try std.testing.expect(matches(find("\\u{1FA79}"), "hotfix"));
try std.testing.expect(matches(find("\\u{1F433}"), "docker"));
try std.testing.expect(matches(find("\\u{1F427}"), "linux"));
try std.testing.expect(matches(find("\\u{1F980}"), "rust"));
try std.testing.expect(matches(find("\\u{2705}"), "pass"));
}
'''
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--emoji-test", default=EMOJI_TEST)
parser.add_argument("--annotations", default=ANNOTATIONS)
parser.add_argument("--derived", default=DERIVED)
parser.add_argument("--out", default=None)
parser.add_argument(
"--max-version",
default=MAX_VERSION,
help="newest emoji version to include (default %s)" % MAX_VERSION,
)
parser.add_argument(
"--check",
action="store_true",
help="exit non-zero if the file on disk is not what we would write",
)
args = parser.parse_args()
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
out = args.out or os.path.join(root, "src", "emoji.zig")
version, rows, skipped = build(
read(args.emoji_test),
read(args.annotations),
read(args.derived),
args.max_version,
)
text = render(version, rows, args.max_version)
if args.check:
current = open(out, encoding="utf-8").read() if os.path.exists(out) else ""
if current != text:
print("%s is stale; re-run tools/gen-emoji.py" % out, file=sys.stderr)
return 1
print("%s is up to date (%d emoji)" % (out, len(rows)))
return 0
with open(out, "w", encoding="utf-8") as handle:
handle.write(text)
print("wrote %s: %d emoji from Unicode %s" % (out, len(rows), version))
for introduced, count in sorted(skipped.items()):
print(" held back %d from %s (newer than %s)" % (count, introduced, args.max_version))
return 0
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink